diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 593a8c0d72..3c58401f4a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -21,7 +21,7 @@ src/scripts/check-suid-contract.sh @clubanderson @hanthor @Danathar src/deploy/** @clubanderson @hanthor @Danathar # Egress proxy policy and GitHub API mediation -src/pkg/proxy/** @clubanderson @hanthor @Danathar +src/crux/proxy/** @clubanderson @hanthor @Danathar # Hub key, cookie, impersonation, OAuth, SSO, and session machinery src/pkg/hub/*key*.go @clubanderson @hanthor @Danathar diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 79d4ee9013..a8afa99dd5 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,14 @@ blank_issues_enabled: true contact_links: + - name: Report a security vulnerability + url: https://github.com/hivecommons/hive/security/advisories/new + about: Please do not file vulnerabilities as public issues. Report them privately as described in SECURITY.md. + - name: Contribute to a hive with your own agent + url: https://github.com/hivecommons/hive/blob/v5/src/docs/contributor-relay.md + about: Run the contributor relay to work a hive's backlog with the CLI agent on your own machine. - name: Join our Discord url: https://hivecommons.dev/discord about: Ask setup, contributor, and community questions in the Hive Commons Discord. + - name: Propose a project to join Hive Commons + url: https://github.com/hivecommons/.github/issues/new?template=join-hive-commons.yml + about: Project proposals for the Hive Commons family are handled in the hivecommons/.github repository. diff --git a/.github/release-lines.yml b/.github/release-lines.yml index a21a9124f7..337dd4f808 100644 --- a/.github/release-lines.yml +++ b/.github/release-lines.yml @@ -160,6 +160,15 @@ pinned: # watches (v4 reaches v6 through v5), and v6 is the merge TARGET, not a # trigger. v6-topup.yml: [v5, -v4] + # Publishes src/crux to hivecommons/crux (#14138). It watches the ONE line + # the published module is cut from — v5, where the nested module lives and + # where `crux/vX.Y.Z` tags are made. v4 is feature-frozen and has no + # src/crux; publishing from two lines would race on hivecommons/crux main. + crux-sync.yml: [-v4] + # Nightly + path-filtered Forgejo smoke (#14116). Its push/pull_request + # filters name v5 only: the forge.Merger it exercises lives on v5, and v4 is + # feature-frozen. + forgejo-smoke.yml: [-v4] # Workflows that deliberately do NOT pin to the release lines. Each must keep a # wildcard branch filter; if one is ever narrowed to a fixed list, the guard diff --git a/.github/scripts/affected-go-packages.sh b/.github/scripts/affected-go-packages.sh index 156282c657..eec1888e15 100755 --- a/.github/scripts/affected-go-packages.sh +++ b/.github/scripts/affected-go-packages.sh @@ -21,8 +21,9 @@ # # Anything this script cannot reason about falls back to `all`: a change # outside src/ (workflows, CI scripts, bin/, config/, Justfile, Makefile, -# dashboard/openapi.json — all read or executed by tests or shards), go.mod or -# go.sum, src/scripts/, the shared test harness under src/internal/, any +# dashboard/openapi.json — all read or executed by tests or shards), go.mod, +# go.sum, go.work or the nested Crux module's crux/go.mod and crux/go.sum, +# src/scripts/, the shared test harness under src/internal/, any # non-Go file under src/ (embedded static assets, docs, policies, fixtures — # several are read across packages), any testdata/ file, a .go file whose # directory is not a package in the module, an empty diff, or any git/go @@ -65,11 +66,14 @@ git rev-parse -q --verify "${head}^{commit}" >/dev/null || full "head revision $ changed=$(git diff --no-renames --name-only "$base" "$head") || full "git diff $base $head failed" [ -n "$changed" ] || full "empty changed-file list" -mod_path=$(go list -m 2>/dev/null) || full "go list -m failed" +# src/go.work puts the go tool in workspace mode: `go list -m` prints every +# workspace module and `./...` covers the root module only, so the nested +# Crux module (src/crux, #13536) is addressed explicitly as ./crux/... . +mod_path=$(GOWORK=off go list -m 2>/dev/null) || full "go list -m failed" # Directory -> import path for every package in the module. -dir_map=$(go list -f '{{.Dir}}|{{.ImportPath}}' ./... 2>/dev/null) || full "go list ./... failed" -[ -n "$dir_map" ] || full "go list ./... returned no packages" +dir_map=$(go list -f '{{.Dir}}|{{.ImportPath}}' ./... ./crux/... 2>/dev/null) || full "go list ./... ./crux/... failed" +[ -n "$dir_map" ] || full "go list ./... ./crux/... returned no packages" seeds="" rdep_seeds="" @@ -83,7 +87,7 @@ while IFS= read -r f; do esac rel=${f#src/} case "$rel" in - go.mod|go.sum) full "$f changes the module graph" ;; + go.mod|go.sum|go.work|go.work.sum|crux/go.mod|crux/go.sum) full "$f changes the module graph" ;; scripts/*) full "$f is a repository script" ;; internal/*) full "$f is part of the shared test harness" ;; testdata/*|*/testdata/*) full "$f is a test fixture (fixtures are read across packages)" ;; @@ -112,7 +116,7 @@ if [ -n "$seeds" ]; then # Every package, test variant ("p [p.test]", "p_test [p.test]") and test # binary ("p.test") with its transitive deps; a line's owning package is its # import path with the variant/binary decoration stripped. - graph=$(go list -test -f '{{.ImportPath}}|{{join .Deps "|"}}' ./... 2>/dev/null) || full "go list -test ./... failed" + graph=$(go list -test -f '{{.ImportPath}}|{{join .Deps "|"}}' ./... ./crux/... 2>/dev/null) || full "go list -test ./... ./crux/... failed" printf '%s' "$seeds" >> "$selected" awk -F'|' ' NR == FNR { if ($0 != "") seed[$0] = 1; next } @@ -144,7 +148,7 @@ if [ -n "$seeds" ]; then fi # Bound the answer by the package set the shards actually test. -universe=$(go list ./pkg/... ./cmd/... ./internal/... 2>/dev/null) || full "go list of the tested package set failed" +universe=$(go list ./pkg/... ./cmd/... ./internal/... ./crux/... 2>/dev/null) || full "go list of the tested package set failed" LC_ALL=C sort -u "$selected" | grep -Fx -f <(printf '%s\n' "$universe") > "$out" || true echo "affected-go-packages: $(grep -c . "$out" || true) of $(printf '%s\n' "$universe" | grep -c .) tested packages selected" >&2 diff --git a/.github/scripts/crux-scope.sh b/.github/scripts/crux-scope.sh index b85050a6c4..9897bbc8d7 100755 --- a/.github/scripts/crux-scope.sh +++ b/.github/scripts/crux-scope.sh @@ -26,25 +26,13 @@ set -uo pipefail base=${1:?base revision} head=${2:?head revision} -# The Crux package set: pkg/agent and its subpackages plus the runtime -# packages the RFC lists. KEEP IN SYNC with the "Crux boundary" header -# comment in .github/workflows/v2-tests.yml. +# The Crux package set: the nested Go module at src/crux (module +# github.com/hivecommons/hive/crux, #13536 step 4) — crux/agent and the +# runtime ring the RFC lists. KEEP IN SYNC with the "Crux boundary" header +# comment in .github/workflows/v2-tests.yml and with cruxModuleDir in +# src/internal/testutil/crux_outbound_ratchet_test.go. CRUX_DIRS=( - src/pkg/agent - src/pkg/agentmode - src/pkg/agentparse - src/pkg/agentsmd - src/pkg/claude - src/pkg/sandbox - src/pkg/kubejob - src/pkg/toolapprove - src/pkg/skillreg - src/pkg/inference - src/pkg/openrouter - src/pkg/watchdog - src/pkg/trajectory - src/pkg/logscrub - src/pkg/proxy + src/crux ) changed=$(git diff --no-renames --name-only "$base" "$head" 2>/dev/null) || { echo full; exit 0; } @@ -55,7 +43,7 @@ while IFS= read -r f; do [ -n "$f" ] || continue case "$f" in # Fail-safe: anything that can change how every package builds or tests. - src/go.mod|src/go.sum|src/internal/*|src/pkg/config/*|.github/workflows/*|.github/scripts/*|src/scripts/*) + src/go.mod|src/go.sum|src/go.work|src/go.work.sum|src/crux/go.mod|src/crux/go.sum|src/internal/*|src/pkg/config/*|.github/workflows/*|.github/scripts/*|src/scripts/*) echo full exit 0 ;; diff --git a/.github/scripts/patch-coverage.sh b/.github/scripts/patch-coverage.sh new file mode 100755 index 0000000000..a7d910cca6 --- /dev/null +++ b/.github/scripts/patch-coverage.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# patch-coverage.sh — calculate and report patch coverage on changed Go lines. +# +# Part of Phase 1 of issue hivecommons/hive#14161. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec python3 "${SCRIPT_DIR}/patch_coverage.py" "$@" diff --git a/.github/scripts/patch_coverage.py b/.github/scripts/patch_coverage.py new file mode 100755 index 0000000000..ff5610ea0d --- /dev/null +++ b/.github/scripts/patch_coverage.py @@ -0,0 +1,749 @@ +#!/usr/bin/env python3 +""" +patch_coverage.py — Calculate and report patch coverage on changed Go lines. + +Part of Phase 1 of issue hivecommons/hive#14161. +Mechanics: +1. Diff: git diff -U0 --no-color + Excludes: _test.go, testdata/, generated files, blank lines, single-line comments. +2. Coverprofiles: Parse all blocks from specified profile(s) and aggregate counts. +3. Map each added line to coverprofile blocks: + - covered if any enclosing block has count > 0 + - uncovered if every enclosing block has count == 0 + - unscored if no enclosing block (comments, declarations, excluded packages) +4. Score = covered / (covered + uncovered). +5. Post or edit-in-place comment on PR if PR number is given. +6. Write summary to GITHUB_STEP_SUMMARY. +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os +import re +import subprocess +import sys +from typing import Dict, List, Optional, Set, Tuple + +MARKER = "" + +GENERATED_FILE_PATTERNS = [ + re.compile(r".*\.pb\.go$"), + re.compile(r".*_generated\.go$"), + re.compile(r".*zz_generated.*\.go$"), +] + +GENERATED_CONTENT_PATTERNS = [ + re.compile(r"^// Code generated .* DO NOT EDIT\.$", re.IGNORECASE), + re.compile(r"^// Code generated by .* DO NOT EDIT", re.IGNORECASE), + re.compile(r"^/\* Code generated .* DO NOT EDIT", re.IGNORECASE), + re.compile(r"^// DO NOT EDIT", re.IGNORECASE), + re.compile(r"^// auto-generated", re.IGNORECASE), + re.compile(r"^// GENERATED", re.IGNORECASE), +] + + +class ProfileBlock: + __slots__ = ("file", "start_line", "start_col", "end_line", "end_col", "stmts", "count") + + def __init__( + self, + file: str, + start_line: int, + start_col: int, + end_line: int, + end_col: int, + stmts: int, + count: int, + ) -> None: + self.file = file + self.start_line = start_line + self.start_col = start_col + self.end_line = end_line + self.end_col = end_col + self.stmts = stmts + self.count = count + + +def normalize_path(path: str) -> str: + p = path.replace("\\", "/").strip() + if p.startswith("./"): + p = p[2:] + if p.startswith("src/"): + p = p[4:] + for prefix in ("github.com/hivecommons/hive/", "github.com/kubestellar/hive/"): + if p.startswith(prefix): + p = p[len(prefix) :] + return p + + +def format_ranges(lines: List[int]) -> str: + if not lines: + return "" + sorted_lines = sorted(set(lines)) + ranges: List[str] = [] + start = sorted_lines[0] + end = start + for l in sorted_lines[1:]: + if l == end + 1: + end = l + else: + if start == end: + ranges.append(str(start)) + else: + ranges.append(f"{start}-{end}") + start = end = l + if start == end: + ranges.append(str(start)) + else: + ranges.append(f"{start}-{end}") + return ", ".join(ranges) + + +def is_generated_file(repo_root: str, rel_path: str) -> bool: + for pat in GENERATED_FILE_PATTERNS: + if pat.match(rel_path): + return True + candidates = [ + os.path.join(repo_root, rel_path), + os.path.join(repo_root, "src", rel_path), + ] + full_path = None + for cand in candidates: + if os.path.isfile(cand): + full_path = cand + break + if full_path and os.path.isfile(full_path): + try: + with open(full_path, "r", encoding="utf-8", errors="ignore") as f: + for idx, line in enumerate(f): + if idx > 50: + break + s = line.strip() + for cpat in GENERATED_CONTENT_PATTERNS: + if cpat.match(s): + return True + except Exception: + pass + return False + + +def is_excluded_file(repo_root: str, file_path: str) -> bool: + if not file_path.endswith(".go"): + return True + if file_path.endswith("_test.go"): + return True + parts = file_path.replace("\\", "/").split("/") + if "testdata" in parts: + return True + if is_generated_file(repo_root, file_path): + return True + return False + + +def get_file_lines(repo_root: str, file_path: str) -> Dict[int, str]: + candidates = [ + os.path.join(repo_root, file_path), + os.path.join(repo_root, "src", file_path), + ] + full_path = None + for cand in candidates: + if os.path.isfile(cand): + full_path = cand + break + res: Dict[int, str] = {} + if full_path and os.path.isfile(full_path): + try: + with open(full_path, "r", encoding="utf-8", errors="ignore") as f: + for idx, line in enumerate(f, start=1): + res[idx] = line + except Exception: + pass + return res + + +def get_changed_lines( + repo_root: str, base_ref: str, head_ref: str +) -> Tuple[Dict[str, Set[int]], Dict[str, str]]: + """ + Returns (changed_lines, display_paths). + changed_lines: {normalized_path: set_of_added_line_numbers} + display_paths: {normalized_path: repo_relative_display_path} + """ + diff_base = base_ref + try: + mb = subprocess.check_output( + ["git", "merge-base", base_ref, head_ref], + cwd=repo_root, + stderr=subprocess.DEVNULL, + text=True, + ).strip() + if mb: + diff_base = mb + except Exception: + pass + + try: + cmd = ["git", "diff", "-U0", "--no-color", diff_base, head_ref] + diff_out = subprocess.check_output(cmd, cwd=repo_root, text=True, errors="ignore") + except Exception as e: + print(f"Warning: git diff failed: {e}", file=sys.stderr) + return {}, {} + + changed_lines: Dict[str, Set[int]] = {} + display_paths: Dict[str, str] = {} + current_norm: Optional[str] = None + file_lines: Dict[int, str] = {} + hunk_re = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") + + for line in diff_out.splitlines(): + if line.startswith("+++ b/"): + raw_path = line[6:].strip() + if raw_path == "/dev/null" or is_excluded_file(repo_root, raw_path): + current_norm = None + else: + norm = normalize_path(raw_path) + current_norm = norm + display_paths[norm] = raw_path + if current_norm not in changed_lines: + changed_lines[current_norm] = set() + file_lines = get_file_lines(repo_root, raw_path) + continue + elif line.startswith("+++ "): + current_norm = None + continue + + if current_norm is None: + continue + + m = hunk_re.match(line) + if m: + start_line = int(m.group(1)) + count = int(m.group(2)) if m.group(2) is not None else 1 + for l in range(start_line, start_line + count): + line_text = file_lines.get(l, "").strip() + # Skip blank lines, comments, and package/import declarations + if not line_text or line_text.startswith("//"): + continue + if line_text.startswith("package ") or line_text == "package": + continue + if line_text.startswith("import ") or line_text == "import (" or line_text == "import": + continue + changed_lines[current_norm].add(l) + + # Filter out files with no added lines + filtered_changed = {f: lines for f, lines in changed_lines.items() if lines} + return filtered_changed, display_paths + + +def parse_coverprofiles(profile_paths: List[str]) -> Dict[str, List[ProfileBlock]]: + """ + Parses coverprofile files, aggregating counts for identical blocks across shards. + Returns {normalized_path: list_of_ProfileBlock}. + """ + # Key: (norm_file, start_line, start_col, end_line, end_col, stmts) -> total_count + aggregated: Dict[Tuple[str, int, int, int, int, int], int] = {} + + for path in profile_paths: + if not os.path.isfile(path): + continue + try: + with open(path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + line = line.strip() + if not line or line.startswith("mode:"): + continue + parts = line.split() + if len(parts) != 3: + continue + loc_part, stmts_str, count_str = parts[0], parts[1], parts[2] + try: + stmts = int(stmts_str) + count = int(count_str) + except ValueError: + continue + + colon_idx = loc_part.rfind(":") + if colon_idx == -1: + continue + raw_file = loc_part[:colon_idx] + loc = loc_part[colon_idx + 1 :] + + try: + start_part, end_part = loc.split(",") + start_line, start_col = map(int, start_part.split(".")) + end_line, end_col = map(int, end_part.split(".")) + except ValueError: + continue + + norm_file = normalize_path(raw_file) + key = (norm_file, start_line, start_col, end_line, end_col, stmts) + aggregated[key] = aggregated.get(key, 0) + count + except Exception as e: + print(f"Warning: error reading profile {path}: {e}", file=sys.stderr) + + by_file: Dict[str, List[ProfileBlock]] = {} + for (norm_file, start_line, start_col, end_line, end_col, stmts), count in aggregated.items(): + block = ProfileBlock(norm_file, start_line, start_col, end_line, end_col, stmts, count) + by_file.setdefault(norm_file, []).append(block) + + return by_file + + +def score_patch_coverage( + changed_lines: Dict[str, Set[int]], + profiles_by_file: Dict[str, List[ProfileBlock]], +) -> Tuple[int, int, Dict[str, List[int]], Dict[str, List[int]]]: + """ + Scores added lines against coverprofile blocks. + Returns: (total_covered, total_uncovered, covered_by_file, uncovered_by_file) + """ + total_covered = 0 + total_uncovered = 0 + covered_by_file: Dict[str, List[int]] = {} + uncovered_by_file: Dict[str, List[int]] = {} + + for norm_file, lines in changed_lines.items(): + blocks = profiles_by_file.get(norm_file) + if blocks is None: + # Fallback suffix match + for pf, pblocks in profiles_by_file.items(): + if pf.endswith("/" + norm_file) or norm_file.endswith("/" + pf): + blocks = pblocks + break + + cov_lines = [] + uncov_lines = [] + + if blocks: + for l in sorted(lines): + enclosing = [b for b in blocks if b.start_line <= l <= b.end_line] + if not enclosing: + # Unscored (outside function / declarations / comments) + continue + if any(b.count > 0 for b in enclosing): + cov_lines.append(l) + else: + uncov_lines.append(l) + + if cov_lines: + covered_by_file[norm_file] = cov_lines + total_covered += len(cov_lines) + if uncov_lines: + uncovered_by_file[norm_file] = uncov_lines + total_uncovered += len(uncov_lines) + + return total_covered, total_uncovered, covered_by_file, uncovered_by_file + + +def generate_comment_body( + percent: float, + covered: int, + uncovered: int, + uncovered_by_file: Dict[str, List[int]], + display_paths: Dict[str, str], + min_percent: float, + min_uncovered: int, + is_advisory: bool, +) -> str: + if uncovered == 0: + return ( + f"{MARKER}\n" + "### Patch Coverage: 100.0% :white_check_mark:\n\n" + "All changed Go statements are covered by tests." + ) + + status_tag = "(advisory)" if is_advisory else "" + lines = [ + MARKER, + f"### Patch Coverage: {percent:.1f}% {status_tag}".strip(), + "", + "The following changed Go statements have no test coverage:", + "", + "| File | Uncovered Lines |", + "| :--- | :--- |", + ] + for norm_file in sorted(uncovered_by_file.keys()): + disp = display_paths.get(norm_file, norm_file) + rngs = format_ranges(uncovered_by_file[norm_file]) + lines.append(f"| `{disp}` | {rngs} |") + + lines.extend([ + "", + "
", + "Details", + "", + f"- **Covered statements:** {covered}", + f"- **Uncovered statements:** {uncovered}", + f"- **Patch coverage:** {percent:.1f}% (threshold: {min_percent:.1f}%, min uncovered: {min_uncovered})", + "", + "*Phase 1 is advisory. In Phase 2, PRs with patch coverage below 80% and ≥5 uncovered statements will fail the coverage gate.*", + "
", + ]) + return "\n".join(lines) + + +def write_step_summary( + summary_path: str, + percent: float, + covered: int, + uncovered: int, + uncovered_by_file: Dict[str, List[int]], + display_paths: Dict[str, str], + min_percent: float, + min_uncovered: int, + is_advisory: bool, +) -> None: + if not summary_path: + return + + lines = [ + "## Patch Coverage on Changed Go Lines", + "", + ] + if covered + uncovered == 0: + lines.append("No executable changed Go statements to score.") + elif uncovered == 0: + lines.extend([ + "- **Patch coverage:** 100.0% :white_check_mark:", + f"- **Covered statements:** {covered}", + "- **Uncovered statements:** 0", + ]) + else: + status_str = ( + "Advisory" + if is_advisory + else ( + "Failed" + if (percent < min_percent and uncovered >= min_uncovered) + else "Passed" + ) + ) + lines.extend([ + f"- **Patch coverage:** {percent:.1f}% ({covered} / {covered + uncovered} statements covered)", + f"- **Uncovered statements:** {uncovered}", + f"- **Status:** {status_str} (threshold: {min_percent:.1f}%, min uncovered: {min_uncovered})", + "", + "### Uncovered lines by file", + "", + "| File | Uncovered Lines |", + "| :--- | :--- |", + ]) + for norm_file in sorted(uncovered_by_file.keys()): + disp = display_paths.get(norm_file, norm_file) + rngs = format_ranges(uncovered_by_file[norm_file]) + lines.append(f"| `{disp}` | {rngs} |") + + lines.append("") + try: + with open(summary_path, "a", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + except Exception as e: + print(f"Warning: could not write step summary to {summary_path}: {e}", file=sys.stderr) + + +def upsert_comment(repo: str, pr: int, body: str, uncovered: int, dry_run: bool = False) -> None: + if dry_run: + print(f"[DRY RUN] Would update/post comment on {repo}#{pr}:") + print(body) + return + + # Check for existing comment carrying the marker + comment_id: Optional[str] = None + try: + cmd = [ + "gh", + "api", + f"repos/{repo}/issues/{pr}/comments?per_page=100", + "--paginate", + "--jq", + f'.[] | select(.body | contains("{MARKER}")) | .id', + ] + res = subprocess.run(cmd, capture_output=True, text=True, check=False) + if res.returncode == 0 and res.stdout.strip(): + comment_id = res.stdout.strip().splitlines()[0] + except Exception as e: + print(f"::notice::Could not query comments on PR #{pr}: {e}", file=sys.stderr) + + req_bytes = json.dumps({"body": body}).encode("utf-8") + + if comment_id: + try: + res = subprocess.run( + ["gh", "api", "-X", "PATCH", f"repos/{repo}/issues/comments/{comment_id}", "--input", "-"], + input=req_bytes, + capture_output=True, + check=False, + ) + if res.returncode == 0: + print(f"Updated patch coverage comment on PR #{pr} (comment ID {comment_id})") + else: + err = res.stderr.decode("utf-8", errors="ignore").strip() + print(f"::notice::Could not update patch coverage comment (ID {comment_id}): {err}", file=sys.stderr) + except Exception as e: + print(f"::notice::Error updating patch coverage comment: {e}", file=sys.stderr) + else: + # If no previous comment exists and there are no uncovered statements, don't post a comment + if uncovered == 0: + return + try: + res = subprocess.run( + ["gh", "api", "-X", "POST", f"repos/{repo}/issues/{pr}/comments", "--input", "-"], + input=req_bytes, + capture_output=True, + check=False, + ) + if res.returncode == 0: + print(f"Posted patch coverage comment on PR #{pr}") + else: + err = res.stderr.decode("utf-8", errors="ignore").strip() + print(f"::notice::Could not post patch coverage comment on PR #{pr}: {err}", file=sys.stderr) + except Exception as e: + print(f"::notice::Error posting patch coverage comment: {e}", file=sys.stderr) + + +def resolve_base_ref(repo_root: str, base_arg: Optional[str]) -> str: + if base_arg: + # Verify base_arg is resolvable + try: + subprocess.check_call( + ["git", "rev-parse", "-q", "--verify", base_arg], + cwd=repo_root, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return base_arg + except Exception: + pass + + # Check if HEAD^2 exists (merge commit) + try: + subprocess.check_call( + ["git", "rev-parse", "-q", "--verify", "HEAD^2"], + cwd=repo_root, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return "HEAD^1" + except Exception: + pass + + # Check common base branches + for candidate in ["origin/v5", "v5", "origin/main", "main", "HEAD~1"]: + try: + subprocess.check_call( + ["git", "rev-parse", "-q", "--verify", candidate], + cwd=repo_root, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return candidate + except Exception: + continue + + return "HEAD" + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Calculate and report patch coverage on changed Go lines." + ) + parser.add_argument("--base", default=None, help="Base git ref or commit SHA") + parser.add_argument("--head", default="HEAD", help="Head git ref (default: HEAD)") + parser.add_argument( + "--profile", + action="append", + default=[], + help="Coverprofile file or pattern (can be specified multiple times)", + ) + parser.add_argument( + "--profile-dir", + default=None, + help="Directory containing coverprofiles", + ) + parser.add_argument( + "--min-percent", + type=float, + default=80.0, + help="Minimum patch coverage percentage (default: 80.0)", + ) + parser.add_argument( + "--min-uncovered", + type=int, + default=5, + help="Minimum uncovered statements before threshold fails (default: 5)", + ) + parser.add_argument( + "--advisory", + action="store_true", + default=False, + help="Advisory mode: exit 0 even if threshold is not met (Phase 1 default)", + ) + parser.add_argument( + "--pr", + type=int, + default=None, + help="Pull request number (or set via PR_NUMBER env var)", + ) + parser.add_argument( + "--repo", + default=None, + help="GitHub repo slug owner/name (or set via GITHUB_REPOSITORY env var)", + ) + parser.add_argument( + "--summary-file", + default=None, + help="Path to write GitHub step summary (or set via GITHUB_STEP_SUMMARY env var)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + default=False, + help="Print planned comment without making API calls", + ) + + args = parser.parse_args() + + try: + repo_root = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], + text=True, + ).strip() + except Exception: + repo_root = os.getcwd() + + base_ref = resolve_base_ref(repo_root, args.base) + head_ref = args.head + + pr_number = args.pr + if pr_number is None and os.environ.get("PR_NUMBER"): + try: + pr_number = int(os.environ["PR_NUMBER"]) + except ValueError: + pass + + repo_slug = args.repo or os.environ.get("GITHUB_REPOSITORY") or "hivecommons/hive" + summary_file = args.summary_file or os.environ.get("GITHUB_STEP_SUMMARY") + + # Resolve coverprofiles + profile_inputs: List[str] = [] + for p in args.profile: + for part in p.split(): + profile_inputs.append(part) + + resolved_profiles: List[str] = [] + for p in profile_inputs: + matched = glob.glob(p) + if matched: + resolved_profiles.extend(matched) + elif os.path.isfile(p): + resolved_profiles.append(p) + + if args.profile_dir and os.path.isdir(args.profile_dir): + resolved_profiles.extend(glob.glob(os.path.join(args.profile_dir, "coverage*.out"))) + + # Fallback search if no profile specified or found + if not resolved_profiles: + for cand in [ + "shards/coverage-merged.out", + "shards/coverage-*.out", + "coverage-merged.out", + "coverage-*.out", + "coverage.out", + ]: + matched = glob.glob(cand) + if matched: + resolved_profiles.extend(matched) + break + + # 1. Get changed lines from git diff + changed_lines, display_paths = get_changed_lines(repo_root, base_ref, head_ref) + + if not changed_lines: + print("Patch coverage: No changed Go lines to score.") + if summary_file: + write_step_summary( + summary_file, 100.0, 0, 0, {}, {}, args.min_percent, args.min_uncovered, args.advisory + ) + return 0 + + # 2. Parse coverprofiles + profiles_by_file = parse_coverprofiles(resolved_profiles) + + # 3. Score patch coverage + covered, uncovered, covered_by_file, uncovered_by_file = score_patch_coverage( + changed_lines, profiles_by_file + ) + total_scored = covered + uncovered + + if total_scored == 0: + percent = 100.0 + else: + percent = (covered / total_scored) * 100.0 + + threshold_failed = (percent < args.min_percent) and (uncovered >= args.min_uncovered) + + # Output to stdout + print("=" * 60) + print("Patch Coverage Summary:") + print(f" Base ref: {base_ref}") + print(f" Head ref: {head_ref}") + print(f" Profiles parsed: {len(resolved_profiles)}") + print(f" Changed Go files scored: {len(changed_lines)}") + print(f" Scored statements: {total_scored} ({covered} covered, {uncovered} uncovered)") + print(f" Patch coverage: {percent:.1f}%") + print(f" Threshold: >= {args.min_percent:.1f}% (min uncovered: {args.min_uncovered})") + + if threshold_failed: + if args.advisory: + print(" Result: ADVISORY (threshold not met; gate is advisory in Phase 1)") + else: + print(" Result: FAILED (threshold not met)") + else: + print(" Result: PASS") + + if uncovered_by_file: + print("\nUncovered Go lines:") + for norm_file in sorted(uncovered_by_file.keys()): + disp = display_paths.get(norm_file, norm_file) + rngs = format_ranges(uncovered_by_file[norm_file]) + print(f" {disp}: {rngs}") + print("=" * 60) + + # Write step summary + if summary_file: + write_step_summary( + summary_file, + percent, + covered, + uncovered, + uncovered_by_file, + display_paths, + args.min_percent, + args.min_uncovered, + args.advisory, + ) + + # Post or update PR comment if PR number is available + if pr_number: + comment_body = generate_comment_body( + percent, + covered, + uncovered, + uncovered_by_file, + display_paths, + args.min_percent, + args.min_uncovered, + args.advisory, + ) + upsert_comment(repo_slug, pr_number, comment_body, uncovered, dry_run=args.dry_run) + + if threshold_failed and not args.advisory: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/test-patch-coverage.sh b/.github/scripts/test-patch-coverage.sh new file mode 100755 index 0000000000..ded6e6db5a --- /dev/null +++ b/.github/scripts/test-patch-coverage.sh @@ -0,0 +1,315 @@ +#!/usr/bin/env bash +# test-patch-coverage.sh — hermetic regression test for patch-coverage.sh. +# +# Exercises git diff parsing, coverprofile aggregation, exclusions +# (_test.go, testdata, generated code, comments, blank lines), +# threshold calculation, advisory mode, and PR comment posting / in-place updates. +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SCRIPT="${ROOT}/.github/scripts/patch-coverage.sh" +TMP_ROOT="${ROOT}/.github/.test-tmp" +mkdir -p "$TMP_ROOT" +TMP="$(mktemp -d "$TMP_ROOT/patch-cov.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT + +failures=0 +pass() { printf ' ok: %s\n' "$*"; } +bad() { + printf ' FAIL: %s\n' "$*" + failures=$((failures + 1)) +} +assert_contains() { + local haystack="$1" needle="$2" msg="$3" + if grep -qF -- "$needle" <<<"$haystack"; then pass "$msg"; else bad "$msg: expected '$needle' in '$haystack'"; fi +} +assert_not_contains() { + local haystack="$1" needle="$2" msg="$3" + if grep -qF -- "$needle" <<<"$haystack"; then bad "$msg: did not expect '$needle' in '$haystack'"; else pass "$msg"; fi +} +assert_exit0() { + local rc="$1" out="$2" msg="$3" + if [[ $rc -eq 0 ]]; then pass "$msg"; else bad "$msg: exited $rc: $out"; fi +} +assert_exit1() { + local rc="$1" out="$2" msg="$3" + if [[ $rc -ne 0 ]]; then pass "$msg"; else bad "$msg: expected non-zero, got 0: $out"; fi +} + +export GIT_AUTHOR_NAME=test GIT_AUTHOR_EMAIL=test@example.invalid +export GIT_COMMITTER_NAME=test GIT_COMMITTER_EMAIL=test@example.invalid +export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 + +REPO_DIR="$TMP/repo" +git init -q -b v5 "$REPO_DIR" + +commit_file() { # relpath content msg + local dir + dir="$(dirname "$REPO_DIR/$1")" + mkdir -p "$dir" + printf '%s\n' "$2" >"$REPO_DIR/$1" + git -C "$REPO_DIR" add "$1" + git -C "$REPO_DIR" commit -q -m "$3" +} + +# Base commit: initial Go file +commit_file "src/pkg/sample/sample.go" 'package sample + +func Initial() int { + return 42 +} +' 'initial commit' +base_commit="$(git -C "$REPO_DIR" rev-parse HEAD)" + +# Fake gh CLI in PATH +MOCK_BIN="$TMP/mock_bin" +mkdir -p "$MOCK_BIN" +GH_LOG="$TMP/gh_calls.log" +cat > "$MOCK_BIN/gh" <> "$GH_LOG" +if [[ "\$*" == *"--input"* ]]; then + cat >> "$GH_LOG" +fi + +if [[ "\$*" == *"repos/example/hive/issues/42/comments?per_page=100"* ]]; then + if [[ -f "$TMP/existing_comment_id" ]]; then + cid=\$(cat "$TMP/existing_comment_id") + echo "\$cid" + fi + exit 0 +fi + +if [[ "\$*" == *"repos/example/hive/issues/42/comments"* ]]; then + # POST new comment + exit 0 +fi + +if [[ "\$*" == *"repos/example/hive/issues/comments/"* ]]; then + # PATCH existing comment + exit 0 +fi +exit 0 +EOF +chmod +x "$MOCK_BIN/gh" +export PATH="$MOCK_BIN:$PATH" + +# ============================================================================== +# Test 1: Full patch coverage (100%) +# ============================================================================== +printf '\n--- Test 1: 100%% patch coverage ---\n' +git -C "$REPO_DIR" checkout -q -b pr-full "$base_commit" +commit_file "src/pkg/sample/sample.go" 'package sample + +func Initial() int { + return 42 +} + +func Added(x int) int { + y := x * 2 + return y + 1 +} +' 'add Added func' + +cat > "$TMP/coverage-full.out" <<'EOF' +mode: atomic +github.com/hivecommons/hive/pkg/sample/sample.go:3.20,5.2 1 1 +github.com/hivecommons/hive/pkg/sample/sample.go:7.23,10.2 2 1 +EOF + +SUM="$TMP/summary1.md" +rm -f "$SUM" "$GH_LOG" +out=$(cd "$REPO_DIR" && bash "$SCRIPT" --base "$base_commit" --head HEAD --profile "$TMP/coverage-full.out" --pr 42 --repo example/hive --summary-file "$SUM" 2>&1) +rc=$? +assert_exit0 $rc "$out" "full coverage exits 0" +assert_contains "$out" "Patch coverage: 100.0%" "reports 100% patch coverage" +assert_contains "$out" "Result: PASS" "reports PASS" +assert_contains "$(cat "$SUM")" "100.0%" "step summary records 100%" +# No previous comment existed and uncovered=0, so no comment posted +assert_not_contains "$(cat "$GH_LOG" 2>/dev/null || true)" "POST" "does not post comment on clean PR with no prior comment" + +# If a previous comment existed, it is updated to 100% +echo "11111" > "$TMP/existing_comment_id" +rm -f "$GH_LOG" +out_prev=$(cd "$REPO_DIR" && bash "$SCRIPT" --base "$base_commit" --head HEAD --profile "$TMP/coverage-full.out" --pr 42 --repo example/hive 2>&1) +assert_exit0 $? "$out_prev" "full coverage with prior comment exits 0" +assert_contains "$(cat "$GH_LOG")" "PATCH repos/example/hive/issues/comments/11111" "updates existing comment to 100%" +assert_contains "$(cat "$GH_LOG")" "Patch Coverage: 100.0%" "updated comment reflects 100%" +rm -f "$TMP/existing_comment_id" + +# ============================================================================== +# Test 2: Uncovered lines below threshold (threshold failure in non-advisory) +# ============================================================================== +printf '\n--- Test 2: Uncovered lines below threshold ---\n' +git -C "$REPO_DIR" checkout -q -b pr-uncovered "$base_commit" +commit_file "src/pkg/sample/sample.go" 'package sample + +func Initial() int { + return 42 +} + +func Complex(x int) int { + if x > 100 { + a := 1 + b := 2 + c := 3 + d := 4 + e := 5 + return a + b + c + d + e + } + return x +} +' 'add Complex func with untested branch' + +cat > "$TMP/coverage-uncovered.out" <<'EOF' +mode: atomic +github.com/hivecommons/hive/pkg/sample/sample.go:3.20,5.2 1 1 +github.com/hivecommons/hive/pkg/sample/sample.go:7.25,8.13 1 1 +github.com/hivecommons/hive/pkg/sample/sample.go:8.13,16.3 6 0 +github.com/hivecommons/hive/pkg/sample/sample.go:17.2,17.10 1 1 +EOF + +SUM="$TMP/summary2.md" +rm -f "$SUM" "$GH_LOG" +out_advisory=$(cd "$REPO_DIR" && bash "$SCRIPT" --base "$base_commit" --head HEAD --profile "$TMP/coverage-uncovered.out" --pr 42 --repo example/hive --summary-file "$SUM" --advisory 2>&1) +rc_advisory=$? +assert_exit0 $rc_advisory "$out_advisory" "advisory mode exits 0 despite low coverage" +assert_contains "$out_advisory" "Result: ADVISORY" "reports ADVISORY in advisory mode" +assert_contains "$out_advisory" "src/pkg/sample/sample.go:" "lists uncovered file" + +# Verify non-advisory exits 1 +out_strict=$(cd "$REPO_DIR" && bash "$SCRIPT" --base "$base_commit" --head HEAD --profile "$TMP/coverage-uncovered.out" --pr 42 --repo example/hive --summary-file "$SUM" 2>&1) +rc_strict=$? +assert_exit1 $rc_strict "$out_strict" "strict mode exits 1 when threshold failed" +assert_contains "$out_strict" "Result: FAILED" "reports FAILED in strict mode" + +# Verify mock gh received POST with comment containing marker and uncovered lines +assert_contains "$(cat "$GH_LOG")" "POST repos/example/hive/issues/42/comments" "posts comment to PR" +assert_contains "$(cat "$GH_LOG")" "" "comment contains marker" +assert_contains "$(cat "$GH_LOG")" "src/pkg/sample/sample.go" "comment lists uncovered file" + +# ============================================================================== +# Test 3: Edit-in-place existing comment +# ============================================================================== +printf '\n--- Test 3: Edit-in-place existing comment ---\n' +echo "98765" > "$TMP/existing_comment_id" +rm -f "$GH_LOG" +out_update=$(cd "$REPO_DIR" && bash "$SCRIPT" --base "$base_commit" --head HEAD --profile "$TMP/coverage-uncovered.out" --pr 42 --repo example/hive --advisory 2>&1) +assert_exit0 $? "$out_update" "advisory update exits 0" +assert_contains "$(cat "$GH_LOG")" "PATCH repos/example/hive/issues/comments/98765" "edits existing comment in-place" +assert_not_contains "$(cat "$GH_LOG")" "POST repos/example/hive/issues/42/comments" "does not create new comment when one exists" +rm -f "$TMP/existing_comment_id" + +# ============================================================================== +# Test 4: Small uncovered statements (< min-uncovered) do not trip gate +# ============================================================================== +printf '\n--- Test 4: Small uncovered statements (< 5) ---\n' +git -C "$REPO_DIR" checkout -q -b pr-small "$base_commit" +commit_file "src/pkg/sample/sample.go" 'package sample + +func Initial() int { + return 42 +} + +func Small(err error) error { + if err != nil { + return err + } + return nil +} +' 'add Small func with 1 uncovered return err' + +cat > "$TMP/coverage-small.out" <<'EOF' +mode: atomic +github.com/hivecommons/hive/pkg/sample/sample.go:3.20,5.2 1 1 +github.com/hivecommons/hive/pkg/sample/sample.go:7.29,8.16 1 1 +github.com/hivecommons/hive/pkg/sample/sample.go:8.16,10.3 1 0 +github.com/hivecommons/hive/pkg/sample/sample.go:11.2,11.12 1 1 +EOF + +rm -f "$GH_LOG" +out_small=$(cd "$REPO_DIR" && bash "$SCRIPT" --base "$base_commit" --head HEAD --profile "$TMP/coverage-small.out" --pr 42 --repo example/hive --min-percent 80 --min-uncovered 5 2>&1) +rc_small=$? +assert_exit0 $rc_small "$out_small" "uncovered < 5 does not trip failure even if < 80%" +assert_contains "$out_small" "Result: PASS" "small return err arm passes gate" + +# ============================================================================== +# Test 5: Exclusions (_test.go, testdata, generated files, comments, blank lines) +# ============================================================================== +printf '\n--- Test 5: Exclusions ---\n' +git -C "$REPO_DIR" checkout -q -b pr-exclusions "$base_commit" +commit_file "src/pkg/sample/sample_test.go" 'package sample +import "testing" +func TestSample(t *testing.T) {} +' 'add test file' +commit_file "src/pkg/sample/testdata/fixture.go" 'package testdata +var Fixture = 123 +' 'add testdata' +commit_file "src/pkg/sample/types.pb.go" 'package sample +type ProtobufType struct {} +' 'add pb.go' +commit_file "src/pkg/sample/gen.go" '// Code generated by tool. DO NOT EDIT. +package sample +func Generated() {} +' 'add generated comment file' +commit_file "src/pkg/sample/comments.go" 'package sample + +// Only a comment line + +' 'add comments and blanks only' + +rm -f "$GH_LOG" +out_excl=$(cd "$REPO_DIR" && bash "$SCRIPT" --base "$base_commit" --head HEAD --profile "$TMP/coverage-full.out" --pr 42 --repo example/hive 2>&1) +rc_excl=$? +assert_exit0 $rc_excl "$out_excl" "diff with only excluded files exits 0" +assert_contains "$out_excl" "No changed Go lines to score" "all excluded files ignored" + +# ============================================================================== +# Test 6: Multi-shard coverprofile merging +# ============================================================================== +printf '\n--- Test 6: Multi-shard coverprofile merging ---\n' +git -C "$REPO_DIR" checkout -q -b pr-multi "$base_commit" +commit_file "src/pkg/sample/sample.go" 'package sample + +func Initial() int { + return 42 +} + +func FuncA() int { + return 1 +} + +func FuncB() int { + return 2 +} +' 'add FuncA and FuncB' + +cat > "$TMP/shard-1.out" <<'EOF' +mode: atomic +github.com/hivecommons/hive/pkg/sample/sample.go:7.18,9.2 1 1 +github.com/hivecommons/hive/pkg/sample/sample.go:11.18,13.2 1 0 +EOF + +cat > "$TMP/shard-2.out" <<'EOF' +mode: atomic +github.com/hivecommons/hive/pkg/sample/sample.go:7.18,9.2 1 0 +github.com/hivecommons/hive/pkg/sample/sample.go:11.18,13.2 1 1 +EOF + +rm -f "$GH_LOG" +out_multi=$(cd "$REPO_DIR" && bash "$SCRIPT" --base "$base_commit" --head HEAD --profile "$TMP/shard-1.out" --profile "$TMP/shard-2.out" 2>&1) +rc_multi=$? +assert_exit0 $rc_multi "$out_multi" "multiple shard profiles merge correctly" +assert_contains "$out_multi" "Patch coverage: 100.0%" "both functions covered across shards" + +# ============================================================================== +# Test Summary +# ============================================================================== +printf '\n=======================================================\n' +if [[ $failures -eq 0 ]]; then + printf 'All patch-coverage self-tests PASSED!\n' + exit 0 +else + printf 'patch-coverage self-tests FAILED: %d failure(s)\n' "$failures" + exit 1 +fi diff --git a/.github/workflows/ci-runner-image.yml b/.github/workflows/ci-runner-image.yml index 8d68a28340..457d98eceb 100644 --- a/.github/workflows/ci-runner-image.yml +++ b/.github/workflows/ci-runner-image.yml @@ -160,7 +160,7 @@ jobs: - name: Log in to GHCR if: github.event_name == 'workflow_dispatch' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/coverage-hourly.yml b/.github/workflows/coverage-hourly.yml index a242cdee67..688fc81ef8 100644 --- a/.github/workflows/coverage-hourly.yml +++ b/.github/workflows/coverage-hourly.yml @@ -190,7 +190,7 @@ jobs: id: cover continue-on-error: true run: | - go test ./pkg/... ./cmd/... ./internal/... -short -race -coverprofile=coverage.out -count=1 -timeout 600s \ + go test ./pkg/... ./cmd/... ./internal/... ./crux/... -short -race -coverprofile=coverage.out -count=1 -timeout 600s \ 2>&1 | tee race-run.log exit "${PIPESTATUS[0]}" @@ -217,7 +217,7 @@ jobs: [dashboard]=84 # 86.2% on 2026-09-01; residue is inception tmux watcher + copilot/claude device-flow + /data file I/O + k8s [dashchat]=64 # initial browser transport seam (#8304); chat.Service command paths are covered in pkg/chat, surface tests pin guards [hub]=87 # kubectl-shelling + live-GitHub paths; see #2355. Was 89 against v2's 89.9%; v4/v5 measure 87.6%. - [agent]=88 # 88.9% after the v5→v6 top-up; pre-existing shortfall tracked in #2355. + [crux/agent]=88 # 88.9% after the v5->v6 top-up (#14340); pre-existing shortfall inherited from #2350, NOT a new regression. See #2355. Keyed crux/ since the module cut (#13536) [adminmcp]=53 # 53.3% on PR #8807; phase-1 skeleton covers refusal/cap/scrub/core JSON-RPC paths while provider breadth lands in later phases # ── Floors recorded when the gate moved from v2 to v4 (#3903) ───── @@ -257,11 +257,11 @@ jobs: [hivectl]=88 [hivectl/commands]=89 # 89.9% after the v5→v6 forward-merge (#13896); the union of both branches' tests sits just under the 90 default [knowledge]=89 - [proxy]=89 + [crux/proxy]=89 [ioscan]=96 - [sandbox]=98 - [pushbroker]=99 - [kubejob]=90 # new in #6311; sandbox.Launcher over the in-cluster API, tested against httptest + [crux/sandbox]=98 + [crux/pushbroker]=99 + [crux/kubejob]=90 # new in #6311; sandbox.Launcher over the in-cluster API, tested against httptest # ── Stale module-path parser repaired on v5 (#6066) ───────────── # These packages were pre-existing below-default floors that became @@ -319,7 +319,7 @@ jobs: # 90 default (#7407). [cmd/hivectl]=97 # 100.0% on 2026-09-17; was exempt as an untested main [cmd/flue-fixture]=75 # 80.0% on 2026-09-22 (#8361): main() is the single uncovered statement; run() and signalContext() are tested in-process and the binary is smoke-tested as a child process, which yields no profile - [cmd/crux]=75 # 80.0% on 2026-10-23 (#13536): same shape as flue-fixture; main() is the one uncovered statement, run() and signalContext() are tested in-process + [cmd/crux]=75 # 80.0% on 2026-10-23 (#13536): same shape as flue-fixture; main() is the one uncovered statement, run() is tested in-process. floor_key maps BOTH ./cmd/crux (root host) and ./crux/cmd/crux (the module's own binary, #14138) here; the module one measured 80.0% on 2026-10-25 [crux/cli]=92 # 97.8% on 2026-10-23 (#13536): the crux CLI against a fake crux.Runtime plus a real unix control socket; the residue is Main() and write-error branches ) @@ -346,7 +346,7 @@ jobs: # Both were therefore incapable of failing the gate, so adding a # package with zero tests was the one guaranteed way to stay green. # They are now hard failures in their own right. - go test -cover ./pkg/... ./cmd/... ./internal/... -short -count=1 -timeout 600s 2>&1 | tee cover-run.log || true + go test -cover ./pkg/... ./cmd/... ./internal/... ./crux/... -short -count=1 -timeout 600s 2>&1 | tee cover-run.log || true # Floor-map key for a package import path: cmd packages are keyed # with a "cmd/" prefix so cmd/hivectl can never resolve to @@ -359,11 +359,18 @@ jobs: # pkg/hub/spoke (90.7%), which both keyed to "spoke", so the [spoke]=95 # floor #7146 set for pkg/spoke would have been enforced against # pkg/hub/spoke too and failed it. Keys are now "spoke" vs "hub/spoke". + # + # The nested Crux module (src/crux, #13536) is keyed "crux/" + # with the module root package as "crux", so crux/config can never + # alias pkg/config and crux/agent keeps the floor pkg/agent carried + # before the module cut. floor_key() { case "$1" in - */cmd/*) echo "cmd/${1##*/}" ;; - */pkg/*) echo "${1##*/pkg/}" ;; - *) echo "${1##*/}" ;; + */cmd/*) echo "cmd/${1##*/}" ;; + */pkg/*) echo "${1##*/pkg/}" ;; + */crux/*) echo "crux/${1##*/crux/}" ;; + */crux) echo "crux" ;; + *) echo "${1##*/}" ;; esac } diff --git a/.github/workflows/crux-sync.yml b/.github/workflows/crux-sync.yml new file mode 100644 index 0000000000..2e45b7b930 --- /dev/null +++ b/.github/workflows/crux-sync.yml @@ -0,0 +1,236 @@ +name: Crux sync + +# Publishes src/crux (module github.com/hivecommons/hive/crux) to the public +# hivecommons/crux repository as module github.com/hivecommons/crux (#14138). +# +# Hive stays the source of truth. hivecommons/crux is a published, +# module-path-rewritten copy of the nested module: every push to v5 that +# touches src/crux re-runs src/scripts/crux-publish.sh, which rebuilds the +# publish tree, rewrites the module path, proves the result builds with +# GOWORK=off, and commits it on top of hivecommons/crux `main` — one +# fast-forward commit per sync naming the hive sha it came from (.hive-sync). +# Hand-maintained scaffolding in hivecommons/crux (.github/, .goreleaser.yaml, +# Dockerfile, README.md, LICENSE) is preserved; everything else is replaced. +# +# Releases are cut from hive: tagging hive `crux/vX.Y.Z` runs this workflow +# too, which syncs that commit and then pushes tag `vX.Y.Z` to +# hivecommons/crux, whose release workflow builds binaries, the container +# image and the Homebrew inputs from it. +# +# Credentials: pushing to another repository needs more than the job's +# GITHUB_TOKEN. The push uses CRUX_SYNC_TOKEN when set (a fine-grained PAT or +# App token with Contents: write on hivecommons/crux), falling back to +# TOPUP_PUSH_TOKEN (the PAT v5-topup/cli-pin-bump already use). If neither +# reaches hivecommons/crux the push step fails with a 403 and the run summary +# names the secret to provision; the publish tree is still uploaded as an +# artifact so a maintainer can push it by hand with the recipe below. +# +# Manual sync (same script, same result): +# git clone https://github.com/hivecommons/crux crux-publish +# src/scripts/crux-publish.sh . crux-publish +# (cd crux-publish && git add -A && git commit -s -m "🌱 sync: hive@$(git -C .. rev-parse --short HEAD)" && git push) + +on: + push: + branches: [v5] + paths: + - 'src/crux/**' + - 'src/scripts/crux-publish.sh' + - '.github/workflows/crux-sync.yml' + tags: + - 'crux/v*' + workflow_dispatch: + inputs: + tag: + description: 'Also push this tag (vX.Y.Z) to hivecommons/crux at the synced commit' + type: string + default: '' + dry_run: + description: 'Build the publish tree and upload it, but do not push' + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: crux-sync + cancel-in-progress: false + +env: + CRUX_REPO: hivecommons/crux + +jobs: + sync: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out hive + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: src/crux/go.mod + cache-dependency-path: src/crux/go.sum + + - name: Resolve push token + id: token + env: + CRUX_SYNC_TOKEN: ${{ secrets.CRUX_SYNC_TOKEN }} + TOPUP_PUSH_TOKEN: ${{ secrets.TOPUP_PUSH_TOKEN }} + run: | + set -euo pipefail + if [ -n "$CRUX_SYNC_TOKEN" ]; then + echo "source=CRUX_SYNC_TOKEN" >> "$GITHUB_OUTPUT" + echo "token=$CRUX_SYNC_TOKEN" >> "$GITHUB_OUTPUT" + elif [ -n "$TOPUP_PUSH_TOKEN" ]; then + echo "source=TOPUP_PUSH_TOKEN" >> "$GITHUB_OUTPUT" + echo "token=$TOPUP_PUSH_TOKEN" >> "$GITHUB_OUTPUT" + else + echo "source=none" >> "$GITHUB_OUTPUT" + echo "token=" >> "$GITHUB_OUTPUT" + fi + + - name: Check out hivecommons/crux + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: hivecommons/crux + ref: main + path: crux-publish + # Falls back to the job token (read-only on another repo) when no + # push token is configured, so dry runs and the artifact still work. + token: ${{ steps.token.outputs.token || github.token }} + persist-credentials: true + fetch-depth: 1 + + - name: Build the publish tree + run: src/scripts/crux-publish.sh "$GITHUB_WORKSPACE" "$GITHUB_WORKSPACE/crux-publish" + + - name: Resolve release tag + id: tag + env: + REF_TYPE: ${{ github.ref_type }} + REF_NAME: ${{ github.ref_name }} + INPUT_TAG: ${{ github.event.inputs.tag }} + run: | + set -euo pipefail + tag="" + if [ "$REF_TYPE" = tag ]; then + tag="${REF_NAME#crux/}" + elif [ -n "$INPUT_TAG" ]; then + tag="$INPUT_TAG" + fi + if [ -n "$tag" ] && ! [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([-+].+)?$ ]]; then + echo "::error::tag '$tag' is not vX.Y.Z" >&2 + exit 1 + fi + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + - name: Commit + id: commit + working-directory: crux-publish + env: + HIVE_SHA: ${{ github.sha }} + HIVE_REF: ${{ github.ref_name }} + TAG: ${{ steps.tag.outputs.tag }} + run: | + set -euo pipefail + git config user.name "hive-release[bot]" + git config user.email "hive-release@hive.kubestellar.io" + git add -A + if [ -n "$TAG" ]; then + # Release mode: never move main from a tag run (the tag may point + # at a hive commit older than what the push trigger has already + # synced, and a commit here would roll main back). Tag the commit + # already on main whose tree matches this hive commit instead. + target="" + for c in $(git rev-list -n 200 HEAD); do + if git diff --cached --quiet "$c" -- . ':!.hive-sync'; then target="$c"; break; fi + done + if [ -z "$target" ]; then + echo "::error::no commit on $CRUX_REPO main matches the publish tree of hive@${HIVE_SHA:0:12}; sync it first (push to v5 or workflow_dispatch without a tag), then re-run the tag." >&2 + exit 1 + fi + git reset -q --hard HEAD + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "target=$target" >> "$GITHUB_OUTPUT" + echo "Tag $TAG → existing synced commit $target (tree of hive@${HIVE_SHA:0:12})." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + # .hive-sync carries a timestamp, so it alone is not a change worth a commit. + if [ "$(git diff --cached --name-only)" = ".hive-sync" ]; then + git restore --staged --worktree .hive-sync + fi + if git diff --cached --quiet; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Publish tree already matches hive@${HIVE_SHA:0:12}; nothing to commit." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + git commit -q -s \ + -m "🌱 sync: hive@${HIVE_SHA:0:12} (src/crux)" \ + -m "Source: https://github.com/hivecommons/hive/tree/${HIVE_SHA}/src/crux" \ + -m "Ref: ${HIVE_REF}" \ + -m "Module path rewritten github.com/hivecommons/hive/crux -> github.com/hivecommons/crux by src/scripts/crux-publish.sh. File issues and pull requests in hivecommons/hive." + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "target=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + git --no-pager show --shortstat --format="%H %s" HEAD + + - name: Upload publish tree + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: crux-publish-tree + path: | + crux-publish + !crux-publish/.git + retention-days: 7 + + - name: Push + if: github.event.inputs.dry_run != 'true' && (steps.commit.outputs.changed == 'true' || steps.tag.outputs.tag != '') + working-directory: crux-publish + env: + TOKEN_SOURCE: ${{ steps.token.outputs.source }} + TAG: ${{ steps.tag.outputs.tag }} + TARGET: ${{ steps.commit.outputs.target }} + CHANGED: ${{ steps.commit.outputs.changed }} + HIVE_SHA: ${{ github.sha }} + run: | + set -euo pipefail + if [ "$TOKEN_SOURCE" = none ]; then + { + echo "## Crux sync: not pushed" + echo + echo "No \`CRUX_SYNC_TOKEN\` (or \`TOPUP_PUSH_TOKEN\`) secret reaches \`$CRUX_REPO\`." + echo "Provision a fine-grained PAT or App token with **Contents: write** on \`$CRUX_REPO\`" + echo "as repository secret \`CRUX_SYNC_TOKEN\`, then re-run. The publish tree is attached as" + echo "the \`crux-publish-tree\` artifact; see the manual recipe at the top of crux-sync.yml." + } >> "$GITHUB_STEP_SUMMARY" + echo "::error::no push token configured for $CRUX_REPO (CRUX_SYNC_TOKEN)" + exit 1 + fi + if [ "$CHANGED" = true ]; then + # Fast-forward only: the sync is one commit on top of whatever main + # is; if main moved under us the concurrency group has already + # serialised the other run, so a rejected push means a hand edit + # landed in the published repo and should be reconciled there. + if ! git push origin HEAD:main 2>push.err; then + cat push.err >&2 + if grep -qi "403\|permission\|denied" push.err; then + echo "::error::push to $CRUX_REPO rejected (token source: $TOKEN_SOURCE lacks Contents: write on $CRUX_REPO?)" + echo "## Crux sync: push rejected (token source: $TOKEN_SOURCE)" >> "$GITHUB_STEP_SUMMARY" + fi + exit 1 + fi + echo "Pushed $(git rev-parse HEAD) to $CRUX_REPO main (hive ${HIVE_SHA:0:12})" | tee -a "$GITHUB_STEP_SUMMARY" + fi + if [ -n "$TAG" ]; then + if git ls-remote --exit-code --tags origin "refs/tags/$TAG" >/dev/null; then + echo "::error::tag $TAG already exists on $CRUX_REPO; delete it there first if this is a re-cut." >&2 + exit 1 + fi + git tag -a "$TAG" -m "crux $TAG (hive ${HIVE_SHA})" "$TARGET" + git push origin "refs/tags/$TAG" + echo "Pushed tag $TAG at $TARGET → https://github.com/$CRUX_REPO/releases/tag/$TAG" | tee -a "$GITHUB_STEP_SUMMARY" + fi diff --git a/.github/workflows/dco-post-merge.yml b/.github/workflows/dco-post-merge.yml index 3f6113b423..9484d2a89f 100644 --- a/.github/workflows/dco-post-merge.yml +++ b/.github/workflows/dco-post-merge.yml @@ -263,6 +263,32 @@ env: # maintainer disposition from clubanderson in #13607: the maintainer merging # the PR that adds this entry IS that disposition. + # 00070300002c88fda6850278edaea2498e8ca941 — the squash of #13794 (review: + # one review kick in flight per agent; count load in kicks), on v5, reported + # by the monitor in #13903. Missing-signoff caused by the trailer being + # dropped when the squash message was edited: both branch commits + # (a99e07712, 8ff464a98) carried `Signed-off-by: Gregory Hunt + # ` matching the author, and the PR was reviewed and + # squash-merged by maintainer clubanderson. Protected v5 history cannot be + # rewritten to restore the trailer, so the correct instrument is this + # per-commit waiver. Accepted by maintainer disposition from clubanderson in + # #13904, which added the SHA to DCO_WAIVED_COMMITS; this entry completes the + # rationale block the waiver integrity self-test requires. + + # 511027d8174be02b1dd77c25e9f8d5265f7fb7d4 — the squash of #13997 (fix: + # preserve diagnostic and CI fields in process snapshots), on v5, reported + # by the monitor in #14064. Missing-signoff caused by the squash body being + # dropped at merge time: the single branch commit (b101cd0e) carried + # `Signed-off-by: thatguywob ` + # matching its author, the PR was reviewed (51/51 checks, DCO yes) and + # squash-merged by maintainer clubanderson with a subject-only message. + # Every App-merged PR in the same window kept its trailer, so this is a + # one-off from the human merge path, not a hive defect. Protected v5 + # history cannot be rewritten to restore the trailer, so the correct + # instrument is this per-commit waiver. Accepted by maintainer disposition + # from clubanderson in #14064: the maintainer merging the PR that adds + # this entry IS that disposition. + # KNOWN FAILING COMMITS THAT ARE DELIBERATELY *NOT* WAIVED # # Deep history carries further genuine mismatches — 8b536911 and, on v5 @@ -274,7 +300,7 @@ env: # regression. They are intentionally left unwaived: pre-approving them would # grow this list for commits that are not actually being reported, which is # the exact failure mode #6770 was filed about. - DCO_WAIVED_COMMITS: 'b798382a4d98ada28b0cb813644a595db0c58b08 7e01fee4a2c1a37f4a4072bf2d17e76e27113d01 d8f376e8e95f7b0c7d48d2b05b3a2a07ac9ee69b 4b9261183378a1a02a921cd0cc9df31e6f37993b 01bd2469e7a4022fff3fc04c42226c4d90c2f410 e29343efbb6e0721ca665ed8bd587d08997325e2 a8104b0cbeee1c4ac41034ac3f9951e6b5ac1200 692d068ffa198895ab91bd7ad663971f2b8ad358 b893b01c90e6ffff951523d20b19e9dfbfa80184 a0de3f920ded7e0cfc287bb5009d542e5a14f014 c6c78c3a26c33c6d1530996df20ead0a6b699dda a9200f3d278fcdbf6291f37f921845113814686e fb153585e6dab7b6f6808cb328e706d691f9be79' + DCO_WAIVED_COMMITS: 'b798382a4d98ada28b0cb813644a595db0c58b08 7e01fee4a2c1a37f4a4072bf2d17e76e27113d01 d8f376e8e95f7b0c7d48d2b05b3a2a07ac9ee69b 4b9261183378a1a02a921cd0cc9df31e6f37993b 01bd2469e7a4022fff3fc04c42226c4d90c2f410 e29343efbb6e0721ca665ed8bd587d08997325e2 a8104b0cbeee1c4ac41034ac3f9951e6b5ac1200 692d068ffa198895ab91bd7ad663971f2b8ad358 b893b01c90e6ffff951523d20b19e9dfbfa80184 a0de3f920ded7e0cfc287bb5009d542e5a14f014 c6c78c3a26c33c6d1530996df20ead0a6b699dda a9200f3d278fcdbf6291f37f921845113814686e fb153585e6dab7b6f6808cb328e706d691f9be79 00070300002c88fda6850278edaea2498e8ca941 511027d8174be02b1dd77c25e9f8d5265f7fb7d4' # Tide/tagged-release failures discussed in #6276 came from human squash # commits and release metadata, not bot-authored commits. Skip bots here so # release automation is not paged for bracketed bot-address edge cases that diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index db35cccdd3..ed5979414f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -305,7 +305,7 @@ jobs: - name: Log in to GHCR if: needs.gate.outputs.push == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -541,7 +541,7 @@ jobs: driver-opts: network=host - name: Log in to GHCR - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -603,7 +603,7 @@ jobs: # credential for public reads regardless of org. - name: Log in to GHCR as the cross-org publisher if: steps.crosspat.outputs.available == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -702,7 +702,7 @@ jobs: - name: Log in to GHCR if: needs.gate.outputs.push == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -778,7 +778,7 @@ jobs: driver-opts: network=host - name: Log in to GHCR - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -820,7 +820,7 @@ jobs: - name: Log in to GHCR as the cross-org publisher if: steps.crosspat.outputs.available == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -916,7 +916,7 @@ jobs: - name: Log in to GHCR if: needs.gate.outputs.push == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -1001,7 +1001,7 @@ jobs: driver-opts: network=host - name: Log in to GHCR - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -1043,7 +1043,7 @@ jobs: - name: Log in to GHCR as the cross-org publisher if: steps.crosspat.outputs.available == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/docs-link-check.yml b/.github/workflows/docs-link-check.yml index 1164b22587..023cb9bb56 100644 --- a/.github/workflows/docs-link-check.yml +++ b/.github/workflows/docs-link-check.yml @@ -50,6 +50,7 @@ on: - 'src/scripts/check-docs-citations.py' - 'src/scripts/test-check-docs-citations.sh' - 'src/scripts/check-security-docs-readability.sh' + - 'src/scripts/test-check-security-docs-readability.sh' - 'src/scripts/check-security-docs-rot.sh' - 'src/scripts/check-api-reference-citations.sh' - 'src/scripts/test-check-api-reference-citations.sh' @@ -74,6 +75,7 @@ on: - 'src/scripts/check-docs-citations.py' - 'src/scripts/test-check-docs-citations.sh' - 'src/scripts/check-security-docs-readability.sh' + - 'src/scripts/test-check-security-docs-readability.sh' - 'src/scripts/check-security-docs-rot.sh' - 'src/scripts/check-api-reference-citations.sh' - 'src/scripts/test-check-api-reference-citations.sh' @@ -123,6 +125,9 @@ jobs: # Security docs must stay readable by a non-expert: Flesch-Kincaid grade # 11 or lower for SECURITY.md and the src/docs/security*/ioscan* set. + - name: Self-test the security docs readability checker + run: bash src/scripts/test-check-security-docs-readability.sh + - name: Check security docs readability run: bash src/scripts/check-security-docs-readability.sh --verbose diff --git a/.github/workflows/forgejo-smoke.yml b/.github/workflows/forgejo-smoke.yml new file mode 100644 index 0000000000..88868a3eba --- /dev/null +++ b/.github/workflows/forgejo-smoke.yml @@ -0,0 +1,153 @@ +# Forgejo smoke — the minimal governor loop against a real Forgejo instance. +# +# Every Gitea/Forgejo adapter test in src/pkg/forge runs against httptest fakes +# that encode our reading of the API. This workflow starts a throwaway Forgejo +# container, seeds a repo, an issue and a pull request through its REST API, +# and runs src/pkg/forge/forgejo_smoke_test.go (build tag forgejo_smoke): read +# the issue → kick it (comment + label) → merge the PR through forge.Merger, +# including the hold gate (#14116, part of #14096). +# +# Cadence (maintainer ruling on #14116): nightly, plus push/PR runs +# path-filtered to the forge adapter, the API proxy write-gate and this file. +# NOT a required check yet — promote it once it has run green for two weeks. +# One job, one runner, 30-minute cap; the image is pinned by digest. Bump the +# tag and digest together. +name: Forgejo smoke + +on: + schedule: + # Nightly at :50 — the :00 (coverage-hourly), :30 (v2-tests) and :45 + # (backend-smoke) slots are taken. + - cron: '50 4 * * *' + push: + branches: + - v5 + paths: + - 'src/pkg/forge/**' + - 'src/pkg/apiproxy/**' + - '.github/workflows/forgejo-smoke.yml' + pull_request: + branches: + - v5 + paths: + - 'src/pkg/forge/**' + - 'src/pkg/apiproxy/**' + - '.github/workflows/forgejo-smoke.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +# Build-cache isolation for the self-hosted pool (see +# .github/scripts/go-cache-guard.sh). +env: + GO_CACHE_GUARD_BUILD_CACHE: ${{ vars.HIVE_GO_BUILD_CACHE || 'job' }} + FORGEJO_IMAGE: codeberg.org/forgejo/forgejo:16.0.6@sha256:e2c34a699543ca8a00217ee9ab2bea5feedc98bd6d0ea95ec287d3b7ae1355c6 + FORGEJO_URL: http://127.0.0.1:3000 + FORGEJO_ADMIN: hive-smoke + +jobs: + smoke: + if: github.repository == 'hivecommons/hive' + runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - name: Record start time + run: echo "SMOKE_STARTED=$(date +%s)" >> "$GITHUB_ENV" + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Warm shared Go tool cache (serialize the cold-cache population) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + cache: false + go-version-file: src/go.mod + - name: Guard shared Go cache (fall back when the runner volume is full) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + + - name: Start Forgejo + run: | + set -euo pipefail + name="forgejo-smoke-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "FORGEJO_CONTAINER=$name" >> "$GITHUB_ENV" + docker run -d --name "$name" -p 127.0.0.1:3000:3000 \ + -e USER_UID=1000 -e USER_GID=1000 \ + -e FORGEJO__database__DB_TYPE=sqlite3 \ + -e FORGEJO__security__INSTALL_LOCK=true \ + -e FORGEJO__server__ROOT_URL="$FORGEJO_URL/" \ + -e FORGEJO__server__DISABLE_SSH=true \ + -e FORGEJO__service__DISABLE_REGISTRATION=true \ + -e FORGEJO__repository__DEFAULT_BRANCH=main \ + "$FORGEJO_IMAGE" + for _ in $(seq 1 90); do + if curl -fsS "$FORGEJO_URL/api/healthz" >/dev/null 2>&1; then + curl -fsS "$FORGEJO_URL/api/v1/version"; echo + exit 0 + fi + sleep 2 + done + echo '::error::Forgejo did not become healthy within 3 minutes' + docker logs "$name" | tail -100 + exit 1 + + - name: Seed admin, token, repo, issue and pull request + run: | + set -euo pipefail + pass="Sm0ke-$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')" + docker exec -u git "$FORGEJO_CONTAINER" forgejo admin user create \ + --admin --username "$FORGEJO_ADMIN" --password "$pass" \ + --email "$FORGEJO_ADMIN@example.invalid" --must-change-password=false + api() { curl -fsS -H 'Content-Type: application/json' "$@"; } + token="$(api -u "$FORGEJO_ADMIN:$pass" -X POST "$FORGEJO_URL/api/v1/users/$FORGEJO_ADMIN/tokens" \ + -d '{"name":"smoke","scopes":["write:repository","write:issue","write:user"]}' | jq -r .sha1)" + echo "::add-mask::$token" + echo "FORGEJO_TOKEN=$token" >> "$GITHUB_ENV" + auth=(-H "Authorization: token $token") + repo="$FORGEJO_ADMIN/smoke" + api "${auth[@]}" -X POST "$FORGEJO_URL/api/v1/user/repos" \ + -d '{"name":"smoke","auto_init":true,"default_branch":"main","readme":"Default"}' >/dev/null + for l in 'hold:#e11d21' 'hive/kicked:#0e8a16'; do + api "${auth[@]}" -X POST "$FORGEJO_URL/api/v1/repos/$repo/labels" \ + -d "$(jq -nc --arg n "${l%%:*}" --arg c "${l#*:}" '{name:$n,color:$c}')" >/dev/null + done + # One commit on a new branch, via the contents API. + api "${auth[@]}" -X POST "$FORGEJO_URL/api/v1/repos/$repo/contents/SMOKE.md" \ + -d "$(jq -nc --arg c "$(printf 'forgejo smoke %s\n' "$GITHUB_RUN_ID" | base64 -w0)" \ + '{content:$c,message:"smoke: add SMOKE.md",branch:"main",new_branch:"smoke-change"}')" >/dev/null + issue="$(api "${auth[@]}" -X POST "$FORGEJO_URL/api/v1/repos/$repo/issues" \ + -d '{"title":"Smoke issue","body":"Seeded by the Forgejo smoke workflow."}' | jq -r .number)" + pr="$(api "${auth[@]}" -X POST "$FORGEJO_URL/api/v1/repos/$repo/pulls" \ + -d '{"title":"Smoke change","head":"smoke-change","base":"main","body":"Seeded by the Forgejo smoke workflow."}' | jq -r .number)" + { + echo "FORGEJO_REPO=$repo" + echo "FORGEJO_ISSUE=$issue" + echo "FORGEJO_PR=$pr" + } >> "$GITHUB_ENV" + echo "Seeded $repo issue #$issue, PR #$pr" + + - name: Minimal governor loop (read issue → kick → merge via forge.Merger) + working-directory: src + run: go test -tags forgejo_smoke -run '^TestForgejoSmoke$' -count=1 -v ./pkg/forge/ + + - name: Forgejo logs + if: failure() + run: docker logs "$FORGEJO_CONTAINER" 2>&1 | tail -200 + + - name: Stop Forgejo + if: always() + run: | + if [ -n "${FORGEJO_CONTAINER:-}" ]; then + docker rm -f -v "$FORGEJO_CONTAINER" >/dev/null 2>&1 || true + fi + + - name: Record job runtime + if: always() + run: | + secs=$(( $(date +%s) - ${SMOKE_STARTED:-$(date +%s)} )) + echo "Forgejo smoke runtime: ${secs}s" + echo "Forgejo smoke runtime: **${secs}s** on one runner (\`${{ runner.name }}\`)" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/go-security-analysis.yml b/.github/workflows/go-security-analysis.yml index 21667af798..7548710b52 100644 --- a/.github/workflows/go-security-analysis.yml +++ b/.github/workflows/go-security-analysis.yml @@ -171,7 +171,7 @@ jobs: - name: Scan working-directory: src - run: govulncheck ./... + run: govulncheck ./... ./crux/... gosec: needs: required-context-scope @@ -228,7 +228,7 @@ jobs: - name: Scan working-directory: src continue-on-error: true - run: gosec -exclude-generated ./... + run: gosec -exclude-generated ./... ./crux/... golangci-lint: needs: required-context-scope @@ -261,6 +261,9 @@ jobs: with: version: v2.13.1 working-directory: src + # src/go.work: ./... covers the root module only; the nested Crux + # module (#13536) is addressed as ./crux/... from src/. + args: ./... ./crux/... skip-cache: true action-pins: diff --git a/.github/workflows/pr-verifier.yml b/.github/workflows/pr-verifier.yml index b6e54b66f1..e296ce3387 100644 --- a/.github/workflows/pr-verifier.yml +++ b/.github/workflows/pr-verifier.yml @@ -37,7 +37,7 @@ jobs: # objected, so it must not be treated more harshly. - name: Login to GHCR continue-on-error: true - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/price-table-refresh.yml b/.github/workflows/price-table-refresh.yml index 8db2a3562e..e80afd77fa 100644 --- a/.github/workflows/price-table-refresh.yml +++ b/.github/workflows/price-table-refresh.yml @@ -66,7 +66,9 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: src/go.mod - cache-dependency-path: src/go.sum + cache-dependency-path: | + src/go.sum + src/crux/go.sum - name: Table age before refresh working-directory: src @@ -128,7 +130,11 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git checkout -B "$HEAD_BRANCH" # One fragment per refresh; drop any earlier unmerged one on this branch. - git rm -q --ignore-unmatch 'changelog.d/changed-price-table-*.md' || true + while IFS= read -r -d '' fragment; do + if ! git cat-file -e "origin/${BASE_BRANCH}:${fragment}" 2>/dev/null; then + git rm -q -- "$fragment" + fi + done < <(git ls-files -z -- 'changelog.d/changed-price-table-*.md') changes="$(sed -n 's/^### Changes (\([0-9]*\)).*/\1/p' "$RUNNER_TEMP/report.md")" if [ "${changes:-0}" -gt 0 ]; then printf -- '- Refreshed the model list-price table from OpenRouter (%s rows moved; list prices as of %s); the Cost panel estimate now uses the new rates.\n' "$changes" "$DATE" \ diff --git a/.github/workflows/promote-stable.yml b/.github/workflows/promote-stable.yml index b9baa56510..4dfd3f8a57 100644 --- a/.github/workflows/promote-stable.yml +++ b/.github/workflows/promote-stable.yml @@ -138,7 +138,7 @@ jobs: uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Log in to native GHCR - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -192,7 +192,7 @@ jobs: - name: Log in to GHCR as the cross-org publisher if: steps.crosspat.outputs.available == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/tagged-release.yml b/.github/workflows/tagged-release.yml index c8bf446d4b..fed1258b22 100644 --- a/.github/workflows/tagged-release.yml +++ b/.github/workflows/tagged-release.yml @@ -455,7 +455,7 @@ jobs: uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Log in to GHCR - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -558,7 +558,7 @@ jobs: - name: Log in to GHCR as the cross-org publisher if: steps.crosspat.outputs.available == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index fcf8f87550..7ffd508812 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -161,6 +161,25 @@ jobs: - name: Build run: go build ./cmd/hive/ + # The crux binary builds twice (#14138): the root-module host + # (./cmd/crux, Hive's hive.yaml loader) and the nested module's own + # ./crux/cmd/crux, which must build with GOWORK=off so a checkout of + # src/crux alone (or a downstream repo consuming the module) can + # `go build ./cmd/crux` without the workspace or the root module. + - name: Build crux (root host and standalone module) + run: | + go build -o /dev/null ./cmd/crux + (cd crux && GOWORK=off go build -o /dev/null ./cmd/crux) + + # crux/scripts/smoke.sh (`make -C crux smoke` locally; the self-hosted + # runners have no make) starts `crux serve` on a private unix socket + # with an empty runtime, drives it with `crux --server … ps` and GET + # /v1/health, then stops it. No agents or tmux involved; it fails the + # PR if the server stops starting, listening or shutting down cleanly. + - name: Crux module smoke (crux serve over a unix socket) + working-directory: src/crux + run: GOWORK=off bash scripts/smoke.sh + - name: CI install tool apt cache tests working-directory: . run: bash .github/scripts/test-ci-install-tool-apt-cache.sh @@ -177,6 +196,10 @@ jobs: working-directory: . run: python3 .github/scripts/test-runner-cache-prune.py + - name: Patch coverage script tests + working-directory: . + run: bash .github/scripts/test-patch-coverage.sh + # Runner-infra classifier, rerun-once guard and rate alert (#9664): # hermetic, real 2026-09-29 log excerpts as fixtures, about a second. - name: CI infra classifier tests @@ -184,7 +207,7 @@ jobs: run: python3 .github/scripts/test-ci-infra.py - name: Vet - run: go vet ./... + run: go vet ./... ./crux/... # The model list-price table behind /api/cost is refreshed weekly by # price-table-refresh.yml (which also re-dates it after 30 quiet days). @@ -1496,7 +1519,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Log in to GHCR - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/v2-tests.yml b/.github/workflows/v2-tests.yml index dc33af8c81..6061f75983 100644 --- a/.github/workflows/v2-tests.yml +++ b/.github/workflows/v2-tests.yml @@ -84,7 +84,7 @@ concurrency: # ── Sharding layout (#4118) ────────────────────────────────────────────────── # The suite is split so PR wall-clock ≈ the slowest single piece, not the sum. # Measured on CI (v4, ubuntu-latest, -short -race): pkg/hub ≈ 117s and -# pkg/agent ≈ 429s of TEST time each — every other package is ≤ 51s and they +# crux/agent ≈ 429s of TEST time each — every other package is ≤ 51s and they # all run concurrently inside one `go test` invocation. Round-robin over the # 840 sorted agent test names measures a worst slice of ~127s at N=5. So: # @@ -96,22 +96,36 @@ concurrency: # critical path until sliced the # same way as agent) — starts # immediately, self-partitions -# test (agent i/N) ./pkg/agent N jobs, each running a +# test (agent i/N) ./crux/agent N jobs, each running a # deterministic 1/N slice of the # package's test FUNCTIONS via -run # (the package is a serial 429s # monolith; package-level sharding # cannot help it) — starts # immediately, self-partitions -# list-packages go list minus hub/agent, greedily bucketed by measured -# weight into R balanced groups (job outputs → matrix) +# test (dashboard i/D) ./pkg/dashboard D jobs; same function slicing +# test (scheduler i/S) ./pkg/scheduler S jobs; same function slicing +# (one test-sliced job with an +# include matrix; both packages +# had grown into unsplittable +# ~200s+ rest-bucket floors — +# see the job comment) +# list-packages go list minus hub/agent/dashboard/scheduler, greedily +# bucketed by measured weight into R balanced groups +# (job outputs → matrix) # test (rest i/R) one bucket each # # Shard counts (H=4, N=5, R=5) were re-tuned on 2026-09-25 once the self-hosted # runner fleet dropped per-shard setup overhead to ~25s: at H=3/R=4 the gate # was bounded by hub 2/3 (185s) and rest 2/4 (180s) against a 4.0-minute run, # so one more slice of each shaves ~1 minute off the critical path at the cost -# of two runner pods. Re-tune when the slowest shard drifts past ~150s. +# of two runner pods. Re-tuned again on 2026-10-26 (D=4, S=3, R=4): the gate +# had drifted to `test (rest 1/5)` at 325s because pkg/dashboard alone ran +# 224s inside it (the weight table said 123s) and pkg/scheduler 194s (table: +# 74s). Slicing both out of the rest partition caps every rest bucket near its +# next-heaviest package (pkg/hub/spoke, ~85s) and makes the dashboard shuffle +# lane (~190s) the expected critical path. Re-tune when the slowest shard +# drifts past ~150s; the per-job step summary ranks each shard's slowest tests. # # Every shard uses the exact same flags: -short -race -count=1 (untouched), # plus -coverprofile for the parse-only coverage gate below, plus: @@ -133,14 +147,15 @@ concurrency: # test-dashboard-shuffle below for why both bounds are load-bearing. # # Completeness invariants (nothing can be silently untested): -# - rest buckets are computed by EXCLUDING exactly pkg/hub and pkg/agent from -# the full `go list` and PARTITIONING the remainder — a new package always -# lands in a bucket (a future subpackage of either sliced package included, -# since the slices run the bare package path), a removed one just -# disappears from its bucket. -# - hub and agent slices partition the sorted `go test -list` output round-robin — -# a new test function always lands in a slice. Anchored full-name regexes -# (^(A|B|...)$) make slices pairwise disjoint and jointly exhaustive. +# - rest buckets are computed by EXCLUDING exactly pkg/hub, crux/agent, +# pkg/dashboard and pkg/scheduler from the full `go list` and PARTITIONING +# the remainder — a new package always lands in a bucket (a future +# subpackage of any sliced package included, since the slices run the bare +# package path), a removed one just disappears from its bucket. +# - hub, dashboard, scheduler and agent slices partition the sorted +# `go test -list` output round-robin — a new test function always lands in +# a slice. Anchored full-name regexes (^(A|B|...)$) make slices pairwise +# disjoint and jointly exhaustive. # - the aggregate `test` job (the required-check name) fails closed: it # requires list-packages AND every shard to succeed. # @@ -156,27 +171,29 @@ concurrency: # Crux boundary filter (#13536), pull requests only, layered on the selection # above and consulted ONLY when it is `partial` (an `all` verdict from the # dependency analysis is never narrowed). .github/scripts/crux-scope.sh -# classifies the diff against the Crux package set — pkg/agent/... and -# pkg/{agentmode,agentparse,agentsmd,claude,sandbox,kubejob,toolapprove, -# skillreg,inference,openrouter,watchdog,trajectory,logscrub,proxy} — as: +# classifies the diff against the Crux package set — the nested Go module at +# src/crux/ (module github.com/hivecommons/hive/crux: crux/agent/... and its +# ring, addressed from src/ as ./crux/... through src/go.work) — as: # crux-only every change is inside the set: the agent slices run, the rest # buckets keep only the selected Crux packages plus the # always-on tree-scanning packages (internal/..., pkg/config, -# pkg/rotation), and the hub slices and the dashboard shuffle -# lane report success without testing. The inbound consumers of -# pkg/agent (dashboard, hub, cmd/hive, ...) are deliberately NOT +# pkg/rotation), and the hub, dashboard and scheduler slices and +# the dashboard shuffle lane report success without testing. +# The inbound consumers of crux/agent (dashboard, hub, +# cmd/hive, ...) are deliberately NOT # run on such a PR; v2-ci.yml still builds and vets the whole # tree, and the push/scheduled full runs are the behavioural net. # none no change touches the set: the agent slices report success -# without testing and pkg/agent is dropped from the selection +# without testing and crux/agent is dropped from the selection # EVEN IF the dependency analysis selected it (a PR that only -# edits a non-Crux package pkg/agent imports, e.g. pkg/github, -# does not run pkg/agent tests on the PR). This is the mirror of +# edits a root-module package, e.g. pkg/github, does not run +# crux/agent tests on the PR). This is the mirror of # the crux-only trade: the boundary is tested from inside the # set, and the push/scheduled full runs are the net. Everything # else follows the selection. # mixed both sides changed: the selection is used unchanged. # full a fail-safe path changed — src/go.mod, src/go.sum, +# src/go.work, src/crux/go.mod, src/crux/go.sum, # src/internal/**, src/pkg/config/**, .github/workflows/**, # .github/scripts/**, src/scripts/** — or the diff could not be # read: everything runs. @@ -243,7 +260,7 @@ jobs: fi # pkg/hub is one package of ~2,700 test functions that takes ~125s serially - # under -race — after #4118 sliced pkg/agent, this single job was the PR + # under -race — after #4118 sliced crux/agent, this single job was the PR # gate's critical path (~3.3 min wall clock against ~2.4 min for the widest # rest bucket). It is sliced exactly like test-agent below: each job derives # the same sorted `go test -list` output and runs rows where NR mod H == @@ -395,22 +412,80 @@ jobs: HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' run: | set -o pipefail - go test ./pkg/hub -short -list '.*' | grep -E '^(Test|Example|Fuzz)' | LC_ALL=C sort > all-tests.txt + ROOT="$PWD" + export LC_ALL=C # Regex length guards below must count bytes. + export TEST_PACKAGE=github.com/hivecommons/hive/pkg/hub + export TEST_DIR="$ROOT/pkg/hub" + export TEST_BINARY="$ROOT/hub.test" + go test -c -race -cover -o "$TEST_BINARY" ./pkg/hub + (cd "$TEST_DIR" && "$TEST_BINARY" -test.short -test.list '.*') | grep -E '^(Test|Example|Fuzz)' | LC_ALL=C sort > all-tests.txt if [ ! -s all-tests.txt ]; then echo "::error::go test -list produced no test functions for pkg/hub — refusing to run an empty slice (fail closed)" exit 1 fi awk -v n="$TOTAL" -v i="$IDX" 'NR % n == i - 1' all-tests.txt > slice.txt echo "slice $IDX/$TOTAL: $(wc -l < slice.txt) of $(wc -l < all-tests.txt) test functions" - RUN="^($(paste -sd'|' slice.txt))$" - SHUFFLE="" - if [ "$GITHUB_EVENT_NAME" != "pull_request" ]; then SHUFFLE="-shuffle=on"; fi - bash "$GITHUB_WORKSPACE/.github/scripts/ci-gotestsum-preserve-coverprofile.sh" \ - gotestsum --format=standard-verbose --rerun-fails=2 --rerun-fails-max-failures=5 \ + if [ ! -s slice.txt ]; then + echo "::error::empty hub slice — failing closed" + exit 1 + fi + export CHUNK_DIR TEST_SHUFFLE=off + CHUNK_DIR=$(mktemp -d "$ROOT/test-chunks.XXXXXX") + split -l 1000 slice.txt "$CHUNK_DIR/chunk-" + if [ "$GITHUB_EVENT_NAME" != "pull_request" ]; then TEST_SHUFFLE=on; fi + # raw-command preserves one JUnit/flake report across all chunks. + # gotestsum appends -test.run and package arguments on reruns. + cat > "$CHUNK_DIR/run.sh" <<'RUNNER' + #!/usr/bin/env bash + set -uo pipefail + cd "$TEST_DIR" || exit 1 + run_chunk() { + local RUN="$1" PROFILE="$2" + # MAX_ARG_STRLEN is 131072 bytes; never approach it, even if a + # future test has an exceptionally long name. Guard before exec. + if [ "${#RUN}" -gt 120000 ]; then + echo "::error::chunk regex exceeds 120000 bytes; reduce chunk size" >&2 + return 1 + fi + go tool test2json -t -p "$TEST_PACKAGE" "$TEST_BINARY" \ + -test.short -test.count=1 -test.v=test2json -test.timeout=10m \ + -test.shuffle="$TEST_SHUFFLE" "$RUN" -test.coverprofile="$PROFILE" + } + if [ "$#" -gt 0 ]; then + # Reruns must not overwrite the first-pass coverage. + run_chunk "$1" "$CHUNK_DIR/rerun.out" + exit $? + fi + status=0 + for chunk in "$CHUNK_DIR"/chunk-*; do + RUN="-test.run=^($(paste -sd'|' "$chunk"))$" + run_chunk "$RUN" "$chunk.out" || status=1 + done + exit "$status" + RUNNER + status=0 + if gotestsum --format=standard-verbose --rerun-fails=2 --rerun-fails-max-failures=5 \ --rerun-fails-report=flaky-tests-hub-"$IDX".txt \ - --packages="./pkg/hub" --junitfile junit-hub-"$IDX".xml \ - -- -short -race -count=1 -v $SHUFFLE -run "$RUN" -coverprofile=coverage-hub-"$IDX".out \ - 2>&1 | grep --line-buffered -vE '^=== (RUN|PAUSE|CONT) ' | tee test-hub-"$IDX".log + --junitfile junit-hub-"$IDX".xml --raw-command -- bash "$CHUNK_DIR/run.sh" \ + 2>&1 | grep --line-buffered -vE '^=== (RUN|PAUSE|CONT) ' | tee test-hub-"$IDX".log; then + : + else + status=1 + fi + # The test binary runs under test2json, not the `go test` driver, so + # nothing prints the `ok ` / `FAIL ` summary line the + # coverage scorer classifies packages by (#14135). Append it. + if [ "$status" -eq 0 ]; then + printf 'ok \t%s\t(slice %s/%s)\n' "$TEST_PACKAGE" "$IDX" "$TOTAL" >> test-hub-"$IDX".log + else + printf 'FAIL\t%s\t(slice %s/%s)\n' "$TEST_PACKAGE" "$IDX" "$TOTAL" >> test-hub-"$IDX".log + fi + # Sum first-pass counters for identical blocks; never count reruns. + # Do this even after test failure so the coverage gate has evidence. + awk 'FNR == 1 { next } { key=$1 " " $2; hits[key]+=$3 } + END { print "mode: atomic"; for (key in hits) print key, hits[key] }' \ + "$CHUNK_DIR"/chunk-*.out > coverage-hub-"$IDX".out + exit "$status" - name: Surface flaky reruns id: flaky-reruns @@ -445,7 +520,295 @@ jobs: if-no-files-found: warn retention-days: 1 - # pkg/agent is a single package whose tests take ~429s serially on CI — it + # Function-sliced lanes for the two root-module packages whose serial + # -short -race wall clock is the rest lane's floor: pkg/dashboard (~220s, + # ~7,300 test functions) and pkg/scheduler (~200s, mostly serial; one + # 75s test). Measured 2026-10-26 (v5 push runs 40383615331, 40382244011): + # `test (rest 1/5)` took 325s with pkg/dashboard alone running 224s of it, + # `test (rest 3/5)` 253s with pkg/scheduler at 194s, every other rest + # bucket ≤ 154s. A rest bucket's wall clock is its heaviest package (the + # packages run concurrently under one go test), so package-level + # rebalancing cannot move either floor. Both are sliced exactly like + # pkg/hub above and EXCLUDED from the rest partition (their subpackages — + # pkg/dashboard/webstatic, pkg/dashboard/collect, pkg/dashboard/theme — + # stay in the rest buckets). The context name is `test ( i/N)`; the + # coverage job merges the N partial profiles exactly as for hub and + # agent. Adding a lane = one include row per slice below plus a + # run- flag in list-packages and the aggregate `test` allowlist. + test-sliced: + needs: [required-context-scope, list-packages] + if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + name: test (${{ matrix.lane }} ${{ matrix.idx }}/${{ matrix.total }}) + strategy: + fail-fast: false + matrix: + include: + - { lane: dashboard, pkg: pkg/dashboard, idx: 1, total: 4 } + - { lane: dashboard, pkg: pkg/dashboard, idx: 2, total: 4 } + - { lane: dashboard, pkg: pkg/dashboard, idx: 3, total: 4 } + - { lane: dashboard, pkg: pkg/dashboard, idx: 4, total: 4 } + - { lane: scheduler, pkg: pkg/scheduler, idx: 1, total: 3 } + - { lane: scheduler, pkg: pkg/scheduler, idx: 2, total: 3 } + - { lane: scheduler, pkg: pkg/scheduler, idx: 3, total: 3 } + runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + # Affected-package selection (#12454): when this lane has nothing to + # test on a PR it still runs and reports success, so no context the + # aggregates or branch protection wait on can go missing. Anything but + # an explicit "nothing selected" runs the lane (fail toward testing). + - name: Affected-package selection + id: select + working-directory: . + env: + SELECTION: ${{ needs.list-packages.outputs.selection }} + LANE: ${{ matrix.lane }} + PKG: ${{ matrix.pkg }} + RUN_DASHBOARD: ${{ needs.list-packages.outputs.run-dashboard }} + RUN_SCHEDULER: ${{ needs.list-packages.outputs.run-scheduler }} + run: | + case "$LANE" in + dashboard) RUN_LANE=$RUN_DASHBOARD ;; + scheduler) RUN_LANE=$RUN_SCHEDULER ;; + *) RUN_LANE=true ;; + esac + if [ "$SELECTION" = partial ] && [ "$RUN_LANE" = false ]; then + echo "Affected-package selection: $PKG is not affected by this diff; reporting success without running tests. Push and scheduled runs test it in full." + echo "run=false" >> "$GITHUB_OUTPUT" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: steps.select.outputs.run == 'true' + + # Populate the shared node-local Go tool cache under the runner pods' + # node-wide lock before setup-go, so parallel jobs on a cold cache (a + # src/go.mod bump) stop clobbering each other's rm -rf + copy (#11175, + # #10234). Never fails the job. + - name: Warm shared Go tool cache (serialize the cold-cache population) + if: steps.select.outputs.run == 'true' + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + if: steps.select.outputs.run == 'true' + with: + cache: false + go-version-file: src/go.mod + - name: Guard shared Go cache (fall back when the runner volume is full) + if: steps.select.outputs.run == 'true' + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + + - name: Install gotestsum + if: steps.select.outputs.run == 'true' + run: | + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + go install gotest.tools/gotestsum@v1.13.0 + "$(go env GOPATH)/bin/gotestsum" --version + + # Warm compile cache (#7480 follow-up): the warm-build-cache job on every + # v4 push compiles the whole tree (race + norace) and saves GOCACHE keyed + # by SHA; PR jobs restore the nearest ancestor via the prefix restore-key, + # turning the ~2-minute per-job -race recompile into an incremental one. + # Restore-only in PR jobs: balance/miss costs seconds, never correctness. + - name: Restore warm Go build cache + if: ${{ steps.select.outputs.run == 'true' && (!contains(vars.HIVE_RUNNER_LABELS, 'hive-runners')) }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/go-build + key: hive-go-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + restore-keys: | + hive-go-build-${{ runner.os }}-${{ runner.arch }}- + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + if: steps.select.outputs.run == 'true' + with: + node-version: '22' + - name: Guard shared node tool cache (repair a half-written entry) + # setup-node reports "Found in cache" for a node//x64 entry whose + # .complete marker exists but bin/npm does not (#10505, #10518). The + # canary repairs this too, but its cron is starved to a ~5h cadence, so + # heal the node from the hot path: drop the marker (never the directory, + # a sibling job may be mid-run) so the next job re-extracts. + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: | + set -u + command -v npm >/dev/null 2>&1 && exit 0 + for marker in "${RUNNER_TOOL_CACHE:-/nonexistent}"/node/*/x64.complete; do + [ -e "$marker" ] || continue + dir="${marker%.complete}" + [ -x "$dir/bin/npm" ] && continue + echo "::warning::node tool cache $dir has a .complete marker but no executable bin/npm; removing the marker so the next job on $RUNNER_NAME rebuilds it" + rm -f "$marker" + done + echo "::error::npm is not on PATH after actions/setup-node on $RUNNER_NAME (half-written shared tool cache, see #10518); marker removed, re-run this job" + exit 1 + + - name: Restore cgo toolchain package cache + if: steps.select.outputs.run == 'true' + id: restore-cgo-apt-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + restore-keys: | + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev- + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}- + hive-cgo-race-apt- + + # Bounded + annotated by .github/scripts/ci-install-tool.sh (#6648/#6870/#6935): + # skip apt when the tool already exists, install restored .debs offline + # when possible, retry short egress flakes with backoff, and fail loudly + # with a ::error:: if neither cache nor mirrors can provide the tool. + - name: Prepare cgo toolchain for race tests + if: steps.select.outputs.run == 'true' + id: prepare-cgo-toolchain + env: + HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + run: | + bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ + --require gcc --for '-race tests' \ + --apt 'gcc libc6-dev' --apk 'gcc musl-dev' --verify 'gcc --version' + + - name: Save cgo toolchain package cache + if: ${{ steps.select.outputs.run == 'true' && (always() && steps.restore-cgo-apt-cache.outputs.cache-hit != 'true' && steps.prepare-cgo-toolchain.outputs.apt-cache-populated == 'true') }} + continue-on-error: true + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + + - name: Test slice of ${{ matrix.pkg }} + if: ${{ steps.select.outputs.run == 'true' && (steps.prepare-cgo-toolchain.outputs.toolchain-ready == 'true') }} + env: + IDX: ${{ matrix.idx }} + # Per-lane slice count: strategy.job-total would be the size of the + # whole include matrix, not of this package's slices. + TOTAL: ${{ matrix.total }} + LANE: ${{ matrix.lane }} + PKG: ${{ matrix.pkg }} + # The converted skips in these packages gate on repo files being + # present in the checkout — a precondition + # actions/checkout above GUARANTEES here, so a skip at a converted + # site means the test broke, not that the environment is unsuitable + # (#5388). Only testutil.SkipUnlessRequired sites become fatal; + # plain t.Skip capability gates are unaffected. + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: | + set -o pipefail + ROOT="$PWD" + export LC_ALL=C # Regex length guards below must count bytes. + export TEST_PACKAGE="github.com/hivecommons/hive/$PKG" + export TEST_DIR="$ROOT/$PKG" + export TEST_BINARY="$ROOT/$LANE.test" + go test -c -race -cover -o "$TEST_BINARY" "./$PKG" + (cd "$TEST_DIR" && "$TEST_BINARY" -test.short -test.list '.*') | grep -E '^(Test|Example|Fuzz)' | LC_ALL=C sort > all-tests.txt + if [ ! -s all-tests.txt ]; then + echo "::error::go test -list produced no test functions for $PKG — refusing to run an empty slice (fail closed)" + exit 1 + fi + awk -v n="$TOTAL" -v i="$IDX" 'NR % n == i - 1' all-tests.txt > slice.txt + echo "slice $IDX/$TOTAL: $(wc -l < slice.txt) of $(wc -l < all-tests.txt) test functions" + if [ ! -s slice.txt ]; then + echo "::error::empty $LANE slice — failing closed" + exit 1 + fi + export CHUNK_DIR TEST_SHUFFLE=off + CHUNK_DIR=$(mktemp -d "$ROOT/test-chunks.XXXXXX") + split -l 1000 slice.txt "$CHUNK_DIR/chunk-" + if [ "$GITHUB_EVENT_NAME" != "pull_request" ]; then TEST_SHUFFLE=on; fi + # raw-command preserves one JUnit/flake report across all chunks. + # gotestsum appends -test.run and package arguments on reruns. + cat > "$CHUNK_DIR/run.sh" <<'RUNNER' + #!/usr/bin/env bash + set -uo pipefail + cd "$TEST_DIR" || exit 1 + run_chunk() { + local RUN="$1" PROFILE="$2" + # MAX_ARG_STRLEN is 131072 bytes; never approach it, even if a + # future test has an exceptionally long name. Guard before exec. + if [ "${#RUN}" -gt 120000 ]; then + echo "::error::chunk regex exceeds 120000 bytes; reduce chunk size" >&2 + return 1 + fi + go tool test2json -t -p "$TEST_PACKAGE" "$TEST_BINARY" \ + -test.short -test.count=1 -test.v=test2json -test.timeout=10m \ + -test.shuffle="$TEST_SHUFFLE" "$RUN" -test.coverprofile="$PROFILE" + } + if [ "$#" -gt 0 ]; then + # Reruns must not overwrite the first-pass coverage. + run_chunk "$1" "$CHUNK_DIR/rerun.out" + exit $? + fi + status=0 + for chunk in "$CHUNK_DIR"/chunk-*; do + RUN="-test.run=^($(paste -sd'|' "$chunk"))$" + run_chunk "$RUN" "$chunk.out" || status=1 + done + exit "$status" + RUNNER + status=0 + if gotestsum --format=standard-verbose --rerun-fails=2 --rerun-fails-max-failures=5 \ + --rerun-fails-report=flaky-tests-"$LANE"-"$IDX".txt \ + --junitfile junit-"$LANE"-"$IDX".xml --raw-command -- bash "$CHUNK_DIR/run.sh" \ + 2>&1 | grep --line-buffered -vE '^=== (RUN|PAUSE|CONT) ' | tee test-"$LANE"-"$IDX".log; then + : + else + status=1 + fi + # The test binary runs under test2json, not the `go test` driver, so + # nothing prints the `ok ` / `FAIL ` summary line the + # coverage scorer classifies packages by (#14135). Append it. + if [ "$status" -eq 0 ]; then + printf 'ok \t%s\t(slice %s/%s)\n' "$TEST_PACKAGE" "$IDX" "$TOTAL" >> test-"$LANE"-"$IDX".log + else + printf 'FAIL\t%s\t(slice %s/%s)\n' "$TEST_PACKAGE" "$IDX" "$TOTAL" >> test-"$LANE"-"$IDX".log + fi + # Sum first-pass counters for identical blocks; never count reruns. + # Do this even after test failure so the coverage gate has evidence. + awk 'FNR == 1 { next } { key=$1 " " $2; hits[key]+=$3 } + END { print "mode: atomic"; for (key in hits) print key, hits[key] }' \ + "$CHUNK_DIR"/chunk-*.out > coverage-"$LANE"-"$IDX".out + exit "$status" + + - name: Surface flaky reruns + id: flaky-reruns + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/ci-gotestsum-flake-summary.sh" "flaky-tests-${{ matrix.lane }}-${{ matrix.idx }}.txt" "${{ matrix.lane }} ${{ matrix.idx }}/${{ matrix.total }}" + + - name: Upload flaky rerun report + if: ${{ steps.select.outputs.run == 'true' && (always() && steps.flaky-reruns.outputs.has-flakes == 'true') }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: flaky-tests-${{ matrix.lane }}-${{ matrix.idx }} + path: src/flaky-tests-${{ matrix.lane }}-${{ matrix.idx }}.txt + if-no-files-found: error + retention-days: 14 + + - name: Slowest tests in this slice + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/slowest-tests.sh" "${{ matrix.lane }} ${{ matrix.idx }}/${{ matrix.total }}" test-${{ matrix.lane }}-${{ matrix.idx }}.log + + # Uploaded even on failure so the coverage job (and humans) can inspect + # the raw `go test` output. `warn` not `error`: on a compile failure the + # coverprofile is never written, and this upload must not mask the real + # test failure with an artifact error. + - name: Upload shard coverage artifacts + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: test-shard-${{ matrix.lane }}-${{ matrix.idx }} + path: | + src/coverage-${{ matrix.lane }}-${{ matrix.idx }}.out + src/test-${{ matrix.lane }}-${{ matrix.idx }}.log + if-no-files-found: warn + retention-days: 1 + + # crux/agent (the Crux module's manager package, pkg/agent before the + # module cut) is a single package whose tests take ~429s serially on CI — it # was the entire "rest" shard's wall clock. Package-granular sharding cannot # split a package, so each of these jobs runs a deterministic 1/N slice of # the package's test functions. The slice is computed INSIDE the job (no @@ -455,7 +818,7 @@ jobs: # test function always lands in exactly one slice. N is ${{ strategy.job-total }}, # so adding an index to the matrix rebalances automatically. # - # Each slice produces a PARTIAL coverage profile for pkg/agent; the coverage + # Each slice produces a PARTIAL coverage profile for crux/agent; the coverage # job below merges them by summing per-block counts before scoring, so the # agent floor is applied to the union, never to a partial run. test-agent: @@ -484,7 +847,7 @@ jobs: RUN_LANE: ${{ needs.list-packages.outputs.run-agent }} run: | if [ "$SELECTION" = partial ] && [ "$RUN_LANE" = false ]; then - echo "Affected-package selection: pkg/agent is not affected by this diff; reporting success without running tests. Push and scheduled runs test it in full." + echo "Affected-package selection: crux/agent is not affected by this diff; reporting success without running tests. Push and scheduled runs test it in full." echo "run=false" >> "$GITHUB_OUTPUT" else echo "run=true" >> "$GITHUB_OUTPUT" @@ -567,7 +930,7 @@ jobs: # tmux is what makes HIVE_TEST_REQUIRE_BEHAVIOURAL legitimate below. # ubuntu-latest happens to ship it, but "happens to" is not a guarantee — - # a runner image change would silently turn ~50 pkg/agent tests back into + # a runner image change would silently turn ~50 crux/agent tests back into # skips, which is the exact failure #5388 exists to prevent. Install it # explicitly and fail the job if it is absent, so the flag below is # asserting something this lane actually enforces. @@ -579,12 +942,12 @@ jobs: --require tmux --for 'the behavioural gate' \ --apt tmux --apk tmux --verify 'tmux -V' - - name: Test slice of pkg/agent + - name: Test slice of crux/agent if: ${{ steps.select.outputs.run == 'true' && (steps.prepare-cgo-toolchain.outputs.toolchain-ready == 'true') }} env: IDX: ${{ matrix.idx }} TOTAL: ${{ strategy.job-total }} - # This lane GUARANTEES the preconditions the pkg/agent skips gate on: + # This lane GUARANTEES the preconditions the crux/agent skips gate on: # tmux is installed and verified above, and the rest are t.TempDir() # writes and TestMain-built fixtures. So a skip at a converted site # here means the test broke, not that the environment is unsuitable — @@ -594,26 +957,81 @@ jobs: HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' run: | set -o pipefail - # -list compiles the test binary (cached for the run below) and - # prints top-level Test/Example/Fuzz function names. All three kinds - # are included so none can fall out of every slice; Benchmarks are - # not run by `go test` without -bench, matching the unsharded run. - go test ./pkg/agent -short -list '.*' | grep -E '^(Test|Example|Fuzz)' | LC_ALL=C sort > all-tests.txt + ROOT="$PWD" + export LC_ALL=C # Regex length guards below must count bytes. + export TEST_PACKAGE=github.com/hivecommons/hive/crux/agent + export TEST_DIR="$ROOT/crux/agent" + export TEST_BINARY="$ROOT/agent.test" + go test -c -race -cover -o "$TEST_BINARY" ./crux/agent + # Include Test/Example/Fuzz, but not opt-in benchmarks. + (cd "$TEST_DIR" && "$TEST_BINARY" -test.short -test.list '.*') | grep -E '^(Test|Example|Fuzz)' | LC_ALL=C sort > all-tests.txt if [ ! -s all-tests.txt ]; then - echo "::error::go test -list produced no test functions for pkg/agent — refusing to run an empty slice (fail closed)" + echo "::error::go test -list produced no test functions for crux/agent — refusing to run an empty slice (fail closed)" exit 1 fi awk -v n="$TOTAL" -v i="$IDX" 'NR % n == i - 1' all-tests.txt > slice.txt echo "slice $IDX/$TOTAL: $(wc -l < slice.txt) of $(wc -l < all-tests.txt) test functions" - RUN="^($(paste -sd'|' slice.txt))$" - SHUFFLE="" - if [ "$GITHUB_EVENT_NAME" != "pull_request" ]; then SHUFFLE="-shuffle=on"; fi - bash "$GITHUB_WORKSPACE/.github/scripts/ci-gotestsum-preserve-coverprofile.sh" \ - gotestsum --format=standard-verbose --rerun-fails=2 --rerun-fails-max-failures=5 \ + if [ ! -s slice.txt ]; then + echo "::error::empty agent slice — failing closed" + exit 1 + fi + export CHUNK_DIR TEST_SHUFFLE=off + CHUNK_DIR=$(mktemp -d "$ROOT/test-chunks.XXXXXX") + split -l 1000 slice.txt "$CHUNK_DIR/chunk-" + if [ "$GITHUB_EVENT_NAME" != "pull_request" ]; then TEST_SHUFFLE=on; fi + # raw-command preserves one JUnit/flake report across all chunks. + # gotestsum appends -test.run and package arguments on reruns. + cat > "$CHUNK_DIR/run.sh" <<'RUNNER' + #!/usr/bin/env bash + set -uo pipefail + cd "$TEST_DIR" || exit 1 + run_chunk() { + local RUN="$1" PROFILE="$2" + # MAX_ARG_STRLEN is 131072 bytes; never approach it, even if a + # future test has an exceptionally long name. Guard before exec. + if [ "${#RUN}" -gt 120000 ]; then + echo "::error::chunk regex exceeds 120000 bytes; reduce chunk size" >&2 + return 1 + fi + go tool test2json -t -p "$TEST_PACKAGE" "$TEST_BINARY" \ + -test.short -test.count=1 -test.v=test2json -test.timeout=10m \ + -test.shuffle="$TEST_SHUFFLE" "$RUN" -test.coverprofile="$PROFILE" + } + if [ "$#" -gt 0 ]; then + # Reruns must not overwrite the first-pass coverage. + run_chunk "$1" "$CHUNK_DIR/rerun.out" + exit $? + fi + status=0 + for chunk in "$CHUNK_DIR"/chunk-*; do + RUN="-test.run=^($(paste -sd'|' "$chunk"))$" + run_chunk "$RUN" "$chunk.out" || status=1 + done + exit "$status" + RUNNER + status=0 + if gotestsum --format=standard-verbose --rerun-fails=2 --rerun-fails-max-failures=5 \ --rerun-fails-report=flaky-tests-agent-"$IDX".txt \ - --packages="./pkg/agent" --junitfile junit-agent-"$IDX".xml \ - -- -short -race -count=1 -v $SHUFFLE -run "$RUN" -coverprofile=coverage-agent-"$IDX".out \ - 2>&1 | grep --line-buffered -vE '^=== (RUN|PAUSE|CONT) ' | tee test-agent-"$IDX".log + --junitfile junit-agent-"$IDX".xml --raw-command -- bash "$CHUNK_DIR/run.sh" \ + 2>&1 | grep --line-buffered -vE '^=== (RUN|PAUSE|CONT) ' | tee test-agent-"$IDX".log; then + : + else + status=1 + fi + # The test binary runs under test2json, not the `go test` driver, so + # nothing prints the `ok ` / `FAIL ` summary line the + # coverage scorer classifies packages by (#14135). Append it. + if [ "$status" -eq 0 ]; then + printf 'ok \t%s\t(slice %s/%s)\n' "$TEST_PACKAGE" "$IDX" "$TOTAL" >> test-agent-"$IDX".log + else + printf 'FAIL\t%s\t(slice %s/%s)\n' "$TEST_PACKAGE" "$IDX" "$TOTAL" >> test-agent-"$IDX".log + fi + # Sum first-pass counters for identical blocks; never count reruns. + # Do this even after test failure so the coverage gate has evidence. + awk 'FNR == 1 { next } { key=$1 " " $2; hits[key]+=$3 } + END { print "mode: atomic"; for (key in hits) print key, hits[key] }' \ + "$CHUNK_DIR"/chunk-*.out > coverage-agent-"$IDX".out + exit "$status" - name: Surface flaky reruns id: flaky-reruns @@ -644,12 +1062,13 @@ jobs: if-no-files-found: warn retention-days: 1 - # Partition everything EXCEPT pkg/hub and pkg/agent (which have dedicated - # jobs above) into R balanced buckets, published as a matrix for test-rest. - # Greedy longest-processing-time assignment over static weight hints - # (measured seconds on CI under -short -race; unlisted packages default to - # 3s — they are all ≤ 8s). The hints only affect BALANCE, never MEMBERSHIP: - # a stale weight makes a bucket slower, not a package untested. + # Partition everything EXCEPT pkg/hub, crux/agent, pkg/dashboard and + # pkg/scheduler (which have dedicated sliced jobs above) into R balanced + # buckets, published as a matrix for test-rest. Greedy + # longest-processing-time assignment over static weight hints (measured + # seconds on CI under -short -race; unlisted packages default to 1s — they + # are all ≤ 5s). The hints only affect BALANCE, never MEMBERSHIP: a stale + # weight makes a bucket slower, not a package untested. list-packages: needs: required-context-scope if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} @@ -663,6 +1082,7 @@ jobs: run-hub: ${{ steps.select.outputs.run-hub }} run-agent: ${{ steps.select.outputs.run-agent }} run-dashboard: ${{ steps.select.outputs.run-dashboard }} + run-scheduler: ${{ steps.select.outputs.run-scheduler }} # Crux boundary filter (#13536): crux-only | none | mixed | full. crux-scope: ${{ steps.select.outputs.crux-scope }} defaults: @@ -765,15 +1185,18 @@ jobs: bash "$GITHUB_WORKSPACE/.github/scripts/affected-go-packages.sh" 'HEAD^1' HEAD "$sel" || echo all > "$sel" fi [ -f "$sel" ] || echo all > "$sel" - mod=$(go list -m) + # src/go.work puts `go list -m` in workspace mode, where it prints + # every workspace module; the selection file is keyed by the root + # module's import paths, so ask for that module alone. + mod=$(GOWORK=off go list -m) crux_scope=full if [ "$(head -n1 "$sel")" = all ]; then selection=full - run_hub=true run_agent=true run_dashboard=true + run_hub=true run_agent=true run_dashboard=true run_scheduler=true else selection=partial has() { grep -Fxq "$mod/$1" "$sel" && echo true || echo false; } - run_hub=$(has pkg/hub) run_agent=$(has pkg/agent) run_dashboard=$(has pkg/dashboard) + run_hub=$(has pkg/hub) run_agent=$(has crux/agent) run_dashboard=$(has pkg/dashboard) run_scheduler=$(has pkg/scheduler) # Crux boundary filter (#13536) — see the header comment. Only a # partial selection is ever narrowed; the script answers `full` # for every fail-safe path and for anything it cannot read, and @@ -782,18 +1205,16 @@ jobs: crux_scope=$(bash "$GITHUB_WORKSPACE/.github/scripts/crux-scope.sh" 'HEAD^1' HEAD) || crux_scope=full case "$crux_scope" in crux-only) - run_hub=false run_dashboard=false run_agent=true + run_hub=false run_dashboard=false run_scheduler=false run_agent=true # Rest buckets: the selected Crux packages plus the always-on # tree-scanning packages (mirrors ALWAYS_PATTERNS in # affected-go-packages.sh); a failure here falls back to the # unnarrowed selection. - if go list ./pkg/agent/... ./pkg/agentmode ./pkg/agentparse ./pkg/agentsmd ./pkg/claude \ - ./pkg/sandbox ./pkg/kubejob ./pkg/toolapprove ./pkg/skillreg ./pkg/inference \ - ./pkg/openrouter ./pkg/watchdog ./pkg/trajectory ./pkg/logscrub ./pkg/proxy \ + if go list ./crux/... \ ./internal/... ./pkg/config ./pkg/rotation > ../crux-pkgs.txt 2>/dev/null \ && [ -s ../crux-pkgs.txt ]; then grep -Fx -f ../crux-pkgs.txt "$sel" > "$sel.crux" || true - echo "crux-only diff: narrowing $(grep -c . "$sel" || true) selected packages to $(grep -c . "$sel.crux" || true) Crux/always-on packages; hub slices and dashboard shuffle lane will not run" + echo "crux-only diff: narrowing $(grep -c . "$sel" || true) selected packages to $(grep -c . "$sel.crux" || true) Crux/always-on packages; hub, dashboard and scheduler slices and the dashboard shuffle lane will not run" mv "$sel.crux" "$sel" else echo "::warning::crux-only diff but go list of the Crux set failed; keeping the unnarrowed selection" @@ -801,11 +1222,11 @@ jobs: fi ;; none) - # Drop pkg/agent from the selection too: the coverage job + # Drop crux/agent from the selection too: the coverage job # requires every selected package to have produced a result, # and the agent slices are the only lane that tests it. run_agent=false - grep -Fxv "$mod/pkg/agent" "$sel" > "$sel.nocrux" || true + grep -Fxv "$mod/crux/agent" "$sel" > "$sel.nocrux" || true mv "$sel.nocrux" "$sel" echo "no Crux package changed: agent slices will not run" ;; @@ -822,6 +1243,7 @@ jobs: echo "run-hub=$run_hub" echo "run-agent=$run_agent" echo "run-dashboard=$run_dashboard" + echo "run-scheduler=$run_scheduler" echo "crux-scope=$crux_scope" } >> "$GITHUB_OUTPUT" { @@ -830,27 +1252,33 @@ jobs: if [ "$selection" = full ]; then echo "Full suite." else - echo "Affected packages only ($(grep -c . "$sel" || true)); crux-scope=$crux_scope; hub=$run_hub agent=$run_agent dashboard-shuffle=$run_dashboard." + echo "Affected packages only ($(grep -c . "$sel" || true)); crux-scope=$crux_scope; hub=$run_hub agent=$run_agent dashboard=$run_dashboard scheduler=$run_scheduler." echo "" sed 's/^/- /' "$sel" fi } >> "$GITHUB_STEP_SUMMARY" - echo "selection=$selection crux-scope=$crux_scope hub=$run_hub agent=$run_agent dashboard=$run_dashboard" + echo "selection=$selection crux-scope=$crux_scope hub=$run_hub agent=$run_agent dashboard=$run_dashboard scheduler=$run_scheduler" - name: Partition packages into balanced buckets id: partition run: | - R=5 - # Drop only the monolithic pkg/hub package: the hub job shards it - # by test function. All hub subpackages belong in the rest buckets + R=4 + # Drop only the monolithic packages that have function-sliced jobs + # of their own: pkg/hub (test-hub), pkg/dashboard and pkg/scheduler + # (test-sliced). All their subpackages belong in the rest buckets # so coverage floors and the no-test-files gate apply to them too. - # pkg/hubbackup et al. stay here. + # pkg/hubbackup, pkg/dashboard/webstatic et al. stay here. # ./internal/... carries the shared test helpers (internal/testutil) # and their contract tests, such as the time.Sleep ratchet in - # sleep_ratchet_test.go. - go list ./pkg/... ./cmd/... ./internal/... \ - | awk '$0 == "github.com/hivecommons/hive/pkg/hub" { next } { print }' \ - | grep -Ev '^github\.com/hivecommons/hive/pkg/agent$' \ + # sleep_ratchet_test.go. ./crux/... is the nested Crux module + # (#13536), reachable from src/ through src/go.work; its manager + # package crux/agent has the dedicated agent slices. + go list ./pkg/... ./cmd/... ./internal/... ./crux/... \ + | awk '$0 == "github.com/hivecommons/hive/pkg/hub" { next } + $0 == "github.com/hivecommons/hive/pkg/dashboard" { next } + $0 == "github.com/hivecommons/hive/pkg/scheduler" { next } + { print }' \ + | grep -Ev '^github\.com/hivecommons/hive/crux/agent$' \ | LC_ALL=C sort > rest-pkgs.txt if [ ! -s rest-pkgs.txt ]; then echo "::error::go list produced no rest packages — failing closed" @@ -867,38 +1295,51 @@ jobs: fi # Weights = measured -short -race package wall seconds on the - # hive-runners fleet, v5 push cfb64d5 (run 36079964584, 2026-09-25). - # Re-measure with: for each `test (rest i/R)` job log, grep the + # hive-runners fleet, mean of two v5 push runs (40383615331 and + # 40382244011, 2026-10-26, rounded). Re-measure with: for each + # `test (rest i/R)` job log, grep the # `ok github.com/hivecommons/hive/ s` lines. Anything - # under ~5s is noise and takes the 1s default. Packages inside a bucket - # run concurrently under one go test, so a bucket's wall clock tracks - # its heaviest package — the table mainly has to keep the big ones apart. pkg/dashboard is the - # floor: one unsplittable package, so the widest bucket can never - # be faster than it. + # under ~5s is noise and takes the 1s default. Packages inside a + # bucket run concurrently under one go test, so a bucket's wall + # clock is its heaviest package plus compile time — the table + # mainly has to keep the big ones apart, and the sum-based greedy + # below does that while spreading the compile load. pkg/hub/spoke + # is the floor now that pkg/dashboard and pkg/scheduler are sliced + # (test-sliced above): the widest bucket can never be faster than + # it. pkg/dashboard and pkg/scheduler are listed so a stale + # exclusion above still yields a sane partition. pkg/github is + # ~1s with ~1% coverage on push runs but 70s on pull requests (PR + # 14407's run 40387365403), where it is weighed — measure it from a + # PR run, not a push run. weight() { case "$1" in - */pkg/dashboard) echo 123 ;; + */pkg/dashboard) echo 219 ;; + */pkg/scheduler) echo 199 ;; */pkg/hub/spoke) echo 85 ;; - */pkg/scheduler) echo 74 ;; - */pkg/github) echo 62 ;; - */pkg/config) echo 57 ;; - */pkg/proxy) echo 55 ;; - */cmd/hive) echo 30 ;; - */pkg/hubbackup) echo 29 ;; - */pkg/ioscan) echo 22 ;; - */pkg/hivectl) echo 19 ;; - */pkg/discord) echo 13 ;; - */pkg/tui) echo 13 ;; - */pkg/apphealth) echo 10 ;; - */pkg/dashboard/webstatic) echo 9 ;; - */pkg/advisory) echo 9 ;; - */internal/testutil) echo 9 ;; - */pkg/rotation) echo 8 ;; - */pkg/hivectl/commands) echo 7 ;; - */pkg/inference) echo 6 ;; - */pkg/auth) echo 6 ;; - */pkg/appkey) echo 5 ;; + */pkg/config) echo 82 ;; + */pkg/github) echo 70 ;; + */cmd/hive) echo 69 ;; + */crux/proxy) echo 56 ;; + */pkg/dashboard/webstatic) echo 35 ;; + */pkg/hubbackup) echo 30 ;; + */pkg/advisory) echo 29 ;; + */internal/testutil) echo 22 ;; + */pkg/ioscan) echo 16 ;; + */pkg/hivectl) echo 15 ;; + */pkg/tui) echo 14 ;; + */pkg/discord) echo 12 ;; + */pkg/fleetreport) echo 11 ;; + */pkg/hivectl/commands) echo 10 ;; + */pkg/stageengine) echo 10 ;; + */pkg/apphealth) echo 9 ;; + */pkg/rotation) echo 9 ;; + */pkg/beads) echo 8 ;; + */pkg/github/automerge) echo 8 ;; + */crux/api/callback) echo 6 ;; + */crux/inference) echo 6 ;; */cmd/hive-backup) echo 5 ;; + */pkg/mint) echo 5 ;; + */pkg/policies) echo 5 ;; *) echo 1 ;; esac } @@ -1084,8 +1525,9 @@ jobs: echo "::error::empty package bucket — failing closed" exit 1 fi - # -timeout=20m: pkg/dashboard alone nearly exhausts go test's default - # 10m on loaded PR runners (#12113) though each test runs seconds. + # -timeout=20m: kept from when pkg/dashboard lived in these buckets + # and nearly exhausted go test's default 10m on loaded PR runners + # (#12113); a loaded runner can still stretch pkg/hub/spoke. SHUFFLE="" if [ "$GITHUB_EVENT_NAME" != "pull_request" ]; then SHUFFLE="-shuffle=on"; fi # shellcheck disable=SC2086 # PKGS and SHUFFLE are deliberate word-split lists @@ -1141,7 +1583,8 @@ jobs: # fresh seed each run, and without the seed a real order-dependence # finding is unreproducible. # No -coverprofile here: pkg/dashboard's coverage floor is scored from the - # rest-bucket profile; this lane exists only to vary the execution order. + # merged test-sliced `test (dashboard i/D)` profiles; this lane exists only + # to vary the execution order. test-dashboard-shuffle: needs: [required-context-scope, list-packages] if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} @@ -1287,29 +1730,64 @@ jobs: # Sliced like test-hub (NR mod N — pairwise disjoint, jointly # exhaustive) so no single job carries the whole ~6-minute lane; # -shuffle still varies order within each slice every run. - go test ./pkg/dashboard -short -list '.*' | grep -E '^(Test|Example|Fuzz)' | LC_ALL=C sort > all-tests.txt + ROOT="$PWD" + export TEST_PACKAGE=github.com/hivecommons/hive/pkg/dashboard + export TEST_DIR="$ROOT/pkg/dashboard" + export TEST_BINARY="$ROOT/dashboard.test" + go test -c -race -o "$TEST_BINARY" ./pkg/dashboard + (cd "$TEST_DIR" && "$TEST_BINARY" -test.short -test.list '.*') | grep -E '^(Test|Example|Fuzz)' | LC_ALL=C sort > all-tests.txt if [ ! -s all-tests.txt ]; then echo "::error::go test -list produced no test functions for pkg/dashboard — refusing to run an empty slice (fail closed)" exit 1 fi awk -v n="$TOTAL" -v i="$IDX" 'NR % n == i - 1' all-tests.txt > slice.txt echo "slice $IDX/$TOTAL: $(wc -l < slice.txt) of $(wc -l < all-tests.txt) test functions" - RUN="^($(paste -sd'|' slice.txt))$" - # Bound on concurrent t.Parallel tests inside the package: high - # enough to keep the lane fast, low enough that the package's many - # httptest servers cannot exhaust ephemeral ports (see job comment). - PARALLEL_TESTS=4 + if [ ! -s slice.txt ]; then + echo "::error::empty dashboard slice — failing closed" + exit 1 + fi + export CHUNK_DIR + CHUNK_DIR=$(mktemp -d "$ROOT/test-chunks.XXXXXX") + split -l 1000 slice.txt "$CHUNK_DIR/chunk-" + # raw-command preserves one JUnit/flake report across all chunks. + # gotestsum appends -test.run and package arguments on reruns. + cat > "$CHUNK_DIR/run.sh" <<'RUNNER' + #!/usr/bin/env bash + set -uo pipefail + export LC_ALL=C # ${#RUN} must count bytes, not Unicode characters. + cd "$TEST_DIR" || exit 1 + run_chunk() { + local RUN="$1" + # MAX_ARG_STRLEN is 131072 bytes; fail clearly before execve. + if [ "${#RUN}" -gt 120000 ]; then + echo "::error::chunk regex exceeds 120000 bytes; reduce chunk size" >&2 + return 1 + fi + # Bound t.Parallel to avoid exhausting httptest ephemeral ports. + go tool test2json -t -p "$TEST_PACKAGE" "$TEST_BINARY" \ + -test.short -test.count=1 -test.v=test2json -test.timeout=10m \ + -test.shuffle=on -test.parallel=4 "$RUN" + } + if [ "$#" -gt 0 ]; then + run_chunk "$1" + exit $? + fi + status=0 + for chunk in "$CHUNK_DIR"/chunk-*; do + RUN="-test.run=^($(paste -sd'|' "$chunk"))$" + run_chunk "$RUN" || status=1 + done + exit "$status" + RUNNER if gotestsum --format=standard-verbose --rerun-fails=2 --rerun-fails-max-failures=5 \ --rerun-fails-report=flaky-tests-dashboard-shuffle-"$IDX".txt \ - --packages="./pkg/dashboard" --junitfile junit-dashboard-shuffle-"$IDX".xml \ - -- -short -race -count=1 -v -shuffle=on -p 1 -parallel "$PARALLEL_TESTS" -run "$RUN" \ + --junitfile junit-dashboard-shuffle-"$IDX".xml --raw-command -- bash "$CHUNK_DIR/run.sh" \ 2>&1 | grep --line-buffered -vE '^=== (RUN|PAUSE|CONT) ' | tee test-dashboard-shuffle-"$IDX".log; then exit 0 fi - # The test binary prints "-test.shuffle " as its first line; - # put the seed in the annotation so the failure is reproducible. - SEED=$(grep -m1 -E '^-test\.shuffle [0-9]+' test-dashboard-shuffle-"$IDX".log | awk '{print $2}') - echo "::error::pkg/dashboard failed under shuffled order (slice $IDX/$TOTAL). Reproduce from src/ with: go test ./pkg/dashboard -short -race -count=1 -p 1 -parallel $PARALLEL_TESTS -run '$RUN' -shuffle=${SEED:-}" + # Each chunk prints its own -test.shuffle seed immediately before + # its tests; do not report the first chunk's seed for a later failure. + echo "::error::pkg/dashboard failed (slice $IDX/$TOTAL); see the failing chunk's -test.shuffle seed in test-dashboard-shuffle-$IDX.log. Reproduce the failing test with: go test ./pkg/dashboard -short -race -count=1 -p 1 -parallel 4 -run '^TestName$' -shuffle=" exit 1 - name: Surface flaky reruns @@ -1355,12 +1833,13 @@ jobs: # outage does not make otherwise-mergeable PRs impossible to merge. runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 10 - needs: [required-context-scope, list-packages, test-hub, test-agent, test-rest, test-dashboard-shuffle] + needs: [required-context-scope, list-packages, test-hub, test-sliced, test-agent, test-rest, test-dashboard-shuffle] steps: - name: Check shard results env: LIST_PACKAGES_RESULT: ${{ needs.list-packages.result }} TEST_HUB_RESULT: ${{ needs.test-hub.result }} + TEST_SLICED_RESULT: ${{ needs.test-sliced.result }} TEST_AGENT_RESULT: ${{ needs.test-agent.result }} TEST_REST_RESULT: ${{ needs.test-rest.result }} TEST_DASHBOARD_SHUFFLE_RESULT: ${{ needs.test-dashboard-shuffle.result }} @@ -1371,6 +1850,7 @@ jobs: RUN_HUB: ${{ needs.list-packages.outputs.run-hub }} RUN_AGENT: ${{ needs.list-packages.outputs.run-agent }} RUN_DASHBOARD: ${{ needs.list-packages.outputs.run-dashboard }} + RUN_SCHEDULER: ${{ needs.list-packages.outputs.run-scheduler }} run: | set -euo pipefail @@ -1398,13 +1878,15 @@ jobs: allow=() [ "${RUN_HUB:-true}" = false ] && allow+=(test-hub) [ "${RUN_AGENT:-true}" = false ] && allow+=(test-agent) - [ "${RUN_DASHBOARD:-true}" = false ] && allow+=(test-dashboard-shuffle) + [ "${RUN_DASHBOARD:-true}" = false ] && allow+=(test-dashboard-shuffle "test-sliced (dashboard)") + [ "${RUN_SCHEDULER:-true}" = false ] && allow+=("test-sliced (scheduler)") echo "Crux scope: ${CRUX_SCOPE:-}; lanes allowed to succeed without testing: ${allow[*]:-}" fi shard_names=( list-packages test-hub + test-sliced test-agent test-rest test-dashboard-shuffle @@ -1412,6 +1894,7 @@ jobs: shard_results=( "${LIST_PACKAGES_RESULT:-}" "${TEST_HUB_RESULT:-}" + "${TEST_SLICED_RESULT:-}" "${TEST_AGENT_RESULT:-}" "${TEST_REST_RESULT:-}" "${TEST_DASHBOARD_SHUFFLE_RESULT:-}" @@ -1484,7 +1967,7 @@ jobs: # # Scoring is computed from the coverage PROFILES, not from the `ok ... # coverage: X%` log lines: the agent slices each emit a PARTIAL percentage - # for pkg/agent, so per-block counts from ALL shard profiles are merged + # for crux/agent, so per-block counts from ALL shard profiles are merged # (summed) first and every package is scored on the union. For unsplit # packages this yields the identical number `go test` prints. The logs are # still parsed for the two states a profile cannot express: FAIL and @@ -1495,20 +1978,20 @@ jobs: # outage does not make otherwise-mergeable PRs impossible to merge. runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 30 - needs: [required-context-scope, list-packages, test-hub, test-agent, test-rest] + needs: [required-context-scope, list-packages, test-hub, test-sliced, test-agent, test-rest] steps: - name: Report coverage not required for this diff if: ${{ needs.required-context-scope.result == 'success' && needs.required-context-scope.outputs.run-tests != 'true' }} run: echo "No v2 coverage paths changed; reporting required coverage context success." - name: Fail if scope failed or a required test shard did not complete - if: ${{ needs.required-context-scope.result != 'success' || (needs.required-context-scope.outputs.run-tests == 'true' && (needs.test-hub.result != 'success' || needs.test-agent.result != 'success' || needs.test-rest.result != 'success')) }} + if: ${{ needs.required-context-scope.result != 'success' || (needs.required-context-scope.outputs.run-tests == 'true' && (needs.test-hub.result != 'success' || needs.test-sliced.result != 'success' || needs.test-agent.result != 'success' || needs.test-rest.result != 'success')) }} run: | if [ "${{ needs.required-context-scope.result }}" != success ]; then echo "::error::required-context-scope failed; refusing to report coverage success without a diff verdict" exit 1 fi - echo "::error::Coverage requires successful test shards (hub=${{ needs.test-hub.result }}, agent=${{ needs.test-agent.result }}, rest=${{ needs.test-rest.result }})" + echo "::error::Coverage requires successful test shards (hub=${{ needs.test-hub.result }}, sliced=${{ needs.test-sliced.result }}, agent=${{ needs.test-agent.result }}, rest=${{ needs.test-rest.result }})" exit 1 # Affected-package selection (#12454). On a partial PR run only the @@ -1546,7 +2029,9 @@ jobs: if: ${{ steps.cov-scope.outputs.score == 'true' }} with: go-version-file: src/go.mod - cache-dependency-path: src/go.sum + cache-dependency-path: | + src/go.sum + src/crux/go.sum - name: List packages present in checkout if: ${{ steps.cov-scope.outputs.score == 'true' }} @@ -1558,7 +2043,7 @@ jobs: # scores must still be bounded by the checked-out tree for this run. # This mirrors coverage-hourly.yml, whose `go test` command can only # emit packages that exist in the checkout. - go list ./pkg/... ./cmd/... ./internal/... | LC_ALL=C sort -u > ../existing-pkgs.txt + go list ./pkg/... ./cmd/... ./internal/... ./crux/... | LC_ALL=C sort -u > ../existing-pkgs.txt if [ ! -s ../existing-pkgs.txt ]; then echo "::error::go list produced no packages for coverage scoring" exit 1 @@ -1569,7 +2054,7 @@ jobs: # `coverage: 0.0%` lines a full run classifies below. : > ../no-test-pkgs.txt if [ "$SELECTION" = partial ]; then - go list -f '{{if not (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}' ./pkg/... ./cmd/... \ + go list -f '{{if not (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}' ./pkg/... ./cmd/... ./crux/... \ | { grep . || true; } | LC_ALL=C sort -u > ../no-test-pkgs.txt fi @@ -1608,7 +2093,7 @@ jobs: [dashboard]=84 # 86.2% on 2026-09-01; the residue is the inception tmux watcher + copilot/claude device-flow + /data file I/O + k8s [dashchat]=64 # initial browser transport seam (#8304); chat.Service command paths are covered in pkg/chat, surface tests pin guards [hub]=87 # kubectl-shelling + live-GitHub paths; see #2355. Was 89 against v2's 89.9%; v4 measures 87.6%. - [agent]=88 # 88.9% after the v5→v6 top-up; pre-existing shortfall tracked in #2355. + [crux/agent]=88 # 88.9% after the v5->v6 top-up (#14340); pre-existing shortfall inherited from #2350, NOT a new regression. See #2355. Keyed crux/ since the module cut (#13536) [adminmcp]=53 # 53.3% on PR #8807; phase-1 skeleton covers refusal/cap/scrub/core JSON-RPC paths while provider breadth lands in later phases # ── Floors recorded when the gate moved from v2 to v4 (#3903) ───── @@ -1632,11 +2117,12 @@ jobs: [github]=88 # 88.9% on 2026-10-05 [hivectl]=88 [hivectl/commands]=89 # 89.9% after the v5→v6 forward-merge (#13896); the union of both branches' tests sits just under the 90 default - [proxy]=89 + [knowledge]=89 + [crux/proxy]=89 [ioscan]=96 - [sandbox]=98 - [pushbroker]=99 - [kubejob]=90 # new in #6311; sandbox.Launcher over the in-cluster API, tested against httptest + [crux/sandbox]=98 + [crux/pushbroker]=99 + [crux/kubejob]=90 # new in #6311; sandbox.Launcher over the in-cluster API, tested against httptest # ── Stale module-path parser repaired on v5 (#6066) ───────────── # These packages were pre-existing below-default floors that became @@ -1684,8 +2170,16 @@ jobs: [cmd/hive-admin-mcp]=44 # 44.5% on PR #8807; stdio SDK/run wiring remains transport-bound while roster/refusal paths are tested [cmd/hivectl]=97 # 100.0% on 2026-09-17; was exempt as an untested main [cmd/flue-fixture]=75 # 80.0% on 2026-09-22 (#8361): main() is the single uncovered statement; run() and signalContext() are tested in-process and the binary is smoke-tested as a child process, which yields no profile - [cmd/crux]=75 # 80.0% on 2026-10-23 (#13536): same shape as flue-fixture; main() is the one uncovered statement, run() and signalContext() are tested in-process + [cmd/crux]=75 # 80.0% on 2026-10-23 (#13536): same shape as flue-fixture; main() is the one uncovered statement, run() is tested in-process. floor_key maps BOTH ./cmd/crux (root host) and ./crux/cmd/crux (the module's own binary, #14138) here; the module one measured 80.0% on 2026-10-25 [crux/cli]=92 # 97.8% on 2026-10-23 (#13536): the crux CLI against a fake crux.Runtime plus a real unix control socket; the residue is Main() and write-error branches + + # ── Crux module (#13536, Stage 2 step 4) ───────────────────────── + # Packages of the nested module at src/crux are keyed "crux/" + # (floor_key below): crux/agent, crux/proxy, crux/sandbox, + # crux/pushbroker, crux/kubejob and crux/cli above carried their + # pkg/ floors across the move unchanged. The module root + # package is keyed "crux". Every other crux/ takes the + # default. ) # Thin flag-parsing mains with no test files, wrapping packages the @@ -1698,7 +2192,7 @@ jobs: UNTESTED_CMD_MAINS='' # Merge every shard profile: per-block counts are summed, so the N - # partial pkg/agent profiles combine into the package's true + # partial crux/agent profiles combine into the package's true # coverage. Blocks are keyed by (location, statement count); package # = the directory of the block's file. Emits "pkg pct" lines plus a # TOTAL over ./pkg/... — the same statements-covered/statements @@ -1708,8 +2202,10 @@ jobs: # a hard-coded module prefix. The previous version matched the literal # module path, so the kubestellar -> hivecommons org rename silently # emptied every filter and disabled the whole gate. A segment match - # survives any future rename, and only this module's own packages - # appear in these profiles, so it cannot over-match. + # survives any future rename, and only this repository's own packages + # appear in these profiles, so it cannot over-match. The nested Crux + # module's packages carry a "/crux" segment (github.com//hive/crux + # and .../crux/) and get a second, informational total. awk ' FNR == 1 { next } # skip each file'\''s "mode:" line { @@ -1725,19 +2221,24 @@ jobs: tot[p] += stmts[k] if (count[k] > 0) cov[p] += stmts[k] } - gt = 0; gc = 0 + gt = 0; gc = 0; xt = 0; xc = 0 for (p in tot) { printf "%s %.1f\n", p, cov[p] * 100 / tot[p] if (p ~ /\/pkg\//) { gt += tot[p]; gc += cov[p] + } else if (p ~ /\/crux(\/|$)/ && p !~ /\/cmd\//) { + xt += tot[p]; xc += cov[p] } } if (gt > 0) printf "TOTAL %.1f\n", gc * 100 / gt + if (xt > 0) printf "CRUX_TOTAL %.1f\n", xc * 100 / xt } ' coverage-*.out > pkg-pcts.txt TOTAL=$(awk '$1 == "TOTAL" {print $2 "%"}' pkg-pcts.txt) + CRUX_TOTAL=$(awk '$1 == "CRUX_TOTAL" {print $2 "%"}' pkg-pcts.txt) echo "Total coverage (./pkg/...): ${TOTAL:-unavailable}" + echo "Total coverage (./crux/...): ${CRUX_TOTAL:-unavailable}" # Partial selection (#12454): every number here covers the selected # packages only. Per-package floors still apply unchanged to each of @@ -1748,10 +2249,10 @@ jobs: if [ "$SELECTION" = partial ]; then PARTIAL=true for p in $SELECTED; do - case "$p" in */pkg/*|*/cmd/*) ;; *) continue ;; esac + case "$p" in */pkg/*|*/cmd/*|*/crux|*/crux/*) ;; *) continue ;; esac grep -Fxq "$p" ../existing-pkgs.txt && SELECTED_SCORED="$SELECTED_SCORED $p" done - echo "Partial selection: scoring only the $(wc -w <<<"$SELECTED_SCORED") selected ./pkg/... and ./cmd/... packages; the total above is over the selected packages and is not comparable to the repo-wide total." + echo "Partial selection: scoring only the $(wc -w <<<"$SELECTED_SCORED") selected ./pkg/..., ./cmd/... and ./crux/... packages; the totals above are over the selected packages and are not comparable to the repo-wide totals." if [ -z "$TOTAL" ] && ! grep -q '/pkg/' <<<"$SELECTED_SCORED"; then TOTAL="n/a (no ./pkg/... package selected)" fi @@ -1770,6 +2271,13 @@ jobs: echo "coverage-*.out was present but produced no ./pkg/... totals." exit 1 fi + # Same guard for the Crux module on a full run: its shards (the + # agent slices and the rest buckets holding ./crux/...) must have + # produced profiles, or the module's floors are vacuously met. + if [ "$SELECTION" != partial ] && [ -z "$CRUX_TOTAL" ]; then + echo "COVERAGE GATE FAILED: no coverage data parsed for the Crux module (./crux/...)." + exit 1 + fi # Classify every package exactly once from the combined logs. With # gotestsum reruns, a package can have an initial FAIL line followed @@ -1789,7 +2297,7 @@ jobs: } $1 == "ok" || $1 == "FAIL" || $1 == "?" { if (!exists[$2]) next - if ($2 ~ /\/(pkg|cmd)\//) { + if ($2 ~ /\/(pkg|cmd)\// || $2 ~ /\/crux(\/|$)/) { seen[$2] = 1 if ($1 == "ok") ok[$2] = 1 else if ($1 == "FAIL") fail[$2] = 1 @@ -1798,7 +2306,7 @@ jobs: next } /^\t.*\t\tcoverage: 0\.0% of statements$/ { - if (exists[$1] && $1 ~ /\/(pkg|cmd)\//) { + if (exists[$1] && ($1 ~ /\/(pkg|cmd)\// || $1 ~ /\/crux(\/|$)/)) { seen[$1] = 1 no_tests[$1] = 1 } @@ -1846,12 +2354,17 @@ jobs: # never resolve to pkg/hivectl's floor; pkg packages are keyed by # their full path below pkg/ so a nested package can never alias the # top-level one sharing its name (v5 has both pkg/spoke and - # pkg/hub/spoke). See coverage-hourly.yml for the full rationale. + # pkg/hub/spoke); the nested Crux module's packages are keyed + # "crux/" (the module root package is "crux") so crux/config + # can never alias pkg/config. See coverage-hourly.yml for the full + # rationale. floor_key() { case "$1" in - */cmd/*) echo "cmd/${1##*/}" ;; - */pkg/*) echo "${1##*/pkg/}" ;; - *) echo "${1##*/}" ;; + */cmd/*) echo "cmd/${1##*/}" ;; + */pkg/*) echo "${1##*/pkg/}" ;; + */crux/*) echo "crux/${1##*/crux/}" ;; + */crux) echo "crux" ;; + *) echo "${1##*/}" ;; esac } @@ -1893,6 +2406,7 @@ jobs: echo "## Per-package coverage" echo "" echo "Total (./pkg/...): ${TOTAL:-unavailable}" + echo "Total (./crux/...): ${CRUX_TOTAL:-unavailable}" echo "" echo -e "${REPORT}" } >> "$GITHUB_STEP_SUMMARY" @@ -1908,14 +2422,33 @@ jobs: [ -n "$FAILED" ] && echo -e "${FAILED}" exit 1 fi - echo "All packages meet their coverage floor" >> "$GITHUB_STEP_SUMMARY" + - name: Patch coverage on changed Go lines (advisory) + if: ${{ steps.cov-scope.outputs.score == 'true' && github.event_name == 'pull_request' && !cancelled() }} + continue-on-error: true + env: + BASE_REF: ${{ github.base_ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} + GH_TOKEN: ${{ github.token }} + run: | + git fetch --no-tags --depth=1 origin "$BASE_REF" || true + if [ -d shards ]; then + awk 'FNR == 1 { next } { key=$1 " " $2; hits[key]+=$3 } + END { print "mode: atomic"; for (key in hits) print key, hits[key] }' \ + shards/coverage-*.out > shards/coverage-merged.out 2>/dev/null || true + fi + bash .github/scripts/patch-coverage.sh \ + --base "origin/$BASE_REF" \ + --profile shards/coverage-merged.out \ + --min-percent 80 --min-uncovered 5 \ + --pr "$PR_NUMBER" \ + --advisory create-issue-on-failure: if: (failure() || cancelled()) && github.event_name == 'schedule' runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 10 - needs: [list-packages, test-hub, test-agent, test-rest, test-dashboard-shuffle] + needs: [list-packages, test-hub, test-sliced, test-agent, test-rest, test-dashboard-shuffle] permissions: contents: read issues: write @@ -1932,6 +2465,7 @@ jobs: const branch = context.ref.replace('refs/heads/', ''); const results = [ '${{ needs.test-hub.result }}', + '${{ needs.test-sliced.result }}', '${{ needs.test-agent.result }}', '${{ needs.test-rest.result }}', '${{ needs.test-dashboard-shuffle.result }}', @@ -1968,10 +2502,10 @@ jobs: }); create-issues-on-flakes: - if: always() && github.event_name == 'push' && (github.ref == 'refs/heads/v5' || github.ref == 'refs/heads/v6') && needs.test-hub.result == 'success' && needs.test-agent.result == 'success' && needs.test-rest.result == 'success' && needs.test-dashboard-shuffle.result == 'success' + if: always() && github.event_name == 'push' && (github.ref == 'refs/heads/v5' || github.ref == 'refs/heads/v6') && needs.test-hub.result == 'success' && needs.test-sliced.result == 'success' && needs.test-agent.result == 'success' && needs.test-rest.result == 'success' && needs.test-dashboard-shuffle.result == 'success' runs-on: ${{ fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') }} timeout-minutes: 10 - needs: [test-hub, test-agent, test-rest, test-dashboard-shuffle] + needs: [test-hub, test-sliced, test-agent, test-rest, test-dashboard-shuffle] permissions: actions: read contents: read @@ -2182,8 +2716,8 @@ jobs: - name: Compile tree into the build cache if: steps.prepare-cgo-toolchain.outputs.toolchain-ready == 'true' run: | - go build ./... - go test -race -count=1 -run '^$' ./pkg/... ./cmd/... ./internal/... > /dev/null + go build ./... ./crux/... + go test -race -count=1 -run '^$' ./pkg/... ./cmd/... ./internal/... ./crux/... > /dev/null - name: Save warm Go build cache continue-on-error: true diff --git a/.gitignore b/.gitignore index a5fcf62c4d..b23f86b2c9 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,14 @@ _inline.js /dash.js /pr-body.md -# Test-run scratch: pkg/agent tests create stub CLI binaries under a +# Agent/session scratch: PR and issue bodies drafted for --body-file, and +# section drafts. Never source; #11588 and #14197 committed them by accident. +/pr-body.* +/issue-body.* +/issue[0-9]*.txt +/section.md + +# Test-run scratch: crux/agent tests create stub CLI binaries under a # randomly-suffixed dir. Generated per run; never part of the tree. .hive-agent-stubs-*/ @@ -36,3 +43,8 @@ _inline.js # committed — see #5423. src/docs/design/_out/ .gomod/ + +# src/crux `make build` / `make smoke` outputs: the module binary and the +# smoke test's socket + work dir. Build products, never source. +src/crux/bin/ +src/crux/.smoke/ diff --git a/.oss-scanner/Dockerfile b/.oss-scanner/Dockerfile index 44e97915a2..2c2783d43e 100644 --- a/.oss-scanner/Dockerfile +++ b/.oss-scanner/Dockerfile @@ -50,10 +50,12 @@ WORKDIR /src/src # Every module the tree depends on, then compile every package and every test # binary once so nothing needs the network later. A failure here fails the -# build and primary_contact gets the log. +# build and primary_contact gets the log. src/go.work puts the go tool in +# workspace mode, where ./... covers the root module only; the nested Crux +# module (src/crux, #13536) is addressed as ./crux/... . RUN go mod download \ - && go build ./... \ - && go test -run '^$' ./... >/dev/null + && go build ./... ./crux/... \ + && go test -run '^$' ./... ./crux/... >/dev/null # The binaries an operator deploys (same targets as src/Dockerfile). RUN go build -o /usr/local/bin/hive ./cmd/hive \ @@ -64,22 +66,22 @@ RUN go build -o /usr/local/bin/hive ./cmd/hive \ && /usr/local/bin/hive --version # Cheap baseline checks so the log shows the image works, without letting a -# failing test abort the build. The full suite (pkg/hub and pkg/agent in -# particular) is sharded across many CI runners, and pkg/agent exercises real +# failing test abort the build. The full suite (pkg/hub and crux/agent in +# particular) is sharded across many CI runners, and crux/agent exercises real # tmux servers and signal delivery, so the baseline here is the set of # security-relevant trust boundaries named in threat_model.md. Everything # else is compiled above and can be run on demand inside the image. # Known baseline noise: a handful of pkg/appkey and pkg/config tests chmod a # directory read-only and expect the write to fail; the build runs as root, # which ignores permission bits, so those report FAIL here and pass in CI. -RUN go vet ./... || echo "WARNING: go vet reported problems" +RUN go vet ./... ./crux/... || echo "WARNING: go vet reported problems" RUN timeout 1500 go test -short -p 4 -timeout 20m \ - ./pkg/proxy/... ./pkg/github/... ./pkg/sentinel/... ./pkg/ioscan/... \ - ./pkg/logscrub/... ./pkg/pushbroker/... ./pkg/sandbox/... ./pkg/appkey/... \ + ./crux/proxy/... ./pkg/github/... ./pkg/sentinel/... ./pkg/ioscan/... \ + ./crux/logscrub/... ./crux/pushbroker/... ./crux/sandbox/... ./pkg/appkey/... \ ./pkg/keyderive/... ./pkg/delegation/... ./pkg/terminalassert/... \ ./pkg/spoke/... ./pkg/mint/... ./pkg/apiproxy/... ./pkg/auth/... \ ./pkg/config/... ./pkg/evidence/... ./pkg/knowledge/... ./pkg/scheduler/... \ - ./pkg/review/... ./pkg/promptsrc/... ./pkg/claims/... ./pkg/toolapprove/... \ + ./pkg/review/... ./pkg/promptsrc/... ./pkg/claims/... ./crux/toolapprove/... \ || echo "WARNING: baseline go test reported failures" # Everything is fetched. From here on the go command must not try to reach a diff --git a/.oss-scanner/threat_model.md b/.oss-scanner/threat_model.md index 4c3431e71c..4b37d78a34 100644 --- a/.oss-scanner/threat_model.md +++ b/.oss-scanner/threat_model.md @@ -7,7 +7,7 @@ process (`src/cmd/hive`, package `github.com/hivecommons/hive` rooted at `src/`) attached to a set of GitHub repositories. It enumerates the open work in those repositories, decides which AI coding agents to wake (`pkg/governor`, `pkg/scheduler`), launches each agent CLI (Claude Code, Copilot, Codex, -Goose, …) inside a per-agent tmux server (`pkg/agent/manager_tmux.go`), feeds +Goose, …) inside a per-agent tmux server (`crux/agent/manager_tmux.go`), feeds it a prompt built from the issues and pull requests it should work on, and then performs the resulting GitHub writes — branches, pull requests, issues, comments, reviews, labels, merges — with a **GitHub App installation token** @@ -25,7 +25,9 @@ The scanned branch is `v5`. The authoritative long-form documents are `src/docs/security-threat-model.md`, `src/docs/acmm-policy-matrix.md`, `src/docs/github-write-surface.md`, `src/docs/hosted-hub.md`, `src/docs/review-swarm.md` and `src/docs/agent-configuration.md`; the package -layout under `src/pkg/` is the ground truth when they disagree. +layout under `src/pkg/` (Hive) and `src/crux/` (the agent runtime) is the +ground truth when they disagree. Package paths below are relative to `src/` +unless they explicitly start with `src/`. Treat everything below as attacker-controlled: @@ -45,9 +47,9 @@ Treat everything below as attacker-controlled: local principal. It runs under its own UID (base 2001, launched via `su-exec`), inside a tmux server on a per-agent socket, with `HTTPS_PROXY` pointed at the hive's GitHub policy proxy - (`pkg/proxy/github_proxy.go`, rules in `pkg/proxy/rules.go`, + (`crux/proxy/github_proxy.go`, rules in `crux/proxy/rules.go`, `linear_rules.go`, `autonomy_levels.go`, `write_surface_enforce.go`, - UID attribution from `/proc/net/tcp` in `pkg/proxy/procnet.go`). Its only + UID attribution from `/proc/net/tcp` in `crux/proxy/procnet.go`). Its only sanctioned write path above its tier is a JSON request file dropped into `/var/run/hive-metrics/{pr,issue,review,merge,push}-requests`, consumed by the relay watchers in `pkg/github` (`pr_request_watcher.go`, @@ -56,9 +58,9 @@ Treat everything below as attacker-controlled: authorises in the order documented in `src/docs/github-write-surface.md`: file-UID forge check and ACMM mode gate, lane allowlist, repo pause and scope, the relay's own content gates, then the GitHub call. Optionally the - agent runs credential-free in a rootless Podman sandbox (`pkg/sandbox`) - with pushes performed by the trusted post-step in `pkg/pushbroker`. -- **The GitHub API responses** the proxy and the hive parse (`pkg/proxy`, + agent runs credential-free in a rootless Podman sandbox (`crux/sandbox`) + with pushes performed by the trusted post-step in `crux/pushbroker`. +- **The GitHub API responses** the proxy and the hive parse (`crux/proxy`, `pkg/github`), including GraphQL error bodies and webhook deliveries (`pkg/hub/webhook.go`, verified with `X-Hub-Signature-256`). - **Spoke → hub traffic.** `POST /api/heartbeat`, `/api/task-status`, the @@ -90,10 +92,10 @@ Treat everything below as attacker-controlled: GitHub App setup, release-channel selection and self-upgrade (`api_version.go` `handleSelfUpgrade`, `deployment_upgrade.go`). - **Model-gateway traffic.** Agents talk to inference backends through the - in-pod translator (`pkg/proxy/inference_*.go`, `anthropic_translate.go`, + in-pod translator (`crux/proxy/inference_*.go`, `anthropic_translate.go`, `inference_auth.go`) using a per-agent placeholder key `sk-hive-`; the real gateway/OpenRouter/watsonx key is swapped in - server-side (`pkg/openrouter`, `pkg/watsonx`, `pkg/inference`). + server-side (`crux/openrouter`, `crux/watsonx`, `crux/inference`). `cmd/apiproxy` is the standalone variant gated by `PROXY_AUTH_TOKEN`. - **Configuration and secrets on disk.** `hive.yaml` (`pkg/config`), the dashboard-persisted overlay, the App private key files (`pkg/appkey`), the @@ -111,11 +113,11 @@ Treat everything below as attacker-controlled: Most important, in this order: -1. **The agent → GitHub boundary.** `pkg/proxy` (`rules.go`, +1. **The agent → GitHub boundary.** `crux/proxy` (`rules.go`, `github_proxy.go`, `write_surface_enforce.go`, `autonomy_levels.go`, `procnet.go`, `agent_github_budget.go`), the relay watchers and `SweepSentinel` in `pkg/github`, `pkg/sentinel` (`Evaluate`), - `pkg/pushbroker`, `pkg/sandbox`, `pkg/agent` (`mode.go`, token + `crux/pushbroker`, `crux/sandbox`, `crux/agent` (`mode.go`, token delivery, tmux launch, per-UID isolation), `pkg/mint` and `pkg/github/app.go` (installation-token minting and tier scoping), `pkg/appkey`. Any way an agent — or GitHub content acting through an @@ -149,25 +151,25 @@ Most important, in this order: Every `exec.Command` of `git`, `gh`, `tmux`, `podman`, `su-exec` or a shell whose arguments include a branch name, repo name, issue title, label, login or file path that came from GitHub or from an agent - (`pkg/agent`, `pkg/pushbroker`, `pkg/sandbox`, `pkg/gitidentity`, + (`crux/agent`, `crux/pushbroker`, `crux/sandbox`, `crux/gitidentity`, `pkg/github`, `pkg/knowledge`, `cmd/hive`). `tmux send-keys` to an agent pane with attacker text, `git` option injection (`-c`, `--upload-pack`, refspecs starting with `-`), or a path that escapes the intended directory. 5. **The prompt-injection defences and secret scrubbing.** `pkg/ioscan`, - `pkg/scheduler/ioscan_enforce.go`, `pkg/logscrub`, `pkg/promptsrc` - size caps, `pkg/toolapprove`, `pkg/claims`. A bypass of a deterministic + `pkg/scheduler/ioscan_enforce.go`, `crux/logscrub`, `pkg/promptsrc` + size caps, `crux/toolapprove`, `pkg/claims`. A bypass of a deterministic rule the docs say holds (zero-width stripping, base64 rescans, canary detection, token redaction in logs/PR bodies) is a real finding; a model that follows a plain-English instruction the classifier did not flag is not (see "leave alone"). 6. **File stores keyed by untrusted names.** `pkg/knowledge`, - `pkg/evidence`, `pkg/beads`, `pkg/trajectory`, `pkg/timeline`, + `pkg/evidence`, `pkg/beads`, `crux/trajectory`, `pkg/timeline`, `pkg/snapshot`, `pkg/hubbackup`/`pkg/spokebackup`, the relay request directories. Path traversal from a repo/agent/issue-derived component, symlink following into `/data/secrets` or the App key, world-readable token files. -7. **`cmd/apiproxy`, `pkg/apiproxy`, `pkg/proxy/inference_*`.** Leaking the +7. **`cmd/apiproxy`, `pkg/apiproxy`, `crux/proxy/inference_*`.** Leaking the upstream key to an unauthenticated caller, or letting one agent's placeholder spend as another's. @@ -194,26 +196,30 @@ The image has only loopback networking. `GOPROXY=off`, `GOSUMDB=off` and `GOTOOLCHAIN=local` are set, every module is in the module cache, every package and test binary has been compiled, and the deployable binaries are in `/usr/local/bin` (`hive`, `hivectl`, `bd`, `hive-backup`, `apiproxy`). -The Go module is `/src/src`; run `go` commands from there. +The Go workspace is `/src/src`; run `go` commands from there. It contains +the Hive module and the nested Crux module in `crux/`; `./...` alone covers +only Hive, so use `./... ./crux/...` when targeting both modules. -- **Tests.** `go test -short ./pkg//...` for any package. The image +- **Tests.** `go test -short ./pkg//...` for a Hive package or + `go test -short ./crux//...` for a Crux package. The image runs as root, so the few tests in `pkg/appkey` and `pkg/config` that chmod a directory read-only and expect a write to fail report FAIL here; they pass in CI and are not findings. `pkg/hub` - and `pkg/agent` are large and are sharded across many runners in CI; use - `-run` to scope them. `pkg/agent` and parts of `pkg/dashboard` spawn real + and `crux/agent` are large and are sharded across many runners in CI; use + `-run` to scope them. `crux/agent` and parts of `pkg/dashboard` spawn real tmux servers and send signals by UID — this is fine inside the container but do not run them on a host that is also running a hive. Table-driven tests next to the code under test are the normal place for a regression - test; `src/internal/testutil` has shared helpers and `pkg/proxy/*_test.go` + test; `src/internal/testutil` has Hive helpers, + `src/crux/internal/testutil` has Crux helpers, and `crux/proxy/*_test.go` shows how to stand up the GitHub policy proxy against an `httptest` upstream. -- **The policy proxy.** `pkg/proxy/rules.go` is a readable table of +- **The policy proxy.** `crux/proxy/rules.go` is a readable table of `(method, path regexp) → minimum agent mode`; `agent_repo_scope_test.go`, `autonomy_enforcement_test.go`, `auth_inject_test.go` and `write_surface_enforce.go` tests show how to drive a request through it as an agent of a given mode and assert `403` with `X-Hive-Proxy-Blocked`. - `pkg/proxy/denial_message.go` is the operator-facing denial text. + `crux/proxy/denial_message.go` is the operator-facing denial text. - **The relays.** `pkg/github/*_request_watcher.go` with their tests show the JSON request shape and the authorisation order; `pkg/sentinel` `Evaluate` is a pure function over a PR's author, title and unified diff. @@ -264,7 +270,7 @@ weighs more than one that needs an opt-in switched on. following in the knowledge/evidence/beads/relay stores that reads or writes outside the intended directory. Disclosure of secrets through logs, config export, snapshots, PR/issue bodies or heartbeats that - `pkg/logscrub`/the secret-free overlay are meant to prevent. CSRF on an + `crux/logscrub`/the secret-free overlay are meant to prevent. CSRF on an owner-only dashboard route. A deterministic `pkg/ioscan` rule (unicode normalisation, base64 rescan, canary, token patterns) that a crafted payload bypasses while the docs state it holds. A heartbeat or webhook @@ -302,8 +308,8 @@ weighs more than one that needs an opt-in switched on. code under test, or a `curl`/shell sequence against `hive` or `apiproxy` on loopback, with the config file used and the exact request. For a proxy/relay finding, name the agent mode (`ADVISORY`, `ISSUES_ONLY`, - `ISSUES_AND_PRS`, `ISSUES_PRS_MERGE` — `pkg/agentmode`, re-exported by - `pkg/agent/mode.go`), the ACMM level and + `ISSUES_AND_PRS`, `ISSUES_PRS_MERGE` — `crux/agentmode`, re-exported by + `crux/agent/mode.go`), the ACMM level and the repo scope in force, and show the request that should have been refused and the write that landed. For a hub finding, show the two tenants and the session/bearer each one holds. @@ -332,7 +338,7 @@ weighs more than one that needs an opt-in switched on. residual risk (hivecommons/hive#2804). Report concrete escapes, not the architecture. - The `pkg/ioscan` semantic classifier and trajectory review - (`pkg/trajectory`) fail open on model outage by design. + (`crux/trajectory`) fail open on model outage by design. - Agents keep read access to the GitHub repos the App is installed on and outbound access to their model backend; "an agent can read the repo" or "can reach api.anthropic.com" is not a finding. diff --git a/AGENTS.md b/AGENTS.md index 0baa6d65f2..56a83fb627 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,8 @@ These rules apply to every AI agent (hive agents, Copilot CLI, Claude Code, Code - **Do not run `go test`, `go build ./...`, `go vet`, `golangci-lint`, or the docs/citation guards locally.** Commit, push, open the PR, and read the CI results (`gh pr checks `, then the failing job log). CI runs the full matrix with the right toolchain, caches and coverage gates; a local run duplicates it and burns tokens and pod CPU/disk. Inside the hive pod, `go test` and `go vet` are blocked for agents by the Go shim; use CI for the verdict. - If CI fails, fix from the job log and push again. Do not "reproduce locally first". The hive opens requested PRs; CI is the sole verdict and records failures for agents to iterate on. - The only local check worth running is `gofmt -l` on files you changed. -- **Never run this repository's test suite from inside an agent pane.** Parts of `pkg/agent` and `pkg/dashboard` exercise real process management (tmux, `/proc` sweeps, SIGKILL by uid). Run as an agent user they can kill the agent's own tmux server, CLI and terminal — this is how the scanner lost its session repeatedly (hivecommons/hive#9416). +- `src/` is a two-module Go workspace (`src/go.work`): the root module and the Crux agent runtime at `src/crux/` (`github.com/hivecommons/hive/crux`, packages `crux`, `crux/agent`, `crux/inference`, …). From `src/`, `./...` is the root module only; the Crux module is `./crux/...`. CI covers both; see [src/docs/design/crux.md](src/docs/design/crux.md). +- **Never run this repository's test suite from inside an agent pane.** Parts of `crux/agent` and `pkg/dashboard` exercise real process management (tmux, `/proc` sweeps, SIGKILL by uid). Run as an agent user they can kill the agent's own tmux server, CLI and terminal — this is how the scanner lost its session repeatedly (hivecommons/hive#9416). ## Working style diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cab578d967..e77e348617 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,7 +20,8 @@ Thank you for helping improve Hive. This guide is for contributing code and docu - `src/` — the current Go module (`github.com/hivecommons/hive`) and the main development target for this repository. - `src/cmd/hive` — main Hive binary. - `src/cmd/hivectl`, `src/cmd/apiproxy`, `src/cmd/hive-backup` — supporting command-line tools. - - `src/pkg/` — Go packages for agents, GitHub integration, scheduling, policies, dashboards, hubs, backups, and related runtime behavior. + - `src/pkg/` — Go packages for GitHub integration, scheduling, policies, dashboards, hubs, backups, and related runtime behavior. + - `src/crux/` — the Crux agent runtime (agent manager, inference, sandbox, egress proxy, tool approval) as a nested Go module; see [`src/docs/design/crux.md`](src/docs/design/crux.md). - `src/policies/` — policy prompts and rule files used by the deterministic/agent pipeline. Treat policy changes like code: review the behavior they enable, test where possible, and explain risk in the PR. - `src/deploy/` and `src/examples/` — deployment manifests and example configuration. - `src/docs/` — architecture and operator/developer reference material. @@ -60,11 +61,11 @@ See [docs/development.md](docs/development.md) for the full local setup guide. T ```bash cd src -go build ./... -go test ./... +go build ./... ./crux/... +go test ./... ./crux/... ``` -The Go version is declared in [`src/go.mod`](src/go.mod). Install that version or newer compatible tooling before building. +`src/` is a two-module Go workspace (`src/go.work`): the root module plus the Crux agent runtime at [`src/crux/`](src/crux/) (`github.com/hivecommons/hive/crux`). `./...` covers the root module only, so the Crux module is named explicitly as `./crux/...`. The Go version is declared in [`src/go.mod`](src/go.mod) (and mirrored in `src/crux/go.mod`). Install that version or newer compatible tooling before building. ## Contributor `just` recipes diff --git a/Justfile b/Justfile index b94dded272..631c869c6e 100644 --- a/Justfile +++ b/Justfile @@ -136,8 +136,8 @@ contribute-check-backend backend="claude": bob) if command -v bob &>/dev/null; then # Bob's browser SSO flow cannot complete in a container, so the - # containerized path needs BOBSHELL_API_KEY exported in your shell. - echo "Bob CLI detected — export BOBSHELL_API_KEY for containerized runs." + # containerized path needs BOB_API_KEY (or BOBSHELL_API_KEY) exported in your shell. + echo "Bob CLI detected — export BOB_API_KEY for containerized runs." else echo "ERROR: Bob CLI not found." exit 1 @@ -318,6 +318,24 @@ contribute-check-backend backend="claude": exit 1 fi ;; + omnigent) + if command -v omnigent &>/dev/null; then + echo "Omnigent CLI detected ($(omnigent --version 2>&1 | head -1))" + echo " Meta-harness for Claude Code, Codex, Cursor, Pi, and custom agents." + echo " Auth: export ANTHROPIC_API_KEY=..., OPENAI_API_KEY=..., or GEMINI_API_KEY=..." + echo " (persisted ~/.omnigent/auth_tokens.json / config.yaml credentials are staged too)" + echo " Hive launches 'omnigent run --server local': a remote server default from" + echo " 'omnigent login ' is NOT honoured — remote-server mode is unsupported." + echo " omnigent has NO OS sandbox on the local CLI path; local mode refuses to launch without" + echo " HIVE_OMNIGENT_DANGEROUSLY_RUN_UNCONFINED=1." + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${OPENAI_API_KEY:-}" && -z "${GEMINI_API_KEY:-}" && -z "${DATABRICKS_TOKEN:-}" && -z "${OMNIGENT_API_KEY:-}" && ! -s "$HOME/.omnigent/auth_tokens.json" && ! -s "$HOME/.omnigent/config.yaml" && ! -s "$HOME/.databrickscfg" ]]; then + echo " WARNING: no provider API key and no ~/.omnigent credentials found — omnigent may fail at task time." + fi + else + echo "ERROR: omnigent CLI not found. Install: https://github.com/omnigent-ai/omnigent" + exit 1 + fi + ;; *) echo "ERROR: Unknown backend '{{backend}}'. Supported: ${KNOWN_BACKENDS// /, }" exit 1 @@ -1034,9 +1052,9 @@ contribute-hive backend="" mode="docker": check-version fi # Bob has no headless credential other than its API key — without it the # agent would launch and sit at Bob's key prompt forever. Fail fast. - if [[ "$BACKEND" == "bob" && -z "${BOBSHELL_API_KEY:-}" ]]; then - echo "ERROR: BOBSHELL_API_KEY not set — Bob cannot authenticate." - echo " export BOBSHELL_API_KEY=your-bob-api-key" + if [[ "$BACKEND" == "bob" && -z "${BOB_API_KEY:-}" && -z "${BOBSHELL_API_KEY:-}" ]]; then + echo "ERROR: BOB_API_KEY (or legacy BOBSHELL_API_KEY) not set — Bob cannot authenticate." + echo " export BOB_API_KEY=your-bob-api-key" echo " then re-run: just contribute-hive bob" exit 1 fi @@ -1280,9 +1298,9 @@ contribute-hive backend="" mode="docker": check-version codex) PERM_FLAG=$(backend_perm_flag_shell "$BACKEND" 2>/dev/null || echo "") ;; - goose|agy|bob|pi|aider|kilo|omp|openhands) + goose|agy|bob|pi|aider|kilo|omp|openhands|omnigent) # No sandbox, filesystem allowlist, or command deny-list exists for - # any of these eight (see the "no confinement mechanism at all" + # any of these nine (see the "no confinement mechanism at all" # block in backends.conf) — refuse to launch unconfined by # default rather than silently grant full host access (#4918). if ! PERM_FLAG=$(unconfined_local_perm_flag_shell "$BACKEND"); then @@ -2032,6 +2050,36 @@ contribute-hive backend="" mode="docker": check-version CLI_MOUNTS="-v ${CLI_STAGE}/.openhands:/home/dev/.openhands${VOLSUF}" fi ;; + omnigent) + # Stage ONLY Omnigent's credential files, never the directory. + # ~/.omnigent also holds omnigent.db (every prior session and turn), + # logs/ (full JSON conversation dumps) and debug/ (SSE event tapes) + # — the same transcript-exposure problem #7836 closed for ~/.claude. + # auth_tokens.json — persisted server session tokens + # config.yaml — provider/LLM configuration. Its `server:` + # default is NOT honoured: the launch line pins + # `--server local` (backends.conf) so a remote + # server from `omnigent login ` can never + # take the session out of this container. + mkdir -p "${CLI_STAGE}/.omnigent" + if [ -d "${HOME}/.omnigent" ]; then + staged="" + for f in auth_tokens.json config.yaml; do + if [ -f "${HOME}/.omnigent/${f}" ] && cp -p "${HOME}/.omnigent/${f}" "${CLI_STAGE}/.omnigent/${f}" 2>/dev/null; then + staged="${staged:+${staged} }${f}" + fi + done + echo "Staged: ${staged:-nothing} from ${HOME}/.omnigent (allowlist: auth_tokens.json config.yaml; omnigent.db, logs/ and debug/ stay on the host)" + fi + CLI_MOUNTS="-v ${CLI_STAGE}/.omnigent:/home/dev/.omnigent${VOLSUF}" + # ~/.databrickscfg is the one credential file outside ~/.omnigent that + # detect_cli (bin/contributor-agent.sh) accepts; stage that single + # file so the container's verdict matches the host preflight. + if [ -s "${HOME}/.databrickscfg" ] && cp -p "${HOME}/.databrickscfg" "${CLI_STAGE}/.databrickscfg" 2>/dev/null; then + echo "Staged: .databrickscfg from ${HOME}" + CLI_MOUNTS="${CLI_MOUNTS} -v ${CLI_STAGE}/.databrickscfg:/home/dev/.databrickscfg${VOLSUF}" + fi + ;; esac CONTAINER_NAME="hive-contributor-${BACKEND}-$(head -c 4 /dev/urandom | od -An -tx1 | tr -d ' ')" # Pi receives ONLY the selected provider's official credential variables. @@ -2050,7 +2098,7 @@ contribute-hive backend="" mode="docker": check-version if [[ -n "$name" ]]; then add_provider_env "$name"; fi done < <(node bin/pi-backend.js --env-names "${AGENT_MODEL}") else - for name in ANTHROPIC_API_KEY OPENROUTER_API_KEY OPENAI_API_KEY OPENAI_HOST OPENAI_BASE_PATH GOOGLE_API_KEY GOOSE_API_KEY GOOSE_PROVIDER GOOSE_MODEL KIRO_API_KEY BOBSHELL_API_KEY HIVE_LITELLM_ENDPOINT HIVE_LITELLM_API_KEY KILO_AUTH_CONTENT KILO_CONFIG_CONTENT KILO_API_KEY KILO_ORG_ID META_API_KEY LLM_API_KEY LLM_BASE_URL LLM_MODEL; do + for name in ANTHROPIC_API_KEY OPENROUTER_API_KEY OPENAI_API_KEY OPENAI_HOST OPENAI_BASE_PATH GOOGLE_API_KEY GEMINI_API_KEY DATABRICKS_TOKEN OMNIGENT_API_KEY GOOSE_API_KEY GOOSE_PROVIDER GOOSE_MODEL KIRO_API_KEY BOB_API_KEY BOBSHELL_API_KEY HIVE_LITELLM_ENDPOINT HIVE_LITELLM_API_KEY KILO_AUTH_CONTENT KILO_CONFIG_CONTENT KILO_API_KEY KILO_ORG_ID META_API_KEY LLM_API_KEY LLM_BASE_URL LLM_MODEL; do add_provider_env "$name" done fi diff --git a/README.md b/README.md index 444ab45c51..d796a31864 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,26 @@ A terminal agent is a power tool: you hold it, aim it, and it stops when you sto Around that, Hive picks work from your real queue, sets limits in code with six trust levels from the **AI Codebase Maturity Model (ACMM)** — a framework for how much autonomy a codebase is ready to give agents ([levels](#acmm-levels) · [paper](https://arxiv.org/abs/2604.09388)), checks before and after with deterministic filters and gates, writes every action to an audit log owned by a named human, and runs around the clock. Same agents you use today — governed, scheduled, scoped, and accountable. +## Outcomes + +Measured from the review evidence Hive already records, not from a demo. Every figure carries its n and a 95% Wilson interval; anything with n < 10 is not reported. Regenerate with `src/scripts/outcome-benchmark.sh --readme`; a test fails if this block drifts from the committed report. + + + +- **34% of reviewed PRs were flagged for changes before they merged** — a `changes_requested` verdict on an earlier head of 490 of 1455 mediated merges [31.3–36.1], 2026-10-27. +- **Revert rate 0.0% mediated vs 0.1% unmediated** — 0/43 vs 1/1513 merges matured 14 days, Wilson [0.0–8.2] vs [0.0–0.4], 2026-10-27. +- **Median time-to-merge 27 min mediated vs 12 min unmediated** — review costs about 15 minutes on the median PR (n=1455 vs 2224), 2026-10-27. +- **≈$2.13 estimated agent spend per merged PR** — `/api/cost` estimate over 2869 hive-attributed merges, 2026-08-08 → 2026-10-27. +- **11571 pull requests across 10 repositories, 15054 review verdicts recorded** — the evidence this is measured from, 2026-10-27. + +How measured: per-PR evidence bundles the spoke already writes, folded per PR; 95% Wilson intervals; n < 10 is never reported. Full tables, trend and methodology in [src/docs/benchmarks/outcomes-2026-10.md](src/docs/benchmarks/outcomes-2026-10.md). + +![mediated vs unmediated](src/docs/benchmarks/assets/outcomes-2026-10-bars.svg) +![trend by month](src/docs/benchmarks/assets/outcomes-2026-10-trend.svg) + + +**One real example.** On [hivecommons/hive#13523](https://github.com/hivecommons/hive/pull/13523) the swarm's [posted review](https://github.com/hivecommons/hive/pull/13523#pullrequestreview-5613922737) flagged a changelog fragment that started with prose (the release compile would have rejected it) and a duplicate fragment; the author fixed both four minutes later and the next head was approved. Methodology and the full tables: [src/docs/benchmarks/](src/docs/benchmarks/README.md). + ## Hub and spoke - **Spoke** — one running Hive, serving your repo(s). It is where agents actually work. Runs on your Kubernetes cluster, Podman, or Docker Compose. @@ -81,7 +101,7 @@ Hive uses an **AI Codebase Maturity Model** (ACMM) with six levels that control | L3 | Quality-Gated (Measured) | Quality agent opens issues and PRs gated by the literal `hold` level-gate label. Others remain advisory. | | L4 | Security-Aware (Adaptive) | All agents file issues. Quality, sec-check, and CI open PRs gated by `hold`. | | L5 | Semi-Autonomous (Semi-Automated) | All agents open PRs gated by `hold`. Humans batch-review and approve. | -| L6 | Fully Autonomous | Agents open PRs and auto-merge when checks pass. Non-outreach PRs have no level hold; outreach PRs remain held. | +| L6 | Autonomous | Agents open PRs and auto-merge when checks pass. Non-outreach PRs have no level hold; outreach PRs remain held. | See [src/docs/acmm-policy-matrix.md](src/docs/acmm-policy-matrix.md) for the full matrix and [src/docs/labels-and-control-signals.md](src/docs/labels-and-control-signals.md) for label behavior. @@ -132,6 +152,10 @@ Moved to [src/docs/quickstart-podman.md](src/docs/quickstart-podman.md). Moved to [src/docs/kubernetes-deployment.md](src/docs/kubernetes-deployment.md). +## Works with + +Hive drives Claude Code, GitHub Copilot CLI, Codex, Goose, Antigravity and OpenAI-compatible gateways; reads work from GitHub Issues, GitHub Projects, Linear and Jira; and runs on GitHub today with GitLab and Gitea/Forgejo planned. See the [integrations page](src/docs/integrations.md) for the full logo grid with status. + ## Related projects - [Goose](https://github.com/aaif-goose/goose), an agent backend whose upstream moved to the [Agentic AI Foundation (AAIF)](https://aaif.io/). Hive integrates Goose as an independent CNCF project; Hive does not join AAIF, seek AAIF membership, or rely on AAIF to host or govern Hive. See [unattended Goose with Hive](docs/goose-at-scale.md) and [Goose upstream tracking](src/docs/landscape.md#agentic-ai-foundation-aaif-and-goose). diff --git a/UPGRADE.md b/UPGRADE.md index eb23beaf07..fc6d861088 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -66,8 +66,13 @@ a release note for the exact target version says to do so. For Kubernetes, preserve the startup budget and capabilities from `src/deploy/k8s/deployment.yaml` when carrying local overlays forward: -- `startupProbe` currently allows 150 seconds (`failureThreshold: 30`, - `periodSeconds: 5`) before liveness restarts are armed. +- `startupProbe` ships with `failureThreshold: 30`, `periodSeconds: 5` in the + manifest (150 seconds of probes, plus the 10-second initial delay). On an + image switch or rollout restart, spoke self-upgrades raise an existing `hive` + container's threshold to at least 118, giving the hosted 10-minute startup + budget with those timings. Set 118 (or higher) in local overlays so reapplying + an overlay preserves that threshold. Larger thresholds, probe handlers and + timings are preserved; containers without a startup probe are unchanged. - The pod must retain the capabilities the entrypoint and per-agent UID switch use (`CHOWN`, `SETUID`, `SETGID`, `SETPCAP`, `DAC_OVERRIDE`, `FOWNER`, `FSETID`, plus `NET_ADMIN` unless proxy egress is explicitly advisory). diff --git a/bin/README.md b/bin/README.md index b1ac88c991..701ebc61c1 100644 --- a/bin/README.md +++ b/bin/README.md @@ -82,7 +82,7 @@ Most production scripts are installed under `/usr/local/bin` by `bin/hive-deploy | `contributor-agent.sh` | Contributor runtime | Contributor-container entrypoint: detects authenticated CLI backend, starts the relay, launches the CLI in tmux, and creates `${HOME}/agent.md` only from a verified live knowledge export. | | `contributor-relay.js` | Contributor runtime | Node.js WebSocket client for ClankeR contributor agents. It authenticates to one or more hubs, receives tasks, injects GitHub tokens, reports progress/results, supports The Commons multi-hive routing strategies (`ranked`, `spread`, `neediest`) between tasks, and supports interactive tmux or headless one-shot delivery. Pure pane-classification (readiness/login/onboarding, the busy/idle/blocked/error state machine) lives in `lib/pane-classifier.js`, which this file `require`s. | | `contributor-relay.sh` | Contributor runtime | Thin POSIX-sh compat wrapper that `exec`s `contributor-relay.js` with the same argv (kubestellar/hive#6429) — kept because the `.sh` path is baked into existing deployments. New callers should invoke `contributor-relay.js` directly. | -| `lib/pane-classifier.js` | Contributor runtime | Pure (no `process`/tmux) pane-classification library required by `contributor-relay.js`: `classifyReadiness`, `classifyPane`, `blockingPromptKey`, `classifyBlockedOnHumanReason`, and the pane-tail/API-error-line detectors they share. Kept test-for-test in sync with `src/pkg/agent/manager.go`'s Go port via the shared fixtures in `testdata/pane-fixtures/` (kubestellar/hive#6427). | +| `lib/pane-classifier.js` | Contributor runtime | Pure (no `process`/tmux) pane-classification library required by `contributor-relay.js`: `classifyReadiness`, `classifyPane`, `blockingPromptKey`, `classifyBlockedOnHumanReason`, and the pane-tail/API-error-line detectors they share. Kept test-for-test in sync with `src/crux/agent/manager.go`'s Go port via the shared fixtures in `testdata/pane-fixtures/` (kubestellar/hive#6427). | | `hivectl-bootstrap.sh` | Contributor runtime | Resolves `hivectl` for contributor Justfile recipes (`just contribute-tui`, `just contribute-hives`) and `exec`s it with the caller's argv (#8200). Resolution order: `HIVECTL` override, `./bin/hivectl`, user/system install, `PATH`; when none is found it stages a checkout-local binary from the Hive image (`HIVECTL_BOOTSTRAP_IMAGE`, default `ghcr.io/hivecommons/hive:stable`) via a throwaway Podman container carrying the #4210 ownership labels, and records the source digest in `bin/.hivectl.source` so later runs refresh or refuse instead of using a stale binary. No Go toolchain required. | | `contributor-quota-control.js` | Contributor runtime | Out-of-band contributor quota override command (#6953). The relay usually runs detached (background, container, K8s pod), so #6833's scoped overrides cannot depend on its stdin; this writes `continue-once` / `continue-until-reset` / `disable-session` / `pause-until-reset` / `resume` overrides into the same per-pool directory (`lib/quota-pool-store.js`) the relay reads on every guard evaluation. Stores only the relay's own published hold state — never a provider credential, raw account identifier, or billing figure. | | `lib/quota-pool-store.js` | Contributor runtime | Shared per-pool quota-override store used by both `contributor-relay.js` (guard evaluation) and `contributor-quota-control.js` (override writes), so the attached banner and the out-of-band command share exactly one mechanism with no second code path to drift (#6953). | diff --git a/bin/agent-launch.sh b/bin/agent-launch.sh index 496e6d27be..ba6671ed66 100755 --- a/bin/agent-launch.sh +++ b/bin/agent-launch.sh @@ -49,7 +49,7 @@ fi # N14 (#3842) follow-up: a native/systemd install (kick-agents.sh, no Go # AgentManager) never sets HIVE_AGENT_TOKEN_CACHE — that per-agent scoped -# cache is minted only by the container-hosted model (src/pkg/agent/manager.go). +# cache is minted only by the container-hosted model (src/crux/agent/manager.go). # Without it, neither of this codebase's two SAFE gh-auth mechanisms apply: # gh-wrapper.sh isn't installed as `gh` for native installs (install.sh ships # it under its own name), and git-credential-hive.sh refuses without a diff --git a/bin/contributor-agent.sh b/bin/contributor-agent.sh index d0d30a9ceb..c14622245c 100755 --- a/bin/contributor-agent.sh +++ b/bin/contributor-agent.sh @@ -313,6 +313,21 @@ detect_cli() { echo "NOT_AUTHED" fi ;; + omnigent) + # `omnigent --version` is a Click version action: it exits 0 with + # no credential at all, so the binary alone proves nothing. Probe the + # credential Omnigent documents: ambient provider keys (ANTHROPIC_API_KEY, + # OPENAI_API_KEY, GEMINI_API_KEY), Databricks auth (~/.databrickscfg, + # DATABRICKS_TOKEN), or persisted session tokens (~/.omnigent/auth_tokens.json + # or ~/.omnigent/config.yaml). + if ! omnigent --version &>/dev/null; then + echo "NOT_AUTHED" + elif [[ -n "${ANTHROPIC_API_KEY:-}" || -n "${OPENAI_API_KEY:-}" || -n "${GEMINI_API_KEY:-}" || -n "${DATABRICKS_TOKEN:-}" || -n "${OMNIGENT_API_KEY:-}" || -s "${HOME}/.omnigent/auth_tokens.json" || -s "${HOME}/.omnigent/config.yaml" || -s "${HOME}/.databrickscfg" ]]; then + echo "OK" + else + echo "NOT_AUTHED" + fi + ;; *) echo "UNKNOWN" ;; @@ -327,7 +342,7 @@ fi # seed_claude_config pre-answers the first-run gates Claude Code raises in # ${HOME}/.claude.json, so none of them can park the tmux pane on a menu that # nothing inside a container is there to answer. Mirrors the hub's -# inferenceUserConfigSeed (src/pkg/agent/manager.go) and the same seed +# inferenceUserConfigSeed (src/crux/agent/manager.go) and the same seed # bin/test_backend_smoke.sh writes for its throwaway HOME. # # hasCompletedOnboarding matters for EVERY claude contributor, not just the @@ -339,7 +354,7 @@ fi # oauthAccount but NOT hasCompletedOnboarding. The CLI then re-runs onboarding # and draws "Select login method" on top of working credentials. That two-file # split is documented for hub-side agents in -# src/pkg/agent/claude_session_state.go (#4596); the contributor relay walks +# src/crux/agent/claude_session_state.go (#4596); the contributor relay walks # into the same wall from the other direction. # # This seeding used to run only for litellm, or for claude when an @@ -471,17 +486,12 @@ case "$STATUS" in ;; esac -# bob is launched with --auth-method api-key (see backend_perm_flag in -# config/backends.conf), which makes BOBSHELL_API_KEY mandatory: without it bob -# comes up sitting at "Enter Bob-Shell API Key" and simply waits. Nobody is -# attached to an unattended relay, so that is an indefinite silent hang rather -# than a failure anyone can see. Fail loudly at startup instead. -# Only the presence of the key is tested — never its value, and it is never -# echoed. -if [[ "$AGENT_BACKEND" == "bob" ]] && [[ -z "${BOBSHELL_API_KEY:-}" ]]; then - echo "ERROR: backend 'bob' requires BOBSHELL_API_KEY to be set." +# Fail before an unattended Bob waits indefinitely for a key. Export both +# names for Bob 1.x/2.x compatibility without printing either value. +if [[ "$AGENT_BACKEND" == "bob" ]] && ! backend_bob_prepare_env; then + echo "ERROR: backend 'bob' requires BOB_API_KEY (or legacy BOBSHELL_API_KEY) to be set." echo " Get a key at https://bob.ibm.com (Scope: Inference), then:" - echo " export BOBSHELL_API_KEY=..." + echo " export BOB_API_KEY=..." echo " The key stays on your machine and is never sent to the hive." exit 1 fi @@ -719,6 +729,15 @@ CONTAINER_NAME="${HIVE_CONTAINER_NAME:-hive-contributor}" # by something older than the recipe that passes this, printing exactly what it # printed before. CONTAINER_RUNTIME="${HIVE_CONTAINER_RUNTIME:-docker}" +# Apptainer/Singularity attach to a named instance and do not accept -it. +case "$CONTAINER_RUNTIME" in + apptainer|singularity) + ATTACH_COMMAND="$CONTAINER_RUNTIME exec instance://$CONTAINER_NAME tmux attach -t $TMUX_SESSION" + ;; + *) + ATTACH_COMMAND="$CONTAINER_RUNTIME exec -it $CONTAINER_NAME tmux attach -t $TMUX_SESSION" + ;; +esac echo "Contributor agent is running." echo " Mode: $CONTRIBUTOR_MODE" echo " CLI: $CMD" @@ -736,7 +755,7 @@ fi if [[ "$KRUN_CONSOLE" == "true" ]]; then echo " Console: $CONTAINER_RUNTIME attach $CONTAINER_NAME (detach: Ctrl-P Ctrl-Q)" elif [[ "$CONTRIBUTOR_MODE" == "interactive" ]]; then - echo " Tmux: $CONTAINER_RUNTIME exec -it $CONTAINER_NAME tmux attach -t $TMUX_SESSION" + echo " Tmux: $ATTACH_COMMAND" else # Headless: no pane to attach to. The relay drives a one-shot CLI per task and # writes its lifecycle state (waiting/working/done/failed) here for a probe. diff --git a/bin/contributor-agent.test.sh b/bin/contributor-agent.test.sh index b3ce359935..dba51504db 100755 --- a/bin/contributor-agent.test.sh +++ b/bin/contributor-agent.test.sh @@ -652,6 +652,25 @@ for link in "${HOME_DIR}/AGENTS.md" "${HOME_DIR}/CLAUDE.md"; do fi done +# Omnigent orchestrates several harnesses, so it gets both link names too. +rm -f "${HOME_DIR}/AGENTS.md" "${HOME_DIR}/CLAUDE.md" +OMNIGENT_AGENT_MD="${HOME_DIR}/agent.md" +env -i \ + PATH="${PATH}" \ + HOME="$HOME_DIR" \ + HIVE_REGISTRATION_TOKEN="test-token" \ + HIVE_CONTRIBUTOR_AGENT_TEST_LINK_KNOWLEDGE=1 \ + HIVE_CONTRIBUTOR_AGENT_TEST_KNOWLEDGE_DEST="$OMNIGENT_AGENT_MD" \ + AGENT_BACKEND=omnigent \ + bash "${ROOT_DIR}/bin/contributor-agent.sh" + +for link in "${HOME_DIR}/AGENTS.md" "${HOME_DIR}/CLAUDE.md"; do + if [[ "$(readlink "$link")" != "$OMNIGENT_AGENT_MD" ]]; then + echo "expected Omnigent knowledge link $link to target the Hive export" >&2 + exit 1 + fi +done + mkdir -p "${WORK_DIR}/omp-bin" cat >"${WORK_DIR}/omp-bin/omp" <<'OMP' #!/bin/sh diff --git a/bin/contributor-relay.js b/bin/contributor-relay.js index 7e82999dcf..f15885e1a0 100755 --- a/bin/contributor-relay.js +++ b/bin/contributor-relay.js @@ -1035,6 +1035,19 @@ function warnQuotaGuardAwaitingPublisherOnce() { 'published for this pool yet — holding until the reading lands or the marker goes stale (kubestellar/hive#6987).'); } +// agy publishes Claude/GPT (`3p-*`) and Gemini (`gemini-*`) windows in one +// reading, and the two provider pools are independent. A window whose id names +// the other family must not hold this contributor. Unknown models and windows +// outside either family stay evaluated (conservative). +function quotaWindowAppliesToModel(window, backend = BACKEND, model = MODEL) { + if (backend !== 'agy') return true; + const m = (model || '').toLowerCase(); + const id = (window.id || '').toString().toLowerCase(); + if (/^gemini/.test(m)) return !id.startsWith('3p-'); + if (/claude|opus|sonnet|haiku|gpt/.test(m)) return !id.startsWith('gemini'); + return true; +} + // opts.baseReserveOnly evaluates against the window reserve alone, ignoring the // per-complexity tier reserves. #6833 specifies the base window reserve for the // `ready` hold; the tier reserves exist to judge a SPECIFIC task, and `ready` @@ -1065,6 +1078,7 @@ function evaluateContributorQuota(task, reading = readContributorQuotaReading(), && Number.isInteger(reading.reset_credits_available) && reading.reset_credits_available > 0; let usedBankedReset = false; for (const window of (reading.limits || [])) { + if (!quotaWindowAppliesToModel(window)) continue; const kind = (window.kind || '').toString(); // Every window is evaluated, including kinds this build has never heard of // (kubestellar/hive#6951). Skipping unrecognized kinds meant an EXHAUSTED @@ -3186,8 +3200,12 @@ const CONTAINER_RUNTIME = process.env.HIVE_CONTAINER_RUNTIME || 'docker'; // "microvm-krun" for a krun launch (#12200). The relay never detects it from // inside the container; unset means nothing is declared. const ISOLATION_CLASS = (process.env.HIVE_ISOLATION_CLASS || '').trim(); +// Apptainer/Singularity use instance URIs instead of Docker's -it/name syntax. +const CONTAINER_EXEC_TARGET = ['apptainer', 'singularity'].includes(CONTAINER_RUNTIME) + ? `instance://${CONTAINER_NAME}` + : `-it ${CONTAINER_NAME}`; const ATTACH_COMMAND = process.env.HIVE_CONTAINER_NAME - ? `${CONTAINER_RUNTIME} exec -it ${CONTAINER_NAME} tmux attach -t ${TMUX_SESSION}` + ? `${CONTAINER_RUNTIME} exec ${CONTAINER_EXEC_TARGET} tmux attach -t ${TMUX_SESSION}` : `tmux attach -t ${TMUX_SESSION}`; // detectCapabilities builds the OPTIONAL, client-declared capability object the @@ -3518,7 +3536,7 @@ function headlessChildEnv() { // without it agy warns "--model requires --effort (available: low, medium, // high)" and silently IGNORES the model, so the contributor's configured model // never takes effect. AGY_DEFAULT_EFFORT mirrors agyDefaultEffort in the -// hub-side launcher (src/pkg/agent/manager.go) so a relay agent and a pod agent +// hub-side launcher (src/crux/agent/manager.go) so a relay agent and a pod agent // resolve the same effort. AGENT_REASONING_EFFORT can override it, but only // with a value agy actually accepts — codex's vocabulary is wider (it takes // "minimal"), and forwarding an unknown token here would make agy reject the @@ -5095,7 +5113,7 @@ const BACKEND_LOGIN_HELP = { steps: () => [ 'bob authenticates with an API key, not an interactive login.', 'Stop the relay, set the key, and start it again:', - ' export BOBSHELL_API_KEY=', + ' export BOB_API_KEY=', ' just contribute-hive bob', ], // The generic closing line would be a lie here: no login is coming, and the @@ -5802,7 +5820,7 @@ function shellQuote(s) { return "'" + s.replace(/'/g, "'\\''") + "'"; } -// Keep category names in exact parity with src/pkg/logscrub/handler.go. The Go +// Keep category names in exact parity with src/crux/logscrub/handler.go. The Go // test reads this declaration and fails if one implementation gains or loses a // category without the other (kubestellar/hive#5478). const RELAY_SECRET_PATTERNS = [ @@ -9648,6 +9666,10 @@ function handleMessage(data, hub) { // on a guessed timeout — means we never sit idle on a hub with no work // while a different configured hub has some. console.log(`No task assigned on ${hub.url} — reason: ${sanitizeHubText(msg.reason) || 'unspecified'}; retrying in ${TASK_UNAVAILABLE_RETRY_MS / 1000}s`); + // #14002: some refusals carry a plain-language explanation (for example + // app_not_installed names the org and the install link the operator must + // use). Print it so the contributor knows whether the fix is theirs. + if (msg.message) console.log(sanitizeHubText(msg.message)); scheduleReadyRetry(hub, 'task_unavailable'); break; diff --git a/bin/contributor-relay.test.js b/bin/contributor-relay.test.js index 6da1ac2665..6c0a499c68 100644 --- a/bin/contributor-relay.test.js +++ b/bin/contributor-relay.test.js @@ -8575,7 +8575,7 @@ test('#6437 bob is told about its API key, not to type /login', () => { const relay = loadRelay({ backend: 'bob' }); try { const text = relay.loginBannerLines('bob', 'podman exec -it c tmux attach -t contributor').join('\n'); - assert.ok(/BOBSHELL_API_KEY/.test(text), 'bob needs its API key named'); + assert.ok(/BOB_API_KEY/.test(text), 'bob needs its API key named'); assert.ok(!/\/login/.test(text), 'typing /login cannot authenticate bob'); assert.ok(!/tmux attach/.test(text), 'attaching to the pane cannot fix a missing API key, so the banner must not send the operator there'); @@ -8641,7 +8641,7 @@ test('#6437 waitForCLI prints the blocked backend\'s own banner, not Claude Code try { assert.ok(/needs authentication/.test(output), 'the needs-login banner should have been printed during startup'); - assert.ok(/BOBSHELL_API_KEY/.test(output), + assert.ok(/BOB_API_KEY/.test(output), 'bob was told to authenticate some other way than with its API key'); assert.ok(!/Claude Code/.test(output), 'a blocked bob is announced as Claude Code'); @@ -8718,6 +8718,37 @@ test('#5145 a docker container is unchanged, with or without the new variable', } finally { teardown(withoutVar); } }); +for (const runtime of ['apptainer', 'singularity']) { + test(`#13976 a ${runtime} authentication banner attaches to the named instance`, () => { + const { relay, output } = captureStartupWithImmediateTimers({ + backend: 'claude', + cliStates: ['Please run /login\n', 'Please run /login\n', 'Please run /login\n'], + env: { + HIVE_CONTAINER_NAME: 'hive-contributor-claude-9a1c', + HIVE_CONTAINER_RUNTIME: runtime, + HIVE_AGENT_SESSION: 'claude-session', + }, + }); + try { + const expected = `${runtime} exec instance://hive-contributor-claude-9a1c tmux attach -t claude-session`; + assert.strictEqual(relay.ATTACH_COMMAND, expected); + assert.ok(output.includes('needs authentication'), 'the login banner must fire'); + assert.ok(output.includes(expected), `the banner must print the instance attach command; captured:\n${output}`); + } finally { teardown(relay); } + }); + + test(`#13976 local mode ignores ${runtime} without a container name`, () => { + const relay = loadRelay({ env: { + HIVE_CONTAINER_NAME: '', + HIVE_CONTAINER_RUNTIME: runtime, + HIVE_AGENT_SESSION: 'local-session', + } }); + try { + assert.strictEqual(relay.ATTACH_COMMAND, 'tmux attach -t local-session'); + } finally { teardown(relay); } + }); +} + test('#5145 local mode names no container at all', () => { // HIVE_CONTAINER_NAME is set ONLY by the container arm of the recipe // (Justfile: `-e HIVE_CONTAINER_NAME=...`). In local mode this relay runs on @@ -13789,6 +13820,21 @@ test('#6951 an exhausted window of an UNRECOGNIZED kind refuses work', () => { assert.strictEqual(d.window_id, 'd1'); }); +test('#13946 agy Gemini ignores an exhausted 3p window but Claude/GPT and Gemini pools still hold', () => { + const limits = [{ id: '3p-5h', kind: 'short', pct_remaining: 0 }, { id: 'gemini-5h', kind: 'short', pct_remaining: 99 }]; + const run = (model, lim) => loadRelay({ env: { AGENT_BACKEND: 'agy', AGENT_MODEL: model, + HIVE_CONTRIBUTOR_QUOTA_READING_JSON: JSON.stringify({ state: 'available', limits: lim }) } }) + .evaluateContributorQuota({ title: 'x', complexity: 'medium' }); + assert.strictEqual(run('gemini-3.8-flash-high', limits).admit, true); + const claude = run('claude-opus-5', limits); + assert.strictEqual(claude.admit, false); + assert.strictEqual(claude.window_id, '3p-5h'); + const gem = run('gemini-3.8-flash-high', [{ id: 'gemini-5h', kind: 'short', pct_remaining: 0 }]); + assert.strictEqual(gem.admit, false); + assert.strictEqual(gem.window_id, 'gemini-5h'); + assert.strictEqual(run('mystery-model', limits).admit, false, 'unknown model stays conservative'); +}); + test('#6951 a recognized kind is held as `guarded`, an unrecognized one as `guarded_unknown_window`', () => { // The two holds are identical in effect. They are named differently so an // operator reading the banner can tell "your weekly quota is low" apart from diff --git a/bin/hive-open-issue.sh b/bin/hive-open-issue.sh index a76d86ca7d..9b57b0d975 100755 --- a/bin/hive-open-issue.sh +++ b/bin/hive-open-issue.sh @@ -16,7 +16,7 @@ # forge-resistance as the gh wrapper; this shim adds no privilege. # # Usage (drop-in for the common gh shapes): -# hive-open-issue --repo --title "" [--body ""|--body-file f] [--label a,b] [--parent ] [--blocked-by ] [--close-on-merge] [--needs-decision] +# hive-open-issue --repo --title "" [--body ""|--body-file f] [--label a,b] [--parent ] [--blocked-by ] [--close-on-merge] [--needs-decision] [--no-consolidate ] # hive-open-issue comment --repo --body "" [--needs-decision] [--blocked-on-pr ] # hive-open-issue claim --repo # hive-open-issue close --repo [--override-reason "..."] @@ -56,6 +56,11 @@ # owner's /hive done reply records the done decision, and the lane that next # picks the issue up re-checks the step and closes it once verified. # +# --no-consolidate (issue only) asks the watcher not to fold this +# request into an existing open issue (hivecommons/hive#14360); use it when the +# request is unrelated work that merely shares a file or component with one. +# The reason is required, and the watcher records it in the result. +# # --blocked-on-pr (comment only) parks the issue on open PR in the # same repo (hivecommons/hive#13093): use it when the code a finding names # exists only in an unmerged PR, so there is nothing to ship until it lands. @@ -134,12 +139,14 @@ CLOSE_ON_MERGE=0 NEEDS_DECISION=0 STEP_KIND=""; RELEASE_TAG=""; RELEASE_TARGET=""; STEP_REQUESTED=0 BLOCKED_ON_PR="" +NO_CONSOLIDATE="" +NO_CONSOLIDATE_SET=0 DRY_RUN=0 LABELS=() REMOVE_LABELS=() REVIEWERS=() TEAM_REVIEWERS=() -SUPPORTED_FLAGS="--repo/-R, --title/-t, --body/-b, --body-file/-F, --label/-l, --remove-label, --reviewer, --team-reviewer, --number, --parent, --blocked-by, --close-on-merge, --needs-decision, --blocked-on-pr, --step-kind, --release-tag, --release-target, --override-reason, --dry-run/-n (plus the ignored gh flags --assignee/-a, --milestone/-m, --project/-p, --template/-T, --web/-w, --editor/-e)" +SUPPORTED_FLAGS="--repo/-R, --title/-t, --body/-b, --body-file/-F, --label/-l, --remove-label, --reviewer, --team-reviewer, --number, --parent, --blocked-by, --close-on-merge, --needs-decision, --no-consolidate, --blocked-on-pr, --step-kind, --release-tag, --release-target, --override-reason, --dry-run/-n (plus the ignored gh flags --assignee/-a, --milestone/-m, --project/-p, --template/-T, --web/-w, --editor/-e)" while [ $# -gt 0 ]; do case "$1" in --repo|-R) REPO="$2"; shift 2;; @@ -183,6 +190,8 @@ while [ $# -gt 0 ]; do --step-kind=*) STEP_KIND="${1#*=}"; STEP_REQUESTED=1; shift;; --release-tag=*) RELEASE_TAG="${1#*=}"; STEP_REQUESTED=1; shift;; --release-target=*) RELEASE_TARGET="${1#*=}"; STEP_REQUESTED=1; shift;; + --no-consolidate) NO_CONSOLIDATE_SET=1; NO_CONSOLIDATE="${2:-}"; shift $(( $# > 1 ? 2 : 1 ));; + --no-consolidate=*) NO_CONSOLIDATE_SET=1; NO_CONSOLIDATE="${1#*=}"; shift;; --blocked-on-pr) BLOCKED_ON_PR="$2"; shift 2;; --blocked-on-pr=*) BLOCKED_ON_PR="${1#*=}"; shift;; --dry-run|-n) DRY_RUN=1; shift;; @@ -235,6 +244,15 @@ if [ "$NEEDS_DECISION" -eq 1 ] && [ "$KIND" != "issue" ] && [ "$KIND" != "commen exit 2 fi +if [ "$NO_CONSOLIDATE_SET" -eq 1 ] && [ "$KIND" != "issue" ]; then + echo "hive-open-issue: --no-consolidate only applies when creating an issue; nothing was written." >&2 + exit 2 +fi +if [ "$NO_CONSOLIDATE_SET" -eq 1 ] && [ -z "${NO_CONSOLIDATE//[[:space:]]/}" ]; then + echo "hive-open-issue: --no-consolidate needs a non-empty reason; nothing was written." >&2 + exit 2 +fi + if [ -n "${BLOCKED_ON_PR:-}" ] && [ "$KIND" != "comment" ]; then echo "hive-open-issue: --blocked-on-pr only applies to a comment; nothing was written." >&2 exit 2 @@ -358,9 +376,10 @@ REMOVE_LABELS_JSON="$(printf '%s\n' "${REMOVE_LABELS[@]:-}" | python3 -c 'import REVIEWERS_JSON="$(printf '%s\n' "${REVIEWERS[@]:-}" | python3 -c 'import json,sys; print(json.dumps([x.rstrip("\n") for x in sys.stdin if x.rstrip("\n")]))')" TEAM_REVIEWERS_JSON="$(printf '%s\n' "${TEAM_REVIEWERS[@]:-}" | python3 -c 'import json,sys; print(json.dumps([x.rstrip("\n") for x in sys.stdin if x.rstrip("\n")]))')" BLOCKED_BY_JSON="$(printf '%s\n' "${BLOCKED_BY[@]:-}" | python3 -c 'import json,sys; print(json.dumps([x.rstrip("\n") for x in sys.stdin if x.rstrip("\n")]))')" -python3 - "$TEMP_FILE" "$REQ_FILE" "$KIND" "$REPO" "$TITLE" "${OVERRIDE_REASON:-$BODY}" "$AGENT" "$LABELS_JSON" "${NUMBER:-0}" "$DRY_RUN" "${PARENT:-0}" "$REMOVE_LABELS_JSON" "$REVIEWERS_JSON" "$TEAM_REVIEWERS_JSON" "$BLOCKED_BY_JSON" "$NEEDS_DECISION" "${BLOCKED_ON_PR:-0}" "$STEP_KIND" "$RELEASE_TAG" "$RELEASE_TARGET" "$STEP_REQUESTED" <<'PY' +python3 - "$TEMP_FILE" "$REQ_FILE" "$KIND" "$REPO" "$TITLE" "${OVERRIDE_REASON:-$BODY}" "$AGENT" "$LABELS_JSON" "${NUMBER:-0}" "$DRY_RUN" "${PARENT:-0}" "$REMOVE_LABELS_JSON" "$REVIEWERS_JSON" "$TEAM_REVIEWERS_JSON" "$BLOCKED_BY_JSON" "$NEEDS_DECISION" "${BLOCKED_ON_PR:-0}" "$STEP_KIND" "$RELEASE_TAG" "$RELEASE_TARGET" "$STEP_REQUESTED" "$NO_CONSOLIDATE" <<'PY' import json, os, re, sys temporary, path, kind, repo, title, body, agent, labels, number, dry_run, parent, remove_labels, reviewers, team_reviewers, blocked_by, needs_decision, blocked_on_pr = sys.argv[1:18] +no_consolidate = sys.argv[22] labels = [part.strip() for value in json.loads(labels) for part in value.split(",") if part.strip()] remove_labels = [part.strip() for value in json.loads(remove_labels) @@ -450,6 +469,10 @@ else: # label itself (hivecommons/hive#11215). if needs_decision == "1": req["needs_decision"] = True + # --no-consolidate: the watcher skips open-issue consolidation and records + # the reason (hivecommons/hive#14360). + if no_consolidate.strip(): + req["no_consolidate_reason"] = no_consolidate.strip() if kind == "comment" and int(blocked_on_pr) > 0: # --blocked-on-pr: the watcher adds the marker and the `blocked` label # itself (hivecommons/hive#13093). @@ -492,5 +515,8 @@ else if [ "$NEEDS_DECISION" -eq 1 ]; then echo "hive-open-issue: will park the issue on a maintainer decision (needs-decision label)" fi + if [ "$NO_CONSOLIDATE_SET" -eq 1 ]; then + echo "hive-open-issue: will not consolidate into an existing issue: $NO_CONSOLIDATE" + fi fi echo "hive-open-issue: request $REQ_FILE (Hive validates and fulfills it within one watcher tick; result appears at ${REQ_FILE%.json}.result.json)" diff --git a/bin/hive-setup.sh b/bin/hive-setup.sh index 4e661fe59f..720e13e747 100755 --- a/bin/hive-setup.sh +++ b/bin/hive-setup.sh @@ -25,7 +25,7 @@ # L1: (local tooling only — no hive needed) # L2: supervisor scanner reviewer (analysis + issue filing, NO PRs) # L3: supervisor scanner reviewer architect (opens PRs) -# L4: supervisor scanner reviewer architect outreach sec-check (full autonomy) +# L4: supervisor scanner reviewer architect outreach sec-check (autonomous) # L5+: supervisor scanner reviewer architect outreach sec-check strategist quality set -euo pipefail diff --git a/bin/lib/knowledge-export.sh b/bin/lib/knowledge-export.sh index c67071c205..4362e27d71 100644 --- a/bin/lib/knowledge-export.sh +++ b/bin/lib/knowledge-export.sh @@ -510,6 +510,12 @@ GOOSECFG ln -sf "$agent_md" "${dest_dir}/AGENTS.md" ln -sf "$agent_md" "${dest_dir}/CLAUDE.md" ;; + omnigent) + # Omnigent orchestrates Claude, Codex, Cursor, Pi, and custom harnesses. + # Link both AGENTS.md and CLAUDE.md for harness compatibility. + ln -sf "$agent_md" "${dest_dir}/AGENTS.md" + ln -sf "$agent_md" "${dest_dir}/CLAUDE.md" + ;; *) ln -sf "$agent_md" "${dest_dir}/CLAUDE.md" ;; diff --git a/bin/test_backend_smoke.sh b/bin/test_backend_smoke.sh index 1bb55a487b..00d619db70 100644 --- a/bin/test_backend_smoke.sh +++ b/bin/test_backend_smoke.sh @@ -135,7 +135,7 @@ $k esac done check "backends without a headless mode are exactly the documented set" \ - "aider bob omp" "$(echo "$NON_HEADLESS" | tr ' ' '\n' | grep -v '^$' | sort | tr '\n' ' ' | sed 's/ $//')" + "aider bob omnigent omp" "$(echo "$NON_HEADLESS" | tr ' ' '\n' | grep -v '^$' | sort | tr '\n' ' ' | sed 's/ $//')" fi echo "" @@ -704,7 +704,7 @@ seed_backend_auth() { if [ "$seeded" = "ok" ]; then # Pre-answer every first-run gate a fresh HOME raises, the way the # hub does (inferenceUserConfigSeed / inferenceSettingsSeed in - # src/pkg/agent/manager.go): onboarding + custom-API-key approval in + # src/crux/agent/manager.go): onboarding + custom-API-key approval in # .claude.json, and — crucially — skipDangerousModePermissionPrompt # in .claude/settings.json, the only key that suppresses the "Bypass # Permissions mode" consent menu, whose default selection is diff --git a/bin/test_hive_open_issue.sh b/bin/test_hive_open_issue.sh index bdff2e0c61..7f0c3f5751 100755 --- a/bin/test_hive_open_issue.sh +++ b/bin/test_hive_open_issue.sh @@ -368,6 +368,27 @@ for bad_case in no-park no-kind no-tag no-target mutable-target marker-injection check_exit "invalid release metadata: $bad_case" 2 run_script "invalidreleasebot" "${args[@]}" done check "invalid release metadata writes no request" "" "$(find_req invalidreleasebot)" +# --no-consolidate carries the reason; an empty reason or a non-create is refused. +run_script "noconsolbot" --repo "hiveorg/widgets" --title "Unrelated" --body "b" --no-consolidate "different work" +REQ_FILE="$(find_req noconsolbot)" +if [ -n "$REQ_FILE" ]; then + GOT="$(python3 -c "import json,sys; print(json.load(open(sys.argv[1])).get('no_consolidate_reason'))" "$REQ_FILE")" + check "--no-consolidate sets no_consolidate_reason" "different work" "$GOT" +else + echo " FAIL: request file not created for --no-consolidate test" + FAIL=$((FAIL + 1)) +fi +rm -rf "$REQ_DIR" +mkdir -p "$REQ_DIR" + +set +e +run_script "noconsolemptybot" --repo "hiveorg/widgets" --title "t" --body "b" --no-consolidate "" >/dev/null 2>&1 +BAD_RC=$? +set -e +check "--no-consolidate with an empty reason is refused (exit 2)" "2" "$BAD_RC" +check "--no-consolidate empty refusal writes no request" "" "$(find_req noconsolemptybot)" +rm -rf "$REQ_DIR" +mkdir -p "$REQ_DIR" # --parent=value style also works. run_script "eqparentbot" --repo=hiveorg/widgets --title="Eq parent" --body="body" --parent=42 diff --git a/bin/ttyd-tmux.sh b/bin/ttyd-tmux.sh index e56001944f..36ce5a950f 100755 --- a/bin/ttyd-tmux.sh +++ b/bin/ttyd-tmux.sh @@ -8,7 +8,7 @@ # NOTE: tmux reads history-limit at PANE creation, so the attach-time raise # cannot deepen an already-created pane. The authoritative deep-scrollback # setting is applied at session creation by the agent manager -# (newSessionCommands in src/pkg/agent/manager.go, override +# (newSessionCommands in src/crux/agent/manager.go, override # HIVE_TMUX_HISTORY_LIMIT). # # NOTE: The container uses src/deploy/ttyd-tmux.sh (copied to @@ -67,7 +67,7 @@ fi # Scrollback state legibility (#4399, #4681). Kept character-for-character # identical to src/deploy/ttyd-tmux.sh and to tmuxStatusRight in -# src/pkg/agent/manager.go — test_terminal_scrollback_status.sh asserts all +# src/crux/agent/manager.go — test_terminal_scrollback_status.sh asserts all # three match, because a terminal that scrolls differently depending on which # wrapper attached is worse than one that is uniformly wrong. TTYD_STATUS_RIGHT="${HIVE_TTYD_STATUS_RIGHT:-#{?pane_in_mode,[SCROLLBACK #{scroll_position}/#{history_size} lines back - not following live output - press q to resume] ,#{?mouse_any_flag,[live - this app handles its own scrolling#, so tmux has no line position] ,[live] }}now %H:%M:%S }" diff --git a/changelog.d/added-13536-crux-client.md b/changelog.d/added-13536-crux-client.md new file mode 100644 index 0000000000..ab03144504 --- /dev/null +++ b/changelog.d/added-13536-crux-client.md @@ -0,0 +1,2 @@ +- `crux/api` gains a Go client (`api.NewClient`) for `crux serve`: every verb, SSE log following and the runtime event stream, over a unix socket or TCP with a bearer token (#13536). +- The `crux` CLI gains a remote mode: `--server ADDR` (or `CRUX_SERVER`) with `--token` (or `CRUX_TOKEN`) makes `run`, `ps`, `logs`, `kick` and `stop` drive a `crux serve` over the API instead of an in-process runtime; `logs -f` follows the agent's output (#13536). diff --git a/changelog.d/added-13536-crux-serve-api.md b/changelog.d/added-13536-crux-serve-api.md new file mode 100644 index 0000000000..39ae2a6fcf --- /dev/null +++ b/changelog.d/added-13536-crux-serve-api.md @@ -0,0 +1 @@ +- `crux serve` exposes the agent runtime over a local HTTP+JSON API (Crux Stage 3, [#13536](https://github.com/hivecommons/hive/issues/13536)): a unix socket (`$CRUX_SOCKET`, default `$XDG_RUNTIME_DIR/crux/crux.sock`, created 0600) and optionally TCP with `--listen`, which is refused without a bearer token (`--token` / `$CRUX_TOKEN`). The routes under `/v1` mirror the CLI verbs — list, get, run from a definition, logs (pane, buffer, full, or followed as SSE), kick, stop — plus an SSE event stream of starts, stops, kicks and observed state changes. The wire contract is the stdlib-only `crux/api`; the server is `crux/serve`. diff --git a/changelog.d/added-13833-review-verdict-labels-and-blocked-backlog.md b/changelog.d/added-13833-review-verdict-labels-and-blocked-backlog.md new file mode 100644 index 0000000000..61322ab6b0 --- /dev/null +++ b/changelog.d/added-13833-review-verdict-labels-and-blocked-backlog.md @@ -0,0 +1 @@ +- The review swarm's outcome now shows on the PR, and its follow-up work is tied to the PR it came from ([#13833](https://github.com/hivecommons/hive/issues/13833)). The new `review.verdict_labels` block maps the current head's review state to one label each: `reviewing`, `approve`, `changes_requested`, `requires_human` (`reject` counts as `requires_human`). It works independently of the auto-merge engine's stage labels, so a hive with auto-merge off still shows "in progress / approved / changes requested / needs human" on every PR. Like `human_decision_label`, the labels are never created and never block. With `review_heads`, backlog items are now filed only from the decision step's reconciled findings, never from each head's raw report. That had produced near-duplicate items whose wording defeated the fingerprint check and spent the PR's daily cap. The items are also listed in a "Follow-ups filed" section of the decided review itself, on every head, rather than in a once-per-PR summary comment. Every review-backlog item, in every destination including Linear, is now parked on the PR it was filed from with the `blocked` label and the blocked-on marker, not only when the cited file is missing on the base branch. A new sweep lifts the block with a one-line note when the PR merges. When the PR closes unmerged it also lifts the block and comments "source PR #N closed unmerged — re-triage against current base", leaving the item open for normal triage: a closed PR is often superseded by one that lands the same lines, so backlog items are never closed or cancelled automatically. diff --git a/changelog.d/added-14037-ioscan-cli-classifier-v5-port.md b/changelog.d/added-14037-ioscan-cli-classifier-v5-port.md new file mode 100644 index 0000000000..e3848f856d --- /dev/null +++ b/changelog.d/added-14037-ioscan-cli-classifier-v5-port.md @@ -0,0 +1 @@ +- v5: the semantic prompt-injection classifier can run through the Claude CLI, launched with no tools, no MCP servers, no GitHub token and an empty working directory, on hives without a model endpoint: set `ioscan.classifier.backend: claude` (or pick it in Settings → Security → Input Defense). A CLI backend makes at most `ioscan.classifier.cli_budget_per_eval` calls per scheduler pass (default 8) and `ioscan.classifier.cli_daily_quota` calls per UTC day (default 500); backends that cannot run without tools (copilot, codex, gemini) are refused at boot. Port of [#14060](https://github.com/hivecommons/hive/pull/14060) and [#14070](https://github.com/hivecommons/hive/pull/14070) from v6 ([#14059](https://github.com/hivecommons/hive/issues/14059), part of [#14037](https://github.com/hivecommons/hive/issues/14037)). diff --git a/changelog.d/added-14094-omnigent-backend.md b/changelog.d/added-14094-omnigent-backend.md new file mode 100644 index 0000000000..1e98ee7d28 --- /dev/null +++ b/changelog.d/added-14094-omnigent-backend.md @@ -0,0 +1 @@ +- **`omnigent` contributor backend (T3 experimental).** Registered the Omnigent meta-harness CLI ([omnigent-ai/omnigent](https://github.com/omnigent-ai/omnigent), Apache-2.0) across `KNOWN_BACKENDS`, `CLIBackends`, local confinement matrix tests, and contributor preflights. Omnigent orchestrates Claude Code, Codex, Cursor, Pi, and custom agent harnesses; on the local CLI path it exposes no OS-level sandbox wired by Hive and requires explicit opt-in via `HIVE_OMNIGENT_DANGEROUSLY_RUN_UNCONFINED=1`. Every launch is `omnigent run --server local [--model ]` (`run` opens the agent REPL; `--server local` pins a local Omnigent server, so remote-server mode is unsupported), and container mode stages only `auth_tokens.json` and `config.yaml` from `~/.omnigent`, never session history. Omnigent's internal policy engine and sandboxing compose additively, while Hive's proxy remains strictly authoritative for all GitHub writes and push operations. diff --git a/changelog.d/added-14107-forge-merge.md b/changelog.d/added-14107-forge-merge.md new file mode 100644 index 0000000000..a175e6dfa0 --- /dev/null +++ b/changelog.d/added-14107-forge-merge.md @@ -0,0 +1 @@ +- The GitHub forge adapter implements `forge.Merger` and passes the shared merge contract (#14107): the hold gate refuses with `ErrHoldActive` and a head that is not the pinned commit with `ErrConflict`, both before the merge call; an already-merged change reports `ErrAlreadyMerged`, GitHub's 405/422 map to `ErrNotMergeable`, and `MergeOptions.DeleteBranch` removes the merged head branch. The merge delegates to `pkg/github.MergePR`, so it keeps the mutation boundary, forward-merge guard and merge audit. diff --git a/changelog.d/added-14109-gitlab-forge-merge.md b/changelog.d/added-14109-gitlab-forge-merge.md new file mode 100644 index 0000000000..fea1311b7d --- /dev/null +++ b/changelog.d/added-14109-gitlab-forge-merge.md @@ -0,0 +1 @@ +- forge(gitlab): implement `Forge.Merge` for GitLab with strategy mapping, head-SHA guard, hold label enforcement, and 405/406/409 error mapping, verified by shared contract tests. diff --git a/changelog.d/added-14110-gitea-forge-merge.md b/changelog.d/added-14110-gitea-forge-merge.md new file mode 100644 index 0000000000..8c609dac25 --- /dev/null +++ b/changelog.d/added-14110-gitea-forge-merge.md @@ -0,0 +1 @@ +- Gitea and Forgejo merge implementation on the forge adapter (`POST /repos/{owner}/{repo}/pulls/{number}/merge`), with merge style mapping, head commit guard, and shared merge contract test coverage ([#14110](https://github.com/hivecommons/hive/issues/14110)). diff --git a/changelog.d/added-14112-repo-forge.md b/changelog.d/added-14112-repo-forge.md new file mode 100644 index 0000000000..278038ee77 --- /dev/null +++ b/changelog.d/added-14112-repo-forge.md @@ -0,0 +1 @@ +- `project.repo_policies[].forge` and `forge_url` put one repo on GitLab or Gitea/Forgejo while the rest of the project stays on `project.forge` ([#14112](https://github.com/hivecommons/hive/issues/14112), part of [#14096](https://github.com/hivecommons/hive/issues/14096)). Both keys are validated at load. The governor's escalation writes go to each repo's own forge, and a repo-scoped kick carries the forge kind and names a non-GitHub forge in its text. Unset keys behave exactly as before. diff --git a/changelog.d/added-14113.md b/changelog.d/added-14113.md new file mode 100644 index 0000000000..cc5d8c3cf7 --- /dev/null +++ b/changelog.d/added-14113.md @@ -0,0 +1 @@ +- A merge requested through the hive-merge relay for a repo on GitLab or Gitea/Forgejo (`project.repo_policies[].forge`) now lands through that forge's merge API, after the same merge-eligible check pinned to the reviewed head. GitHub repos merge exactly as before. Authorship detection also recognises GitLab access-token bots and Gitea/Forgejo Actions as bots ([#14113](https://github.com/hivecommons/hive/issues/14113), part of [#14096](https://github.com/hivecommons/hive/issues/14096)). diff --git a/changelog.d/added-14115-repo-card-forge.md b/changelog.d/added-14115-repo-card-forge.md new file mode 100644 index 0000000000..f0fff4d46b --- /dev/null +++ b/changelog.d/added-14115-repo-card-forge.md @@ -0,0 +1 @@ +- Projects cards on the dashboard now name their forge: each repo in `/api/status` carries `forge` (`github`, `gitlab` or `gitea`) and, for GitLab and Gitea/Forgejo, `forgeUrl`, so a hive with `project.forge: gitlab` or `gitea` shows its own host (for example `🦊 GitLab · gitlab.example.com`) and links to it instead of claiming github.com. A repo moved to another forge by `project.repo_policies[].forge` gets its own card label and links ([#14115](https://github.com/hivecommons/hive/issues/14115)). The new [Forges](src/docs/integrations/forges.md) page documents what the GitLab and Gitea/Forgejo adapters support today. diff --git a/changelog.d/added-14116-forgejo-smoke.md b/changelog.d/added-14116-forgejo-smoke.md new file mode 100644 index 0000000000..079931174a --- /dev/null +++ b/changelog.d/added-14116-forgejo-smoke.md @@ -0,0 +1 @@ +- Nightly Forgejo smoke workflow (`.github/workflows/forgejo-smoke.yml`) that starts a digest-pinned Forgejo container, seeds a repo, issue and pull request, and runs the minimal governor loop (read issue → kick → merge via `forge.Merger`) against it; also runs on pushes and PRs touching `src/pkg/forge/**` or `src/pkg/apiproxy/**` ([#14116](https://github.com/hivecommons/hive/issues/14116)). diff --git a/changelog.d/added-14138-crux-module-binary.md b/changelog.d/added-14138-crux-module-binary.md new file mode 100644 index 0000000000..dac9143a8e --- /dev/null +++ b/changelog.d/added-14138-crux-module-binary.md @@ -0,0 +1 @@ +- The Crux module ships its own `crux` binary, `src/crux/cmd/crux`, buildable from the nested module alone (`cd src/crux && make build`, or `GOWORK=off go build ./cmd/crux`) with the same `run`, `ps`, `logs`, `kick`, `stop` and `serve` verbs, `--server` / `CRUX_SERVER` remote mode and a `--version` flag. It loads agent definitions from plain YAML files with the new `crux/defs` package (`--defs ` / `CRUX_DEFS`: portable `kind: AgentDefinition` documents or bare per-agent overlays); the root `cmd/crux` stays as a thin host that reads `hive.yaml`. `src/crux/Makefile` adds `build`, `test`, `vet` and `smoke` (starts `crux serve` on a private socket and drives it over the API), which CI runs; `src/crux/README.md` documents the module ([#14138](https://github.com/hivecommons/hive/issues/14138)). diff --git a/changelog.d/added-14138-crux-publish.md b/changelog.d/added-14138-crux-publish.md new file mode 100644 index 0000000000..1aad796c46 --- /dev/null +++ b/changelog.d/added-14138-crux-publish.md @@ -0,0 +1 @@ +- `src/crux` is now published to `hivecommons/crux` as module `github.com/hivecommons/crux`: `.github/workflows/crux-sync.yml` runs `src/scripts/crux-publish.sh` on every v5 push touching `src/crux/**`, rewrites the module path, refuses to publish a tree that does not build with `GOWORK=off`, commits fast-forward on `hivecommons/crux` `main`, and turns a hive tag `crux/vX.Y.Z` into tag `vX.Y.Z` there so releases are cut from hive (#14138). diff --git a/changelog.d/added-14316-blocked-row-age.md b/changelog.d/added-14316-blocked-row-age.md new file mode 100644 index 0000000000..6a8982b3c7 --- /dev/null +++ b/changelog.d/added-14316-blocked-row-age.md @@ -0,0 +1 @@ +- **Dashboard activity monitor: blocked rows show their age in the since column too.** A blocked issue without `stage_since` now falls back to `created_at` there, matching the Open chevron (#14316). diff --git a/changelog.d/added-14408-chat-prs-list.md b/changelog.d/added-14408-chat-prs-list.md new file mode 100644 index 0000000000..f33e9a76fe --- /dev/null +++ b/changelog.d/added-14408-chat-prs-list.md @@ -0,0 +1 @@ +- Hive Chat `/prs` now lists the actual open pull requests instead of a per-repo count: one row per PR under each repo (`owner/repo#N — title (@author · stage labels · age · ✅/❌/⏳ CI · approved/changes requested · needs decision · hold · mergeable)`), read from the status snapshot's cached PR rows — no extra GitHub call — and capped at 25 with an "… and N more" footer. The command remainder is now honoured as filters: `waiting on review`, `needs decision`, `hold`, `failing ci`, `mine` / `by `, `repo `, `stale` / `older than Nd`; unrecognised words are reported and the hint lists the filters. `/prs waiting on review` used to answer only "hivecommons/hive: 12 open PR(s)" (hivecommons/hive#14408). diff --git a/changelog.d/added-14499-ioscan-copilot-classifier.md b/changelog.d/added-14499-ioscan-copilot-classifier.md new file mode 100644 index 0000000000..a8e0d8db5b --- /dev/null +++ b/changelog.d/added-14499-ioscan-copilot-classifier.md @@ -0,0 +1 @@ +- The ioscan semantic classifier can now run through Copilot CLI tool-less (`ioscan.classifier.backend: copilot`), alongside `claude`: copilot is launched with `--available-tools` set to a sentinel that names no tool, `--disable-builtin-mcps`, `--no-custom-instructions` and `--no-ask-user`, in an empty working directory with no hive token in its environment, and the prompt is passed as the `-p` value since copilot does not read stdin. When `classifier.backend` is empty and no model endpoint is configured, boot now falls back to the tool-less CLI the hive's own agents run on, so a Copilot-only hive at ACMM L5–L6 gets a classifier without extra configuration instead of silently withholding every kick that carries untrusted text. When the classifier is required at the current level but still cannot run, the dashboard shows a persistent `ioscan-classifier-blocked` alert saying kicks are being blocked and how to fix it (set an endpoint, set `ioscan.classifier.backend`, or `ioscan.classifier.fail_mode: open`); the fail-closed default is unchanged. `codex` and `gemini` remain refused (hivecommons/hive#14499). diff --git a/changelog.d/added-contributor-lane-approve-held-runs.md b/changelog.d/added-contributor-lane-approve-held-runs.md new file mode 100644 index 0000000000..7d52332b0b --- /dev/null +++ b/changelog.d/added-contributor-lane-approve-held-runs.md @@ -0,0 +1 @@ +- Contributor lane approves `action_required` workflow runs once per head for a PR author the contributor registry admitted at a `require_tiers` tier when the lane stops at `required-check-missing:` (fork heads included), posting one `hive:contributor-held-runs-approved` audit comment; toggle `auto_merge.contributor_lane.approve_held_runs_for_registry_trusted` (default on). Claim-marker, auto-promoted and association admissions, sentinel/exclude-labelled PRs and `.github/workflows/**` diffs are never approved for (#14519). diff --git a/changelog.d/added-crux-agy-turn.md b/changelog.d/added-crux-agy-turn.md new file mode 100644 index 0000000000..154a10b270 --- /dev/null +++ b/changelog.d/added-crux-agy-turn.md @@ -0,0 +1 @@ +- `crux agy-turn`: the headless agy turn subprocess is a `crux` subcommand, so a manager running under `crux serve` (which types its own executable into the pane) can drive agy agents the same way `hive agy-turn` does in-process. diff --git a/changelog.d/added-crux-hive-callback-contract.md b/changelog.d/added-crux-hive-callback-contract.md new file mode 100644 index 0000000000..a913813299 --- /dev/null +++ b/changelog.d/added-crux-hive-callback-contract.md @@ -0,0 +1 @@ +- Crux stage 2 (#14138): `crux/api/callback` is the stdlib-only wire contract a sidecar runtime uses to call back into hive for scoped GitHub tokens, REST accounting, usage, canary/text scans, conditions, the Linear credential and PR creation (HTTP+JSON over a hive-served 0600 Unix socket, fail closed when hive is unreachable); `crux/hivecap` adapts it onto the ring's existing seams and `pkg/cruxhost` is the hive-side mount over hive's existing implementations. The runtime never holds the App private key. diff --git a/changelog.d/added-crux-serve-hive-callback.md b/changelog.d/added-crux-serve-hive-callback.md new file mode 100644 index 0000000000..31128b7e0f --- /dev/null +++ b/changelog.d/added-crux-serve-hive-callback.md @@ -0,0 +1 @@ +- `crux serve --hive-callback ADDR` (default `$CRUX_HIVE_SOCKET`, token from `$CRUX_HIVE_TOKEN`) dials hive's capability endpoint and feeds the runtime's seams from it (Crux sidecar plan, stage 3): the manager's condition reporter, sandbox pull-request creator, Linear credential resolver and push-token minter, and the process-wide tool-approve text scanner, all through `crux/hivecap` so each fails closed when hive is unreachable. The endpoint's reachability is reported as `hive_callback` on `GET /v1/health` and as the `hive_callback_unavailable` condition on `GET /v1/events` (`hivecap.Watch`, `serve.Server.Conditions`). diff --git a/changelog.d/added-crux-serve-runtime-families.md b/changelog.d/added-crux-serve-runtime-families.md new file mode 100644 index 0000000000..f60a3bf877 --- /dev/null +++ b/changelog.d/added-crux-serve-runtime-families.md @@ -0,0 +1 @@ +- `crux serve` now exposes every `crux.Runtime` family over its API (Crux sidecar plan, stage 3): per-agent start/restart/pause/resume/restart-count, full `crux.Process` snapshots, async kick + validate + dispatch state, kick-log listing and reads, verbatim definitions (add/update/remove/reconcile), overrides and pins, mode sync/clear, fleet breaker/ACMM/relaunch/startup-queue/crash-restart/token-refresh/kick-log archive, Copilot and Claude credentials, the Resumer session routes (`409 resume_busy`, `410 resume_session_gone`, `422 resume_unsupported`), one supervision read per agent, backend auth, nudges, audit, trajectory views, restore seeds, a synchronous `POST /v1/authorize` (204 / 403, unknown action 400 — clients treat anything but 204 as a denial), and the watchdog seam (`/v1/watchdog/agents`, per-agent observe and conditions; `501 not_supported` on a runtime without one). `GET /v1/events` carries the `crux.Host` observer events (kick outcome, pause transition, relaunch, pause persist, prompt, sandbox audit, audit, condition) with typed `data` payloads, and the server can run the Background loops in-process (`Server.Background`; `GET /v1/health` lists them). `crux/api.RuntimeRoutes` is the method→route table the parity tests hold the interface to; `crux/cruxtest.Recorder` is a recording `crux.Runtime`/`crux.Host` fake for anything that drives a runtime without running one. diff --git a/changelog.d/added-crux-sidecar-client.md b/changelog.d/added-crux-sidecar-client.md new file mode 100644 index 0000000000..0771edd51a --- /dev/null +++ b/changelog.d/added-crux-sidecar-client.md @@ -0,0 +1 @@ +- `crux/api/client.Runtime` implements every `crux.Runtime` family over the `crux serve` API (Crux sidecar plan, stage 3): `crux/runtime.New` returns it in `CRUX_MODE=sidecar` (connecting to `$CRUX_SERVER` or the serve socket) instead of `ErrModeUnavailable`, and `NewWatchdogFleet` composes a watchdog fleet for it from the wire watchdog source plus the runtime's Lifecycle. Kick and Host observer events ride one reconnecting `/v1/events` stream; the Resumer sentinels (`ErrResumeBusy`, `ErrResumeSessionGone`, `ErrResumeUnsupported`) round-trip from the server's status codes; Authorizer refusals and transport failures fail closed. A parity test holds `crux.Runtime` and `api.RuntimeRoutes` to each other in both directions so a method cannot be added without a route. diff --git a/changelog.d/added-crux-stage4-credential-writers.md b/changelog.d/added-crux-stage4-credential-writers.md new file mode 100644 index 0000000000..214b18e07d --- /dev/null +++ b/changelog.d/added-crux-stage4-credential-writers.md @@ -0,0 +1 @@ +- Crux stage 4: sidecar agents now get the same credentials as in-process ones. The hive callback gains `agent_mint` (`POST /v1/tokens/mint`) and `bob` (`GET /v1/bob/credential`) families; `crux serve` installs callback-backed `SetAppAuth` (tier-scoped App installation tokens minted by hive over `POST /v1/tokens` and written in place to the per-agent cache), `SetAgentMint`, and the Bob key resolvers, all failing closed (no credential) when hive is unreachable; `sidecarHost` carries every remaining credential setter. Tokens are never logged. diff --git a/changelog.d/added-crux-stage4-failure-semantics.md b/changelog.d/added-crux-stage4-failure-semantics.md new file mode 100644 index 0000000000..2fdfde39e2 --- /dev/null +++ b/changelog.d/added-crux-stage4-failure-semantics.md @@ -0,0 +1 @@ +- Crux stage 4 failure semantics (#14138): hive supervises `crux serve` as a child (`CRUX_SUPERVISE=child|external`, `CRUX_BINARY`) with readiness checks and restart backoff; a lost runtime socket refuses kicks with `crux.ErrRuntimeUnavailable` (no replay), raises the `crux_runtime_unavailable` fleet-report condition, and surfaces as `crux_runtime` in hive's `/api/health`; `crux serve` `/v1/health` reports `started_at`/`uptime_seconds`. diff --git a/changelog.d/added-crux-stage4-host-setters.md b/changelog.d/added-crux-stage4-host-setters.md new file mode 100644 index 0000000000..1ac534d2b9 --- /dev/null +++ b/changelog.d/added-crux-stage4-host-setters.md @@ -0,0 +1 @@ +- Crux sidecar runtimes (`CRUX_MODE=sidecar`, [#14138](https://github.com/hivecommons/hive/issues/14138) stage 4) now reach two more `crux.Host` setters over the hive callback endpoint: the manager's inference routes go to hive's proxy over `PUT`/`DELETE /v1/inference/routes/{agent}` (pushed by a non-blocking worker with retry and a one-minute resync, so a restarted hive relearns them) and sandbox branch pushes run through hive's journaling mutation boundary two-phase over `POST /v1/effects` / `POST /v1/effects/{id}/end` (an unreachable hive refuses the push; a denial is `effects.ErrDenied`; a sidecar that dies mid-push is journaled as abandoned after 15 minutes). `SetSpanStarter` is documented in-process-only. `api.HostSetters` is the disposition table (callback | event | route | config | in-process-only) for every `Host` setter and `crux/api/client`'s parity test fails when a setter is added without a row. In-process mode is unchanged. diff --git a/changelog.d/added-crux-stage4-policy-setters.md b/changelog.d/added-crux-stage4-policy-setters.md new file mode 100644 index 0000000000..3249ca788c --- /dev/null +++ b/changelog.d/added-crux-stage4-policy-setters.md @@ -0,0 +1 @@ +- A sidecar Crux runtime now receives hive's host policy over the hive callback endpoint ([#14138](https://github.com/hivecommons/hive/issues/14138), stage 4): `GET /v1/policy` and `POST /v1/policy/repo-auto-merge` carry build info, the explain-mode default, the kick log source, configured gateway backends, per-repo auto-merge, dead-session recovery ownership and the `agent_sandbox` config, which `crux serve --hive-callback` polls and installs on the manager. In-process mode is unchanged; inference routes, the sandbox mutation boundary and the span starter are not carried yet. diff --git a/changelog.d/added-crux-stage4-proxy-seams.md b/changelog.d/added-crux-stage4-proxy-seams.md new file mode 100644 index 0000000000..6dd8e3a24b --- /dev/null +++ b/changelog.d/added-crux-stage4-proxy-seams.md @@ -0,0 +1 @@ +- Crux stage 4: hive now serves the inference usage ledger to a sidecar runtime over the hive callback's `usage` capability (it was `not_configured`), so a proxy whose `Accounting`/`Usage`/`Canary` seams are the callback adapters records usage and REST traffic into hive's ledgers, refuses GitHub writes while hive is unreachable and resumes when it returns. diff --git a/changelog.d/added-delegatedecision-llm-decider.md b/changelog.d/added-delegatedecision-llm-decider.md new file mode 100644 index 0000000000..79036e585f --- /dev/null +++ b/changelog.d/added-delegatedecision-llm-decider.md @@ -0,0 +1 @@ +- **"Let hive decide": LLM step for non-mechanical cases.** With the new `decisions.delegate.llm.enabled` (default false), an item the owner-delegate rules leave at the reversible default is decided by one model turn that sees the item, its thread, sentinel findings, linked items and the owner's policy; its structured verdict is validated against the delegate's closed action set and goes through the same safety rails, and any error or invalid reply keeps the reversible default. diff --git a/changelog.d/added-evidence-opens-merges-card.md b/changelog.d/added-evidence-opens-merges-card.md new file mode 100644 index 0000000000..c617b5f495 --- /dev/null +++ b/changelog.d/added-evidence-opens-merges-card.md @@ -0,0 +1 @@ +- The hub Evidence card gains an **Issue opens : PR merges** panel: hive-filed issues (audit-log `agent_issue_created` / `hive_issue_created`) ÷ hive-merged PRs (merged by this hive, or agent-authored — the Outcome by author kind set) per repository and fleet-wide over the selected window, read against the governor's cadence band instead of a flat 1:1 — surge/busy expects ≤ 1 (drain), quiet/idle expects 1–2 (bounded scouting), unknown expects parity ±25% — with a trailing-7-day sparkline over a rolling 30 days that should regress toward 1:1. Shipped as `issue_opens_pr_merges` on `/api/evidence/summary` and `cards["opens-merges"]`; thresholds are named constants in `pkg/evidence/opens_merges.go` (`src/docs/review-evidence.md`). diff --git a/changelog.d/added-hive-sidecar-runtime-branch.md b/changelog.d/added-hive-sidecar-runtime-branch.md new file mode 100644 index 0000000000..b57a7f9c15 --- /dev/null +++ b/changelog.d/added-hive-sidecar-runtime-branch.md @@ -0,0 +1 @@ +- `cmd/hive` boots against a sidecar `crux serve` when `CRUX_MODE=sidecar`: the sidecar client is wrapped so the Host observers (kick outcomes, pause transitions, relaunches, prompts, audits, conditions) are fed from the runtime's event stream, and hive serves its capabilities to the runtime over the hive callback endpoint (`$CRUX_HIVE_SOCKET`). The in-process path is unchanged and still the default. diff --git a/changelog.d/added-hosted-managed-agents.md b/changelog.d/added-hosted-managed-agents.md new file mode 100644 index 0000000000..3d9cfb5439 --- /dev/null +++ b/changelog.d/added-hosted-managed-agents.md @@ -0,0 +1 @@ +- Add the opt-in Claude Managed Agents self-hosted external backend prototype with relay capability gating, durable session reconciliation, and USD 25/day ledger metering. diff --git a/changelog.d/added-let-hive-decide.md b/changelog.d/added-let-hive-decide.md new file mode 100644 index 0000000000..e11c0b6e84 --- /dev/null +++ b/changelog.d/added-let-hive-decide.md @@ -0,0 +1 @@ +- **Tracker: "Let hive decide" for items parked at *Needs human decision*.** The *Decide…* submenu on a parked issue or PR row (and the selection bar's *Decide…* for a bulk run) now opens with **Let hive decide**: the hive makes the decision itself as the owner's delegate, under the owner's standing operating rules (`pkg/delegatedecision.PolicyText` — decisive, reversible, keep work moving; prefer contributor PRs over hive duplicates; approve small low-risk changes and trusted-author sentinel alerts with a "reviewed, intended" line; supersede offline contributors now; dedupe aggressively; secrets/PAT/org-permission items are owner-only; quarantine flaky tests; accept CI hygiene gates as ratchets and design proposals with explicit defaults; never auto-close review-backlog items), and records it exactly as a human decision is recorded — `/hive decision ` comment, park labels lifted, `triage/accepted`, queue approval on a released PR head — so the item leaves 085/045 without the owner. Every delegated comment carries `` and the Decisions log marks the row *delegated*, so delegated and human decisions can be told apart and counted. Rails: the delegate never merges, refuses items wearing `hold`, never grants membership/secrets/permissions (`needs-owner` + `hold` with the exact owner action instead), and only *recommends* on policy or scope changes to the hive itself (keeps 085). New endpoint `POST`/`GET /api/repos/{owner}/{repo}/items/{number}/decide-delegated`; new setting `decisions.delegate.enabled` (default true) with a per-repo override. This release's decider is rules-first (the mechanically decidable rules above; the reversible default otherwise) — the LLM-backed decider for the rest is a follow-up. diff --git a/changelog.d/added-outcome-benchmark.md b/changelog.d/added-outcome-benchmark.md new file mode 100644 index 0000000000..1e0f2201b9 --- /dev/null +++ b/changelog.d/added-outcome-benchmark.md @@ -0,0 +1 @@ +- `hivectl evidence benchmark` and `src/scripts/outcome-benchmark.sh` compute a reproducible mediated-vs-unmediated outcome report (revert rate, change-failure, median time-to-merge, CI at merged head, changes-requested-before-merge, cost per merged PR; Wilson intervals, n < 10 not reported) with generated SVG charts under `src/docs/benchmarks/` and a `--readme` flag that rewrites the README's `` block, guarded by a drift test diff --git a/changelog.d/changed-13536-crux-cmd-crux-host.md b/changelog.d/changed-13536-crux-cmd-crux-host.md new file mode 100644 index 0000000000..722163d614 --- /dev/null +++ b/changelog.d/changed-13536-crux-cmd-crux-host.md @@ -0,0 +1 @@ +- Crux: `cmd/crux` is a host composition root in the root module, like `cmd/hive`, and is no longer counted as a ring package by the outbound ratchet; with its `config`/`defsrc` rows gone the ring imports nothing from Hive in non-test code (Stage 2 prerequisite 3 complete). diff --git a/changelog.d/changed-13536-crux-proxy-leaves.md b/changelog.d/changed-13536-crux-proxy-leaves.md new file mode 100644 index 0000000000..880a69e672 --- /dev/null +++ b/changelog.d/changed-13536-crux-proxy-leaves.md @@ -0,0 +1 @@ +- The inference-gateway health store and the issue/comment shape validators moved into the Crux ring as the stdlib-only `pkg/crux/inferencehealth` and `pkg/crux/issueshape`; `pkg/inferencehealth` and `pkg/issueshape` alias every symbol, so hub, dashboard, github and classify callers are unchanged and the proxy no longer imports either Hive package (Stage 2 prerequisite 3, #13536) diff --git a/changelog.d/changed-13536-crux-proxy-seams.md b/changelog.d/changed-13536-crux-proxy-seams.md new file mode 100644 index 0000000000..94eadf041a --- /dev/null +++ b/changelog.d/changed-13536-crux-proxy-seams.md @@ -0,0 +1 @@ +- The GitHub proxy now takes its canary registry, inference usage sink and GitHub REST accounting through three narrow seams (`proxy.CanaryScanner`, `proxy.UsageSink`, `proxy.GitHubRESTAccounting`) that `cmd/hive` wires to `pkg/ioscan`, `pkg/tokens` and `pkg/github`; `ioscan.CanaryLeak` is now an alias of `pkg/crux/types.CanaryLeak`. No behaviour change for a running hive; the proxy no longer imports any Hive package outside the Crux ring (Stage 2 prerequisite 3, #13536) diff --git a/changelog.d/changed-13913.md b/changelog.d/changed-13913.md new file mode 100644 index 0000000000..de915e1275 --- /dev/null +++ b/changelog.d/changed-13913.md @@ -0,0 +1 @@ +- refactor: share the commit-CI pending predicate between pkg/github and stageengine (#13913) diff --git a/changelog.d/changed-13963.md b/changelog.d/changed-13963.md new file mode 100644 index 0000000000..b0a06564bf --- /dev/null +++ b/changelog.d/changed-13963.md @@ -0,0 +1 @@ +- Crux module API: the `crux/types` leaf package is folded into the `crux` root package, so the runtime's value types (process states, kick dispatch and outcomes, restart events, start-failure classes, turn loss, provider status, audit vocabulary, `ConditionReporter`, `PRResult`, `CanaryLeak`, `ProofProvenance`) are defined as `crux.`; `watchdog.Condition`, `ConditionType` and `ConditionStatus` and their constants are now defined in `crux` and aliased by `crux/watchdog`, so the root no longer imports `crux/watchdog` ([#13963](https://github.com/hivecommons/hive/issues/13963), [#13536](https://github.com/hivecommons/hive/issues/13536)). diff --git a/changelog.d/changed-13965-crux-docs-references.md b/changelog.d/changed-13965-crux-docs-references.md new file mode 100644 index 0000000000..06ac195275 --- /dev/null +++ b/changelog.d/changed-13965-crux-docs-references.md @@ -0,0 +1 @@ +- Updated the scanner threat model and deploy README to cite the Crux module paths (`src/crux/agent/manager_tmux.go`) after the module cut ([#13951](https://github.com/hivecommons/hive/issues/13951)). diff --git a/changelog.d/changed-13969.md b/changelog.d/changed-13969.md new file mode 100644 index 0000000000..e73e397fec --- /dev/null +++ b/changelog.d/changed-13969.md @@ -0,0 +1 @@ +- refactor: route agent-mode and capability parsing through crux (#13969) diff --git a/changelog.d/changed-13998-adequacy-rail-definitive.md b/changelog.d/changed-13998-adequacy-rail-definitive.md new file mode 100644 index 0000000000..17e9c2720d --- /dev/null +++ b/changelog.d/changed-13998-adequacy-rail-definitive.md @@ -0,0 +1 @@ +- The contributor-adequacy rail from [#13998](https://github.com/hivecommons/hive/issues/13998) is the owner's standing policy, not a proposal: `prefer-contributor` still prefers the contributor's PR, and the hive's competing PR is closed only after an adequacy review passes. Policy text and docs no longer call it "proposed". diff --git a/changelog.d/changed-14083-proxy-loopback-log-hint.md b/changelog.d/changed-14083-proxy-loopback-log-hint.md new file mode 100644 index 0000000000..652a84008e --- /dev/null +++ b/changelog.d/changed-14083-proxy-loopback-log-hint.md @@ -0,0 +1 @@ +- proxy: carry loopback remediation guidance as a structured 'hint' attribute on debug log failures instead of embedded prose (#14083) diff --git a/changelog.d/changed-14157.md b/changelog.d/changed-14157.md new file mode 100644 index 0000000000..9242d1cef0 --- /dev/null +++ b/changelog.d/changed-14157.md @@ -0,0 +1 @@ +- crux: Stage 3 status, crux serve how-to, CLI remote mode docs (#14157) diff --git a/changelog.d/changed-14426-crux-stage4-docs.md b/changelog.d/changed-14426-crux-stage4-docs.md new file mode 100644 index 0000000000..446866dd90 --- /dev/null +++ b/changelog.d/changed-14426-crux-stage4-docs.md @@ -0,0 +1 @@ +- Crux stage 4 docs: Host-setter disposition table, callback families for credentials and proxy seams, and the failure-semantics decisions in `docs/design/crux.md`, `docs/crux-serve.md` and `crux/README.md` (#14138). diff --git a/changelog.d/changed-14426.md b/changelog.d/changed-14426.md new file mode 100644 index 0000000000..ea6300e8bd --- /dev/null +++ b/changelog.d/changed-14426.md @@ -0,0 +1 @@ +- Crux stage 4: setter dispositions, callback families, failure semantics (#14426) diff --git a/changelog.d/changed-14463.md b/changelog.d/changed-14463.md new file mode 100644 index 0000000000..40ba8ff589 --- /dev/null +++ b/changelog.d/changed-14463.md @@ -0,0 +1 @@ +- chore(fleetreport): IsBenignMessage comment stops enumerating a stale list (#14463) diff --git a/changelog.d/changed-adopted-pr-automerge.md b/changelog.d/changed-adopted-pr-automerge.md new file mode 100644 index 0000000000..251704bc91 --- /dev/null +++ b/changelog.d/changed-adopted-pr-automerge.md @@ -0,0 +1 @@ +- Contributor PRs the hive adopted in place (`stage/074-adopted-by-hive`) now merge under the hive's own authority: the contributor lane admits the adopted head (`contributor-lane:adopted-by-hive`) without the original author's registry tier, claim marker or GitHub association, once CI, reviewer verdict, DCO, diff fences and soak pass on exactly the head the hive adopted/pushed; a contributor push over it still yields back to 072 and the usual trust gates. Opt out with `governor.labels.stage.adoption.auto_merge: false`. diff --git a/changelog.d/changed-contributor-auto-promotion-hive-scope.md b/changelog.d/changed-contributor-auto-promotion-hive-scope.md new file mode 100644 index 0000000000..f58c730407 --- /dev/null +++ b/changelog.d/changed-contributor-auto-promotion-hive-scope.md @@ -0,0 +1 @@ +- Contributor-lane auto-promotion now counts an author's clean merged PRs across every repository the hive manages (`auto_merge.contributor_lane.auto_promotion_scope: hive`, the default; `repo` restores the per-repository count), so a contributor trusted in the main repo is admitted on their first PR to a satellite repo; the audit marker and admission log carry the scope and a per-repo breakdown (#14517). diff --git a/changelog.d/changed-crux-cmdhive-pause-vocabulary.md b/changelog.d/changed-crux-cmdhive-pause-vocabulary.md new file mode 100644 index 0000000000..f7f51e968a --- /dev/null +++ b/changelog.d/changed-crux-cmdhive-pause-vocabulary.md @@ -0,0 +1 @@ +- crux: `BreakerTrigger`, `PauseCausation` and `PauseTransitionEvent` are defined in the `crux` root (aliased by `crux/agent`) and `cmd/hive` names `ProjectContext` and `DefaultAgentMode` as `crux.`; the inbound boundary ratchet's `cmd/hive` row drops 22 → 12, and what remains is the in-process manager and its host wiring (refs #14138). diff --git a/changelog.d/changed-crux-config-migrate-agent.md b/changelog.d/changed-crux-config-migrate-agent.md index d4a72fa214..a6be130fb4 100644 --- a/changelog.d/changed-crux-config-migrate-agent.md +++ b/changelog.d/changed-crux-config-migrate-agent.md @@ -1 +1 @@ -`pkg/agent`, `pkg/crux` and `pkg/crux/cli` now import the stdlib-only `pkg/crux/config` leaf instead of `pkg/config`; the gateway resolution helpers moved onto `AgentRuntimeGovernorConfig` (still reachable through `GovernorConfig`), the `crux` CLI reads `hive.yaml` and definition files through an injected `cli.DefinitionLoader` that `cmd/crux` supplies, and the Crux outbound ratchet no longer lists `config` for those three packages. +- `pkg/agent`, `pkg/crux` and `pkg/crux/cli` now import the stdlib-only `pkg/crux/config` leaf instead of `pkg/config`; the gateway resolution helpers moved onto `AgentRuntimeGovernorConfig` (still reachable through `GovernorConfig`), the `crux` CLI reads `hive.yaml` and definition files through an injected `cli.DefinitionLoader` that `cmd/crux` supplies, and the Crux outbound ratchet no longer lists `config` for those three packages. diff --git a/changelog.d/changed-crux-dashboard-last-agent-symbols.md b/changelog.d/changed-crux-dashboard-last-agent-symbols.md new file mode 100644 index 0000000000..540e3dd3c7 --- /dev/null +++ b/changelog.d/changed-crux-dashboard-last-agent-symbols.md @@ -0,0 +1 @@ +- crux: `CanonicalizeCopilotModel` and `BackendRequiresInteractiveAuth` are defined in the `crux` root (aliased by `crux/agent`), so `pkg/dashboard` no longer imports `crux/agent`; the inbound boundary ratchet drops its `pkg/dashboard` row (3 → 0), leaving `cmd/hive` as the only consumer naming `agent.` (refs #13536). diff --git a/changelog.d/changed-crux-docs-prereq2.md b/changelog.d/changed-crux-docs-prereq2.md new file mode 100644 index 0000000000..fa899b9025 --- /dev/null +++ b/changelog.d/changed-crux-docs-prereq2.md @@ -0,0 +1 @@ +- `src/docs/design/crux.md`: Stage 2 prerequisite 2 recorded as done with the per-edge move/alias/invert table, the outbound baseline after it, the ring package list including `pkg/pushbroker` and `pkg/watsonx`, and prerequisite 3 (proxy and `cmd/crux` edges) scoped with symbol counts (hivecommons/hive#13536). diff --git a/changelog.d/changed-crux-module-cut.md b/changelog.d/changed-crux-module-cut.md new file mode 100644 index 0000000000..6cc58865ca --- /dev/null +++ b/changelog.d/changed-crux-module-cut.md @@ -0,0 +1 @@ +- The Crux agent runtime (`pkg/agent`, `pkg/crux`, `pkg/inference`, `pkg/proxy`, `pkg/sandbox`, `pkg/toolapprove`, `pkg/watchdog`, `pkg/trajectory`, `pkg/pushbroker`, `pkg/watsonx` and the rest of the ring) now lives in the nested Go module `github.com/hivecommons/hive/crux` at `src/crux/`, joined to the root module by `src/go.work`; import paths moved from `github.com/hivecommons/hive/pkg/` to `github.com/hivecommons/hive/crux/` (`pkg/crux` → module root `crux`), the thin hive-side alias packages keep working, and CI, the Dockerfiles and the coverage floors address both modules (`./... ./crux/...` from `src/`). No runtime behaviour change (#13536). diff --git a/changelog.d/changed-crux-process-own-type.md b/changelog.d/changed-crux-process-own-type.md new file mode 100644 index 0000000000..05e1796a9a --- /dev/null +++ b/changelog.d/changed-crux-process-own-type.md @@ -0,0 +1 @@ +- Crux module API: `crux.Process` is now its own struct (a snapshot `crux/agent` copies out of its internal agent record) instead of an alias of `agent.AgentProcess`, and the `crux` root package no longer imports `crux/agent`. `ProjectContext`, `RingBuffer`, the Copilot-token constants, `GitHubTokenLogin`, `DefaultAgentMode` and `AgentAvailableAtACMMLevel` are defined in `crux`; `crux.FromManager` is now `agent.AsRuntime`, and `SanitizePaneText`, `ClassifyAuthPane`, `AuthPaneState` and `PaneShowsBlockingPrompt` moved to the new `crux/paneclass` leaf. This is the prerequisite for folding `crux/types` into the module root ([#13963](https://github.com/hivecommons/hive/issues/13963), [#13536](https://github.com/hivecommons/hive/issues/13536)). diff --git a/changelog.d/changed-crux-process-slice3.md b/changelog.d/changed-crux-process-slice3.md new file mode 100644 index 0000000000..77f9f918cc --- /dev/null +++ b/changelog.d/changed-crux-process-slice3.md @@ -0,0 +1 @@ +- refactor: route Copilot-token and pane-classification helpers through crux ([#13962](https://github.com/hivecommons/hive/issues/13962)) diff --git a/changelog.d/changed-crux-ring-testutil.md b/changelog.d/changed-crux-ring-testutil.md new file mode 100644 index 0000000000..5c39272a6d --- /dev/null +++ b/changelog.d/changed-crux-ring-testutil.md @@ -0,0 +1 @@ +- Crux: the ring's tests use their own `pkg/crux/testutil` helpers instead of `internal/testutil`, so the Crux outbound ratchet baseline is empty in test code too (#13536). diff --git a/changelog.d/changed-crux-service-edges.md b/changelog.d/changed-crux-service-edges.md new file mode 100644 index 0000000000..4efa5fa055 --- /dev/null +++ b/changelog.d/changed-crux-service-edges.md @@ -0,0 +1 @@ +- The agent runtime's self-reported error conditions (restart storms, repeated kick failures, stalls, data-disk pressure, GitHub App token-mint failures, CI-poll nudges, watchdog crash loops) now reach the fleet-report tracker through a `ConditionReporter` seam the host wires (`Manager.SetConditionReporter`, `watchdog.WithConditionReporter`); `pushbroker` takes its GitHub App credential as a `TokenScoper` interface and joins the Crux ring with `watsonx`. No behaviour change for hive; part of Crux Stage 2 prerequisite 2 (hivecommons/hive#13536). diff --git a/changelog.d/changed-crux-sibling-ring-edges.md b/changelog.d/changed-crux-sibling-ring-edges.md new file mode 100644 index 0000000000..36e89d2efb --- /dev/null +++ b/changelog.d/changed-crux-sibling-ring-edges.md @@ -0,0 +1 @@ +- The agent watchdog's credential probe now reads a two-method `UsageProber` slice of the rotation probers (cmd/hive adapts them), and the tool-approval desk scans payload text through a `toolapprove.TextScanner` that `pkg/ioscan` implements and the host installs at init (`SetDefaultTextScanner`); a desk with no text scanner installed fails closed instead of skipping the scan. No behaviour change for hive; part of Crux Stage 2 prerequisite 2 (hivecommons/hive#13536). diff --git a/changelog.d/changed-crux-stage3-docs.md b/changelog.d/changed-crux-stage3-docs.md new file mode 100644 index 0000000000..d3bfc50ede --- /dev/null +++ b/changelog.d/changed-crux-stage3-docs.md @@ -0,0 +1 @@ +- Crux docs: the design page records stage 3 of the sidecar plan (endpoint families, the sidecar client and parity test, callback wiring, `crux agy-turn`) and what stage 4 must do; `crux-serve.md` gains the per-family route table, the Host observer event types and `--hive-callback`; the module README lists `crux/api/client`, `crux/api/callback` and `crux/hivecap`. diff --git a/changelog.d/changed-crux-value-type-edges.md b/changelog.d/changed-crux-value-type-edges.md new file mode 100644 index 0000000000..9273258888 --- /dev/null +++ b/changelog.d/changed-crux-value-type-edges.md @@ -0,0 +1 @@ +- Crux Stage 2 prerequisite 2, value-type edges: `pkg/effects` moved whole into the stdlib-only `pkg/crux/effects` (aliased), `pkg/gitidentity` to `pkg/crux/gitidentity`, the Jev environment contract and jev-decide skill to `pkg/crux/jev` (aliased), the agent-report schema and validator to `pkg/crux/outputschema` with the receipt provenance handle as `pkg/crux/types.ProofProvenance` (aliased by `pkg/outputschema` and `pkg/convergence/proof`), and `pkg/jsonextract` to `pkg/crux/jsonextract` (aliased). `pkg/agent` no longer imports `effects`, `gitidentity`, `jev` or `outputschema`, and `pkg/trajectory` imports nothing outside the ring ([#13536](https://github.com/hivecommons/hive/issues/13536)). diff --git a/changelog.d/changed-l6-autonomous-wording.md b/changelog.d/changed-l6-autonomous-wording.md new file mode 100644 index 0000000000..c911ce9657 --- /dev/null +++ b/changelog.d/changed-l6-autonomous-wording.md @@ -0,0 +1 @@ +- ACMM L6 is now labelled **Autonomous** rather than "Fully Autonomous" everywhere operators see it — the dashboard header pill (`L6 · Autonomous`), the maturity-level dialog, the hub's level picker and ACMM pages, the `level-6` pack name, advisor criteria text, the tool-approval desk rationale, and the docs. L6 still parks items for human decisions, honours `hold`, cannot grant secrets, org membership, or workflow pushes, and keeps an owner in the loop, so "fully" overstated what the level does. The ladder is otherwise unchanged (L1 Inception → L5 Semi-Autonomous → L6 Autonomous); the numeric `acmm_level: 6` config value and the pack filename `level-6.yaml` are untouched, so no configuration changes are needed. diff --git a/changelog.d/changed-price-table-2026-10-26.md b/changelog.d/changed-price-table-2026-10-26.md new file mode 100644 index 0000000000..d937ca0bb5 --- /dev/null +++ b/changelog.d/changed-price-table-2026-10-26.md @@ -0,0 +1 @@ +- Refreshed the model list-price table from OpenRouter (2 rows moved; list prices as of 2026-10-26); the Cost panel estimate now uses the new rates. diff --git a/changelog.d/changed-priorart-filing-verdict-enforced.md b/changelog.d/changed-priorart-filing-verdict-enforced.md new file mode 100644 index 0000000000..888b7c020e --- /dev/null +++ b/changelog.d/changed-priorart-filing-verdict-enforced.md @@ -0,0 +1 @@ +- Issues filed through `hive-open-issue` now follow the prior-art filing verdict: a high-confidence match is refused (naming the existing item) and recorded as a knowledge bead, and a medium match must carry a "Gap vs existing" section and gets the findings attached; each refusal is audited as `agent_issue_prior_art_refused`; off with `runs.triage.prior_art.enabled: false` ([#13879](https://github.com/hivecommons/hive/issues/13879)). diff --git a/changelog.d/fixed-13793-one-review-kick-per-agent.md b/changelog.d/fixed-13793-one-review-kick-per-agent.md new file mode 100644 index 0000000000..ad97a3c748 --- /dev/null +++ b/changelog.d/fixed-13793-one-review-kick-per-agent.md @@ -0,0 +1 @@ +- Review dispatch no longer stacks several review kicks on one agent ([#13793](https://github.com/hivecommons/hive/issues/13793)). With `review_heads` replicas, the replica picker counted distinct PR heads per agent, so replicas holding the same PR tied and one PR's remaining perspectives all went to the first replica. On a two-replica Sonnet head, four kicks reached one Claude session within three minutes; it ran one turn, the other verdicts never arrived, and they expired and were re-sent to the same replica. The planner now sends a review kick (per-perspective, head, combined or decision) only to an agent with none in flight, picks the first free one in name order, and lets work wait for a later cycle when every candidate is busy. Waiting costs no `max_redispatch` or `head_max_retries`. diff --git a/changelog.d/fixed-13898-ci-maintainer-poll-nudge.md b/changelog.d/fixed-13898-ci-maintainer-poll-nudge.md new file mode 100644 index 0000000000..010f8bcedc --- /dev/null +++ b/changelog.d/fixed-13898-ci-maintainer-poll-nudge.md @@ -0,0 +1 @@ +- The CI-poll guard no longer interrupts CI-maintainer lanes or reports their routine cross-repository workflow inspection as polling an owned PR ([#13898](https://github.com/hivecommons/hive/issues/13898)). diff --git a/changelog.d/fixed-13910-agent-classify-tighten-warns.md b/changelog.d/fixed-13910-agent-classify-tighten-warns.md new file mode 100644 index 0000000000..70de17b918 --- /dev/null +++ b/changelog.d/fixed-13910-agent-classify-tighten-warns.md @@ -0,0 +1 @@ +- Agent: 'failed to tighten …' interactive-home WARNs are now classified as agent_dir_setup_failed instead of unclassified (#13910). diff --git a/changelog.d/fixed-13914-review-backlog-split-on-host-close.md b/changelog.d/fixed-13914-review-backlog-split-on-host-close.md new file mode 100644 index 0000000000..d31d7c9f73 --- /dev/null +++ b/changelog.d/fixed-13914-review-backlog-split-on-host-close.md @@ -0,0 +1 @@ +- Review findings the backlog linker folded into another issue no longer vanish when that issue closes: every ten minutes, alongside the closed-issue stage sweep, the hive reads the `hive:review-backlog-link` comments on recently closed backlog hosts and re-files each still-unchecked `- [ ]` finding as its own backlog issue ("Split from #N (host closed)"), comments the new link on the host, and moves the finding's ledger and fingerprint records to the new issue. Hosts closed as not planned or duplicate keep the findings that match them and only split out unrelated ones; re-runs are idempotent via the new `split_keys` ledger entry keyed on the link-marker key ([#13914](https://github.com/hivecommons/hive/issues/13914), part of [#13906](https://github.com/hivecommons/hive/issues/13906)). diff --git a/changelog.d/fixed-13920-pane-timeout-warn-classify.md b/changelog.d/fixed-13920-pane-timeout-warn-classify.md new file mode 100644 index 0000000000..57233bbd69 --- /dev/null +++ b/changelog.d/fixed-13920-pane-timeout-warn-classify.md @@ -0,0 +1 @@ +- Fleet report: 'prompt timeout — dumping pane' and 'pane content at timeout' agent-readiness diagnostic WARNs are now classified as agent_pane_timeout (low severity) instead of unclassified_error_repeated (#13920). diff --git a/changelog.d/fixed-13921-fleetreport-classify-repeated-warns.md b/changelog.d/fixed-13921-fleetreport-classify-repeated-warns.md new file mode 100644 index 0000000000..0e9cf7a112 --- /dev/null +++ b/changelog.d/fixed-13921-fleetreport-classify-repeated-warns.md @@ -0,0 +1 @@ +- Fleet report: repeated 'failed to ingest bead fact', 'fleet report: cleared comment failed' and 'failed to restore inference home mode' and 'failed to tighten inference home mode' WARNs are now classified instead of filed as unclassified_error_repeated (#13922, #13926, #13930, #13935). diff --git a/changelog.d/fixed-13946.md b/changelog.d/fixed-13946.md new file mode 100644 index 0000000000..6412fb2de2 --- /dev/null +++ b/changelog.d/fixed-13946.md @@ -0,0 +1 @@ +- **Contributor quota guard now scopes to the active model's provider pool** (hivecommons/hive#13946). The contributor relay's quota guard evaluated every agy quota window regardless of which model the contributor actually runs, so an exhausted Claude/GPT (`3p-*`) window paused Gemini contributors that never draw from it. The guard now keeps only the windows of the effective model's pool (Gemini → `gemini-*`, Claude/GPT → `3p-*`); unknown models, non-agy backends and unrecognised window ids are still evaluated conservatively, and the hold message continues to name the triggering window. diff --git a/changelog.d/fixed-13947.md b/changelog.d/fixed-13947.md new file mode 100644 index 0000000000..aa58dceff1 --- /dev/null +++ b/changelog.d/fixed-13947.md @@ -0,0 +1 @@ +- **Review-backlog hosts closed before the split sweep shipped are now split too** (hivecommons/hive#13947). The closed-host split sweep from #13914 only lists issues updated inside its 48h window, so a review-backlog host closed earlier (e.g. #13823, #13629) kept its still-unchecked folded findings closed with it. A one-off `BackfillReviewBacklogSplits` pass now runs after each split sweep: it walks the review-backlog ledger's host index, fetches each host directly and splits every closed one through the same idempotent path (ledger `split_keys`). Once a pass visits every host without hitting the per-pass cap or a transient error, the ledger records `split_backfilled` and later passes are no-ops. diff --git a/changelog.d/fixed-13955-review-watcher-policy-denial.md b/changelog.d/fixed-13955-review-watcher-policy-denial.md new file mode 100644 index 0000000000..39357536c4 --- /dev/null +++ b/changelog.d/fixed-13955-review-watcher-policy-denial.md @@ -0,0 +1 @@ +- Policy denials logged by the request watchers (for example a repo not allowlisted for in-place review revision) are no longer self-reported as repeated unclassified runtime errors ([#13955](https://github.com/hivecommons/hive/issues/13955)). diff --git a/changelog.d/fixed-13976-apptainer-attach-hint.md b/changelog.d/fixed-13976-apptainer-attach-hint.md new file mode 100644 index 0000000000..ef7f2adbbf --- /dev/null +++ b/changelog.d/fixed-13976-apptainer-attach-hint.md @@ -0,0 +1 @@ +- Contributor status and authentication hints now use named-instance syntax for Apptainer and Singularity, so the printed tmux attach command works with those runtimes ([#13976](https://github.com/hivecommons/hive/issues/13976)). diff --git a/changelog.d/fixed-13986-blocked-competitor-checks.md b/changelog.d/fixed-13986-blocked-competitor-checks.md new file mode 100644 index 0000000000..884c9ae589 --- /dev/null +++ b/changelog.d/fixed-13986-blocked-competitor-checks.md @@ -0,0 +1 @@ +- A delegated decision no longer treats a competing contributor PR whose checks never ran or are still pending as mergeable ([#13986](https://github.com/hivecommons/hive/issues/13986)). diff --git a/changelog.d/fixed-13986-delegated-pr-context.md b/changelog.d/fixed-13986-delegated-pr-context.md new file mode 100644 index 0000000000..bf119cb111 --- /dev/null +++ b/changelog.d/fixed-13986-delegated-pr-context.md @@ -0,0 +1 @@ +- Delegated decisions now compare real linked open PRs (including stage-store claim links) and use contributor commits, comments, and reviews within the configured presence window instead of park captions to detect absence; a competitor whose required checks are red is no longer treated as mergeable, and PR decisions fail closed until the dashboard snapshot exists ([#13986](https://github.com/hivecommons/hive/issues/13986)). diff --git a/changelog.d/fixed-13988-process-snapshot.md b/changelog.d/fixed-13988-process-snapshot.md new file mode 100644 index 0000000000..099446e0e2 --- /dev/null +++ b/changelog.d/fixed-13988-process-snapshot.md @@ -0,0 +1 @@ +- Agent status snapshots now preserve CI waiting state, CI polling nudge counts, bootstrap overrides, kick refusals, and diagnostic errors. diff --git a/changelog.d/fixed-13992-opens-merges-repo-mode.md b/changelog.d/fixed-13992-opens-merges-repo-mode.md new file mode 100644 index 0000000000..d150a8a5fd --- /dev/null +++ b/changelog.d/fixed-13992-opens-merges-repo-mode.md @@ -0,0 +1 @@ +- Evidence **Issue opens : PR merges** panel: a `?repo=`-filtered view now reads that repo's own governor cadence mode (`State.RepoModes`, under `cadence_scope: per_repo`) instead of always judging it against the fleet-wide mode, so a quiet repo inside a busy fleet is no longer read against the fleet's drain band; falls back to the fleet mode when the governor has no per-repo mode for it (#13992). diff --git a/changelog.d/fixed-14000-llm-verdict-actions.md b/changelog.d/fixed-14000-llm-verdict-actions.md new file mode 100644 index 0000000000..f8f0ccc242 --- /dev/null +++ b/changelog.d/fixed-14000-llm-verdict-actions.md @@ -0,0 +1 @@ +- Owner-delegate LLM decider: a model reply must carry its recording action (`comment-decision`, or `close-as-duplicate` for the close verdicts) so it can never lift `stage/085-needs-decision` with a bare `labels-remove`; `close-duplicate` closes only the newer item; verdict and actions are validated together so the job and audit log never record a verdict the executor did not carry out (follow-up to #14000). diff --git a/changelog.d/fixed-14001.md b/changelog.d/fixed-14001.md new file mode 100644 index 0000000000..19bfa5c8bc --- /dev/null +++ b/changelog.d/fixed-14001.md @@ -0,0 +1 @@ +- Delegated decisions now check the DCO sign-off and reporter-trust gates before preferring a competing PR, so a PR parked on either gate is no longer approved or superseded just because a duplicate exists (#14001). diff --git a/changelog.d/fixed-14002-app-not-installed.md b/changelog.d/fixed-14002-app-not-installed.md new file mode 100644 index 0000000000..c52215d5a7 --- /dev/null +++ b/changelog.d/fixed-14002-app-not-installed.md @@ -0,0 +1 @@ +- Contributor hubs whose GitHub App has no installation on the org (`installation_id: 0`) now refuse work with `app_not_installed` and a message naming the org and the App install link, instead of `repo_unmintable`; no GitHub request is made and no repository is excluded ([#14002](https://github.com/hivecommons/hive/issues/14002)). diff --git a/changelog.d/fixed-14003-mint-failure-queue.md b/changelog.d/fixed-14003-mint-failure-queue.md new file mode 100644 index 0000000000..650cfe3f89 --- /dev/null +++ b/changelog.d/fixed-14003-mint-failure-queue.md @@ -0,0 +1 @@ +- Exclude repositories with recorded token mint failures from contributor ready/actionable totals during their cooldown, expose the reason and expiry in opted-in withheld diagnostics, and preserve `repo_unmintable` refusals on subsequent work requests ([#14003](https://github.com/hivecommons/hive/issues/14003)). diff --git a/changelog.d/fixed-14011-ledger-unknown-reconcile.md b/changelog.d/fixed-14011-ledger-unknown-reconcile.md new file mode 100644 index 0000000000..0b7837638e --- /dev/null +++ b/changelog.d/fixed-14011-ledger-unknown-reconcile.md @@ -0,0 +1 @@ +- Issue comments posted through the hive client no longer stay fenced forever after an ambiguous failure: an `issue_comment` operation journaled Unknown is now reconciled against the issue's existing comments (replayed as success when the comment landed, retried when it did not), and expired after `DefaultUnresolvedTTL` when the comments cannot be read, clearing the `effect_ledger_op_stuck` self-report ([#14011](https://github.com/hivecommons/hive/issues/14011)). diff --git a/changelog.d/fixed-14012-fleet-report-cleared-403.md b/changelog.d/fixed-14012-fleet-report-cleared-403.md new file mode 100644 index 0000000000..b93e7d6e2d --- /dev/null +++ b/changelog.d/fixed-14012-fleet-report-cleared-403.md @@ -0,0 +1 @@ +- A fleet-report cleared comment refused with 403 (a hosted hive's App cannot write to the fleet repo) is now journaled as a definite refusal instead of an uncertain outcome needing reconciliation, so it is given up once rather than re-warned every tick ([#14012](https://github.com/hivecommons/hive/issues/14012)). diff --git a/changelog.d/fixed-14013-fleetreport-classes.md b/changelog.d/fixed-14013-fleetreport-classes.md new file mode 100644 index 0000000000..4b5bdfcea3 --- /dev/null +++ b/changelog.d/fixed-14013-fleetreport-classes.md @@ -0,0 +1 @@ +- The fleet-report classifier now recognises the GitHub App "installation does not cover every configured repo" warning (new low-severity `github_app_repo_not_covered` class) and the `copilot sniff: client TLS handshake failed` line (now part of `proxy_tls_handshake_failed`), so they no longer surface as `unclassified_error_repeated` ([#14013](https://github.com/hivecommons/hive/issues/14013), [#14014](https://github.com/hivecommons/hive/issues/14014)). diff --git a/changelog.d/fixed-14016-ci-poll-sweep.md b/changelog.d/fixed-14016-ci-poll-sweep.md new file mode 100644 index 0000000000..43fc5379ad --- /dev/null +++ b/changelog.d/fixed-14016-ci-poll-sweep.md @@ -0,0 +1 @@ +- The CI-poll guard no longer reports a merge sweep or CI-repair pass that reads each of several PRs' checks once as polling an owned PR; only command lines repeated within a kick count toward the nudge threshold and the `ci_poll_nudge` self-report ([#14016](https://github.com/hivecommons/hive/issues/14016)). diff --git a/changelog.d/fixed-14020-copilot-model-fallback-selfreport-noise.md b/changelog.d/fixed-14020-copilot-model-fallback-selfreport-noise.md new file mode 100644 index 0000000000..c5e4c26166 --- /dev/null +++ b/changelog.d/fixed-14020-copilot-model-fallback-selfreport-noise.md @@ -0,0 +1 @@ +- A hive whose copilot agents share a model id the Copilot CLI rejects no longer opens an `unclassified_error_repeated` self-report for the burst of `copilot CLI substituted the configured model` warnings; each agent already warns only once per substitution ([#13659](https://github.com/hivecommons/hive/issues/13659)), but many agents warning once each in the same boot crossed the repeat threshold. The substitution still shows on each agent's dashboard error and in the container log ([#14020](https://github.com/hivecommons/hive/issues/14020)). diff --git a/changelog.d/fixed-14031-selfreport-noise-classes.md b/changelog.d/fixed-14031-selfreport-noise-classes.md index a19526ba98..3dbe5b657f 100644 --- a/changelog.d/fixed-14031-selfreport-noise-classes.md +++ b/changelog.d/fixed-14031-selfreport-noise-classes.md @@ -1 +1 @@ -- Fleet report: repeated `proxy: CONNECT dial failed` WARNs are now classified as `proxy_connect_dial_failed` (medium, component `github-proxy`) and `inference upstream failed` / `inference upstream error` ERRORs as `inference_upstream_error` (medium, component `inference-gateway`) instead of `unclassified_error_repeated` ([#14031](https://github.com/hivecommons/hive/issues/14031)). +- Fleet report: repeated `proxy: CONNECT dial failed` WARNs are now classified as `proxy_connect_dial_failed` (medium, component `github-proxy`) and `inference upstream failed` / `inference upstream error` ERRORs as `inference_upstream_error` (medium, component `inference-gateway`) instead of `unclassified_error_repeated`; the self-authored automerge sweep now logs `self-authored automerge sweep skipped PR` at WARN once per PR/reason/error and at Debug on later ticks ([#14031](https://github.com/hivecommons/hive/issues/14031), [#14038](https://github.com/hivecommons/hive/issues/14038), [#14042](https://github.com/hivecommons/hive/issues/14042)). diff --git a/changelog.d/fixed-14033-channel-poll-403-warn-once.md b/changelog.d/fixed-14033-channel-poll-403-warn-once.md new file mode 100644 index 0000000000..bfb9008059 --- /dev/null +++ b/changelog.d/fixed-14033-channel-poll-403-warn-once.md @@ -0,0 +1 @@ +- Hosted hubs no longer repeat HTTP 403 warnings for the channel commit-date, channel-distance and SHA-poll commit-message GitHub fetches on every poll; repeats are logged at debug (#14033, #14034, #14040) diff --git a/changelog.d/fixed-14062-classifier-hooks-off.md b/changelog.d/fixed-14062-classifier-hooks-off.md new file mode 100644 index 0000000000..adca0541c6 --- /dev/null +++ b/changelog.d/fixed-14062-classifier-hooks-off.md @@ -0,0 +1 @@ +- The tool-less `claude` classifier backend now passes `--settings '{"disableAllHooks":true}'`, so hooks in the host's `~/.claude/settings.json` do not run in the classifier session ([#14062](https://github.com/hivecommons/hive/issues/14062)). diff --git a/changelog.d/fixed-14065-ci-poll-baseline-scrollback.md b/changelog.d/fixed-14065-ci-poll-baseline-scrollback.md new file mode 100644 index 0000000000..385c44e0cc --- /dev/null +++ b/changelog.d/fixed-14065-ci-poll-baseline-scrollback.md @@ -0,0 +1 @@ +- CI-poll nudges no longer miss repeated polls of one PR when earlier reads have scrolled out of the 2000-line terminal capture (#14065) diff --git a/changelog.d/fixed-14076-no-proxy-loopback.md b/changelog.d/fixed-14076-no-proxy-loopback.md new file mode 100644 index 0000000000..2ffa62b085 --- /dev/null +++ b/changelog.d/fixed-14076-no-proxy-loopback.md @@ -0,0 +1 @@ +- Set NO_PROXY for loopback on every agent backend and stop warning when the proxy fails to forward to loopback (#14076) diff --git a/changelog.d/fixed-14080.md b/changelog.d/fixed-14080.md new file mode 100644 index 0000000000..662abbf2b4 --- /dev/null +++ b/changelog.d/fixed-14080.md @@ -0,0 +1 @@ +- fix(dashboard): settings parity test matches whole paths, not substrings (#14080) diff --git a/changelog.d/fixed-14086.md b/changelog.d/fixed-14086.md new file mode 100644 index 0000000000..0ae024beb9 --- /dev/null +++ b/changelog.d/fixed-14086.md @@ -0,0 +1 @@ +- Plain-forward proxy transport no longer honours HTTP_PROXY/HTTPS_PROXY on the proxy process, so the guarded destination check always evaluates the agent-named host (#14086) diff --git a/changelog.d/fixed-14091.md b/changelog.d/fixed-14091.md new file mode 100644 index 0000000000..cf70ac93cc --- /dev/null +++ b/changelog.d/fixed-14091.md @@ -0,0 +1 @@ +- fix(close-on-merge): settle a referenced issue that GitHub reports as not found (404) as `issue_not_found` at Info instead of re-reading it every sweep and logging a WARN, which produced repeated `unclassified_error_repeated` self-reports (#14088, #14091) diff --git a/changelog.d/fixed-14113-gitlab-406-closed.md b/changelog.d/fixed-14113-gitlab-406-closed.md new file mode 100644 index 0000000000..33049ddb3e --- /dev/null +++ b/changelog.d/fixed-14113-gitlab-406-closed.md @@ -0,0 +1 @@ +- The GitLab forge adapter no longer reports a closed (unmerged) merge request as already merged: a closed MR is refused as not mergeable before any write, and a 406 from the accept endpoint is resolved against the re-read MR state. Follow-up to [#14151](https://github.com/hivecommons/hive/pull/14151) ([#14113](https://github.com/hivecommons/hive/issues/14113)). diff --git a/changelog.d/fixed-14123-installation-id-lock.md b/changelog.d/fixed-14123-installation-id-lock.md new file mode 100644 index 0000000000..ccb0fea137 --- /dev/null +++ b/changelog.d/fixed-14123-installation-id-lock.md @@ -0,0 +1 @@ +- GitHub App token minting, installation verification and `InstallationID()` now read the installation ID under the auth lock, so adopting a discovered installation can no longer race a concurrent scoped-token mint or send a request with a torn ID ([#14123](https://github.com/hivecommons/hive/issues/14123)). diff --git a/changelog.d/fixed-14135-sliced-lane-ok-line.md b/changelog.d/fixed-14135-sliced-lane-ok-line.md new file mode 100644 index 0000000000..e4fde40a30 --- /dev/null +++ b/changelog.d/fixed-14135-sliced-lane-ok-line.md @@ -0,0 +1 @@ +- CI: the sliced `test-hub` / `test-agent` lanes append the `ok ` / `FAIL ` summary line the coverage scorer classifies packages by, so a partial selection that includes `pkg/hub` or `crux/agent` no longer fails with "produced no test result" and their floors are scored on full runs again (#14135). diff --git a/changelog.d/fixed-14138-crux-sync-sigpipe.md b/changelog.d/fixed-14138-crux-sync-sigpipe.md new file mode 100644 index 0000000000..2c74b67138 --- /dev/null +++ b/changelog.d/fixed-14138-crux-sync-sigpipe.md @@ -0,0 +1,2 @@ +- The `crux-sync` workflow no longer aborts after committing the publish tree: the commit summary was piped through `head`, which under `pipefail` turned into exit 141 on a large first sync (#14138). +- `crux-sync` no longer commits when the only difference is the `.hive-sync` timestamp, so tag-only runs and re-dispatches do not add empty syncs (#14138). diff --git a/changelog.d/fixed-14138-crux-sync-tag.md b/changelog.d/fixed-14138-crux-sync-tag.md new file mode 100644 index 0000000000..957b6e70b7 --- /dev/null +++ b/changelog.d/fixed-14138-crux-sync-tag.md @@ -0,0 +1 @@ +- `crux-sync` tag runs (`crux/vX.Y.Z`) no longer commit to `hivecommons/crux` `main`; they tag the already-synced commit whose tree matches the tagged hive commit, fail clearly when none does, and refuse to overwrite an existing tag (#14138). diff --git a/changelog.d/fixed-14138-hivecap-expiry.md b/changelog.d/fixed-14138-hivecap-expiry.md new file mode 100644 index 0000000000..5221a56d55 --- /dev/null +++ b/changelog.d/fixed-14138-hivecap-expiry.md @@ -0,0 +1 @@ +- `crux/hivecap`'s direct-vs-socket parity test used a fixed token expiry of 2026-10-25T14:00Z and began failing once that moment passed; it now uses a relative expiry (#14138). diff --git a/changelog.d/fixed-14138-logscrub-standalone.md b/changelog.d/fixed-14138-logscrub-standalone.md new file mode 100644 index 0000000000..2bead562cf --- /dev/null +++ b/changelog.d/fixed-14138-logscrub-standalone.md @@ -0,0 +1 @@ +- The Crux module's tests now pass when it is built standalone (the published `hivecommons/crux` copy): `crux/logscrub`'s relay secret-pattern parity test and `crux/agent`'s `deploy/entrypoint.sh` and `src/Dockerfile` checks skip when those hive-root files are absent instead of failing on a missing file; inside hive, `HIVE_TEST_REQUIRE_BEHAVIOURAL=1` still turns a missing file into a failure (#14138). diff --git a/changelog.d/fixed-14146.md b/changelog.d/fixed-14146.md new file mode 100644 index 0000000000..d231848da1 --- /dev/null +++ b/changelog.d/fixed-14146.md @@ -0,0 +1 @@ +- SHA poller now backs off and logs once when a commit message can't be fetched (e.g. GitHub 403), and the fleet report classifies it (#14146) diff --git a/changelog.d/fixed-14150-github-sweeps-skip-other-forges.md b/changelog.d/fixed-14150-github-sweeps-skip-other-forges.md new file mode 100644 index 0000000000..424314185e --- /dev/null +++ b/changelog.d/fixed-14150-github-sweeps-skip-other-forges.md @@ -0,0 +1 @@ +- The automerge sweeps (queued, self-authored, trusted-author, contributor lane) and the close-on-merge ledger and backfill sweeps now skip repos whose `project.forge` / `project.repo_policies[].forge` resolves to GitLab or Gitea. Before, they looked those repos up on GitHub under the same slug ([#14150](https://github.com/hivecommons/hive/issues/14150), part of [#14096](https://github.com/hivecommons/hive/issues/14096)). Merging and closing on those forges is still to come. diff --git a/changelog.d/fixed-14155-supersession-needs-human-clears-with-cause.md b/changelog.d/fixed-14155-supersession-needs-human-clears-with-cause.md new file mode 100644 index 0000000000..71e69bfc44 --- /dev/null +++ b/changelog.d/fixed-14155-supersession-needs-human-clears-with-cause.md @@ -0,0 +1 @@ +- The supersession sweep's `needs-human` now clears with its cause: a contributor PR that only references the closed issue (`Refs #N`, `Part of #N`) is noted, not parked, and a park from an earlier pass is released when the supersession no longer holds (issue reopened, claim rewritten, reference-only link); a person's `needs-human` is left alone. The `stage/085-needs-decision` note names `needs-human` as the condition instead of "the lane condition named above" (#14155). diff --git a/changelog.d/fixed-14163-contribute-ws-abnormal-closure.md b/changelog.d/fixed-14163-contribute-ws-abnormal-closure.md new file mode 100644 index 0000000000..495197a495 --- /dev/null +++ b/changelog.d/fixed-14163-contribute-ws-abnormal-closure.md @@ -0,0 +1 @@ +- A contributor websocket that ends without a close frame (code 1006 — relay killed, laptop slept, NAT timeout) is now logged at INFO as a dropped connection instead of a WARN `[contribute-ws] read error`, so routine contributor network drops no longer surface as `unclassified_error_repeated` fleet self-reports; close codes that name a protocol or policy problem still log at WARN ([#14163](https://github.com/hivecommons/hive/issues/14163)). diff --git a/changelog.d/fixed-14171.md b/changelog.d/fixed-14171.md new file mode 100644 index 0000000000..ade75db43f --- /dev/null +++ b/changelog.d/fixed-14171.md @@ -0,0 +1 @@ +- Retiring the old `needs-decision` / `needs-direction` label definitions no longer logs a warning when its context deadline runs out: a timeout now defers quietly to the next cycle, and a pass whose caller's deadline is already spent makes no request (#14171). diff --git a/changelog.d/fixed-14177-archived-repo-writes.md b/changelog.d/fixed-14177-archived-repo-writes.md new file mode 100644 index 0000000000..4a76a924ff --- /dev/null +++ b/changelog.d/fixed-14177-archived-repo-writes.md @@ -0,0 +1 @@ +- Unpark sweep and stage-label PR-link comments now skip repositories GitHub has archived (403 "Repository was archived so is read-only") instead of retrying and warning on every sweep tick (#14177, #14213). diff --git a/changelog.d/fixed-14178-failure-streak-benign.md b/changelog.d/fixed-14178-failure-streak-benign.md new file mode 100644 index 0000000000..eefbf06c60 --- /dev/null +++ b/changelog.d/fixed-14178-failure-streak-benign.md @@ -0,0 +1 @@ +- The fleet self-report feed no longer reports the contribute-ws "refusing task: contributor failure streak" warning as a repeated runtime error: the claim pause is a policy outcome re-logged on every poll from the paused agent, not a defect ([#14178](https://github.com/hivecommons/hive/issues/14178)). diff --git a/changelog.d/fixed-14188-quality-ci-poll-nudge.md b/changelog.d/fixed-14188-quality-ci-poll-nudge.md new file mode 100644 index 0000000000..20405f2349 --- /dev/null +++ b/changelog.d/fixed-14188-quality-ci-poll-nudge.md @@ -0,0 +1 @@ +- The CI-poll guard now exempts the quality lane's mandatory shared-CI baseline triage, alongside reviewer and CI-maintainer lanes, so cross-PR check reads do not trigger a stop-polling nudge or `ci_poll_nudge` self-report (#14188). diff --git a/changelog.d/fixed-14194-blocked-on-stale-reconciliation.md b/changelog.d/fixed-14194-blocked-on-stale-reconciliation.md new file mode 100644 index 0000000000..d69d34c153 --- /dev/null +++ b/changelog.d/fixed-14194-blocked-on-stale-reconciliation.md @@ -0,0 +1 @@ +- **Blocked-on relations: state check before write, supersede re-targeting, and periodic sweep.** Posting a `hive:blocked-on` comment checks the target PR's state first and refuses to park on an already merged or closed PR (posting a verification note instead); closing a blocker as superseded re-targets relations to the replacement PR (or unblocks when the replacement is already merged); a periodic reconciliation sweep unblocks issues whose blocker merged or closed across cycles (#14194). diff --git a/changelog.d/fixed-14216.md b/changelog.d/fixed-14216.md new file mode 100644 index 0000000000..b31c7840a0 --- /dev/null +++ b/changelog.d/fixed-14216.md @@ -0,0 +1 @@ +- fix(claims): a claim's GitHub side effects and the release of that same claim ran in independent goroutines, so a release that followed a slow claim re-read the issue before the claim's `hive/claimed-by-human` label had landed, skipped it, and the label was then applied onto an already-released issue — orphaned until a person noticed (hivecommons/hive#14065, 2026-10-25). The claim hooks now run one issue's transitions in ledger order; a release only reads the issue after its claim has finished writing. (#14216, fixes #14201) diff --git a/changelog.d/fixed-14239-stale-human-claim-label.md b/changelog.d/fixed-14239-stale-human-claim-label.md new file mode 100644 index 0000000000..492ad9b440 --- /dev/null +++ b/changelog.d/fixed-14239-stale-human-claim-label.md @@ -0,0 +1 @@ +- Agent claims clear a stale `hive/claimed-by-human` label when the latest human claim has a matching release marker, recovering from missed release label writes (#14239). diff --git a/changelog.d/fixed-14240-stage-sweep-cancellation.md b/changelog.d/fixed-14240-stage-sweep-cancellation.md new file mode 100644 index 0000000000..b0836118ee --- /dev/null +++ b/changelog.d/fixed-14240-stage-sweep-cancellation.md @@ -0,0 +1 @@ +- Issue stage-label sweeps stop cleanly when their best-effort budget expires or the request pacer fails fast before the deadline instead of reporting a write failure for every unattempted issue; live-context request failures still warn once and unfinished work retries next cycle. PR sweeps also skip comments and adoption side effects for the unattempted batch tail. diff --git a/changelog.d/fixed-14250-gitlab-nested-project-scope.md b/changelog.d/fixed-14250-gitlab-nested-project-scope.md new file mode 100644 index 0000000000..fd4392241b --- /dev/null +++ b/changelog.d/fixed-14250-gitlab-nested-project-scope.md @@ -0,0 +1 @@ +- Fixed GitLab proxy repo scoping so nested group project paths (`group/sub/repo`) are no longer truncated to `group/sub`, both on `/api/v4/projects/...` and on git smart-HTTP pushes (`/group/sub/repo.git/git-receive-pack`, which previously extracted no repo and skipped the per-repo gates). The per-repo write gates (pause, agent scope, auto-merge, per-repo ACMM mode, repo filter) and the refusal audit entry now read a canonicalized form of the client's escaped path — every segment percent-decoded, with only `%2F` kept as the GitLab project separator — so `%2F`-encoded projects resolve exactly and neither escaped slug letters (`o%72g/repo`) nor escaped structural segments (`/r%65pos/`, `repo%2Egit`, `pro%6Aects`, `merg%65`) can slip past a pause, scope, allowlist or auto-merge entry. Git smart-HTTP slug extraction is also anchored to `info/refs`, `git-upload-pack` and `git-receive-pack`, so a `.git/` elsewhere in an API path no longer yields a bogus slug (#14250) diff --git a/changelog.d/fixed-14296.md b/changelog.d/fixed-14296.md new file mode 100644 index 0000000000..a57a3585c1 --- /dev/null +++ b/changelog.d/fixed-14296.md @@ -0,0 +1 @@ +- `hive-open-pr` no longer folds a new PR into an open bot PR that is held, stalled, needs a rebase or has merge conflicts, since the hive cannot push to it; only the overlapping-files check skips such PRs, and a PR closing the same issue still counts as a duplicate (#14296). diff --git a/changelog.d/fixed-14300.md b/changelog.d/fixed-14300.md new file mode 100644 index 0000000000..ede740387a --- /dev/null +++ b/changelog.d/fixed-14300.md @@ -0,0 +1 @@ +- Dashboard feedback issue refresh: non-JSON (HTML) responses are reported with their HTTP status instead of a bare json decode error, and repeated failures back off instead of warning on every request (#14300). diff --git a/changelog.d/fixed-14307-claude-consent-screen.md b/changelog.d/fixed-14307-claude-consent-screen.md new file mode 100644 index 0000000000..be2e05071d --- /dev/null +++ b/changelog.d/fixed-14307-claude-consent-screen.md @@ -0,0 +1 @@ +- `claude` agents no longer stop on Claude Code's "WARNING: Claude Code running in Bypass Permissions mode" screen with "No, exit" selected. Hive seeds `skipDangerousModePermissionPrompt` into the shared Claude settings that `claude` agents read, so the screen does not appear. If it appears anyway, a kick selects "Yes, I accept" instead of restarting the CLI into the same screen, and the watcher's consent-stuck dismissal now covers `claude` agents as well as inference backends ([#14307](https://github.com/hivecommons/hive/issues/14307)). diff --git a/changelog.d/fixed-14323-wedged-kick-drain-bound.md b/changelog.d/fixed-14323-wedged-kick-drain-bound.md new file mode 100644 index 0000000000..76e1b4c943 --- /dev/null +++ b/changelog.d/fixed-14323-wedged-kick-drain-bound.md @@ -0,0 +1 @@ +- A wedged agent kick can no longer hold a hub-instructed self-upgrade for the full drain window. A kick whose turn-end signal never arrived stayed "in flight" forever, so the drain could never finish and every upgrade waited `hub.upgrade_drain_timeout_s` before it went ahead. Now a kick left unsettled for longer than the agent's `continuous_turn_ceiling` (45m by default, and also when the ceiling is unset or disabled) no longer counts toward the drain. Hive logs one warning per wedged kick with the agent name and kick age, and the drain's deferred, timed-out and complete log lines list the ignored agents as `ignored_wedged` ([#14323](https://github.com/hivecommons/hive/issues/14323)). diff --git a/changelog.d/fixed-14324-gitlab-nested-git-path-scope.md b/changelog.d/fixed-14324-gitlab-nested-git-path-scope.md new file mode 100644 index 0000000000..51b0ab36ef --- /dev/null +++ b/changelog.d/fixed-14324-gitlab-nested-git-path-scope.md @@ -0,0 +1 @@ +- GitLab project paths with a subgroup or project named like a resource word (for example `group/packages/repo`) are documented as ambiguous in the unencoded spelling, with a test pinning the behaviour; nested-group git smart-HTTP pushes are covered by the repo filter tests (#14324, #14321). diff --git a/changelog.d/fixed-14325-jsonresponse-client-disconnect-log.md b/changelog.d/fixed-14325-jsonresponse-client-disconnect-log.md new file mode 100644 index 0000000000..e234a0336f --- /dev/null +++ b/changelog.d/fixed-14325-jsonresponse-client-disconnect-log.md @@ -0,0 +1 @@ +- The dashboard's `jsonResponse`/`jsonError` no longer log a WARN when the client went away mid-response (broken pipe, connection reset, cancelled request, handler timeout); those are logged at DEBUG, and WARN is kept for genuine encode failures ([#14325](https://github.com/hivecommons/hive/issues/14325)). diff --git a/changelog.d/fixed-14327.md b/changelog.d/fixed-14327.md new file mode 100644 index 0000000000..1ebd988f23 --- /dev/null +++ b/changelog.d/fixed-14327.md @@ -0,0 +1 @@ +- fix(agent): only selfreport ci_poll_nudge at 3+ CI polls so the kick's two-status-check allowance is not reported fleet-wide (#14327) diff --git a/changelog.d/fixed-14328-knowledge-primer-cap-log.md b/changelog.d/fixed-14328-knowledge-primer-cap-log.md new file mode 100644 index 0000000000..c485239a68 --- /dev/null +++ b/changelog.d/fixed-14328-knowledge-primer-cap-log.md @@ -0,0 +1 @@ +- The knowledge primer's "dropped facts over the byte cap" notice is now logged at INFO once per agent and cap (again only when the omitted fact count changes) and at DEBUG otherwise, instead of at WARN on every kick; the fleet self-report classifier already treats it as benign ([#14328](https://github.com/hivecommons/hive/issues/14328)). diff --git a/changelog.d/fixed-14341.md b/changelog.d/fixed-14341.md new file mode 100644 index 0000000000..fe2a4a0bbf --- /dev/null +++ b/changelog.d/fixed-14341.md @@ -0,0 +1 @@ +- fix(ci-runners): bump pinned GitHub CLI to 2.103.2 (apt dropped 2.102.0) (#14341) diff --git a/changelog.d/fixed-14344.md b/changelog.d/fixed-14344.md new file mode 100644 index 0000000000..138fdf5a9a --- /dev/null +++ b/changelog.d/fixed-14344.md @@ -0,0 +1 @@ +- An older, retracted `/hive decision` is no longer replayed after an issue is parked again; once a newer decision has been answered, earlier commands are treated as superseded (#14344). diff --git a/changelog.d/fixed-14346-hourly-cap-selfreport.md b/changelog.d/fixed-14346-hourly-cap-selfreport.md new file mode 100644 index 0000000000..8de038b04e --- /dev/null +++ b/changelog.d/fixed-14346-hourly-cap-selfreport.md @@ -0,0 +1 @@ +- Expected per-agent GitHub hourly cap and soft-budget warnings no longer open repeated-error fleet self-reports; quota enforcement and diagnostic logs remain unchanged (#14346). diff --git a/changelog.d/fixed-14351-required-checks-selfreport.md b/changelog.d/fixed-14351-required-checks-selfreport.md new file mode 100644 index 0000000000..6b272529d7 --- /dev/null +++ b/changelog.d/fixed-14351-required-checks-selfreport.md @@ -0,0 +1 @@ +- Stale automerge `required_checks` configuration warnings remain visible in logs without generating repeated runtime-error self-reports; repository protection checks continue to govern merges. diff --git a/changelog.d/fixed-14360-fold-target-vetoes.md b/changelog.d/fixed-14360-fold-target-vetoes.md new file mode 100644 index 0000000000..3ffa3a670b --- /dev/null +++ b/changelog.d/fixed-14360-fold-target-vetoes.md @@ -0,0 +1 @@ +- Issue-request consolidation no longer folds a finding into an open issue parked on a maintainer decision or owned by another `agent/`, and a consolidated result now reports `consolidated_reason` (`subject`, `file_set` or `component:`) (#14360). diff --git a/changelog.d/fixed-14361-review-dispatch-deadlock.md b/changelog.d/fixed-14361-review-dispatch-deadlock.md new file mode 100644 index 0000000000..7745568412 --- /dev/null +++ b/changelog.d/fixed-14361-review-dispatch-deadlock.md @@ -0,0 +1 @@ +- The review swarm no longer deadlocks on verdicts whose `body_digest` or `head_sha` the reviewer miscopied, or when a PR's review runs out of redispatches ([#14361](https://github.com/hivecommons/hive/issues/14361)). The relay now stores every dispatched verdict under the head SHA and body digest its dispatch recorded, ignoring the reviewer's copy. Models had returned a prose summary, a `sha256:`-prefixed hash or a hash of something else. Each such verdict was filed apart from the PR's review, so its perspective never counted as delivered and its pending entry never cleared. Separately, under one review kick in flight per agent (#13794), every pending entry made its agent busy. That included entries an exhausted head deliberately keeps (so it is never re-kicked) and entries already past `pending_ttl` on a PR not yet reached this cycle. Three such PRs pinned every review head and the decision agent, and the swarm dispatched nothing for about 36 hours while every reviewer sat idle. Now only live entries (inside `pending_ttl`, on a head that is not exhausted) count as load. Pending entries naming agents the hive no longer runs (a replica or head removed by lowering `replicas`) are pruned so their perspective is asked again. An exhausted head is labelled with the `requires_human` verdict label instead of staying on `reviewing` indefinitely. Review prompts now tell reviewers to copy `head_sha` and `body_digest` verbatim or leave them out. `review.verdict_labels` fixes: a PR moving to a new state now loses its previous verdict label, even one the hive had already removed once. Before, the second removal was deduplicated as already done, so `hive-review/in-progress` and `hive-review/approved` could both show. A PR that merges or closes while carrying a verdict label now has it removed (at most 25 removals every 10 minutes). diff --git a/changelog.d/fixed-14364-ci-refusal-selfreport.md b/changelog.d/fixed-14364-ci-refusal-selfreport.md new file mode 100644 index 0000000000..b0307911a7 --- /dev/null +++ b/changelog.d/fixed-14364-ci-refusal-selfreport.md @@ -0,0 +1 @@ +- Repeated merge-request CI gate refusals remain visible in operator logs without generating spurious fleet runtime-defect reports (#14364). diff --git a/changelog.d/fixed-14366.md b/changelog.d/fixed-14366.md new file mode 100644 index 0000000000..9663ba543c --- /dev/null +++ b/changelog.d/fixed-14366.md @@ -0,0 +1 @@ +- Pull request creation no longer refuses retries forever after an ambiguous GitHub failure; the unresolved op is reconciled against the open PR for the head branch (#14366) diff --git a/changelog.d/fixed-14368-pack-pause-survives-upgrade.md b/changelog.d/fixed-14368-pack-pause-survives-upgrade.md new file mode 100644 index 0000000000..8e6f8d9d80 --- /dev/null +++ b/changelog.d/fixed-14368-pack-pause-survives-upgrade.md @@ -0,0 +1 @@ +- An agent the ACMM pack paused as "not in pack level N" is no longer resumed by the startup pack apply at that same level when an upgrade ships a pack that now lists it (e.g. `reviewer` joining the L2–L4 packs); it stays paused until the operator resumes it or explicitly re-applies the pack ([#14368](https://github.com/hivecommons/hive/issues/14368)). diff --git a/changelog.d/fixed-14371-headroom-timeout.md b/changelog.d/fixed-14371-headroom-timeout.md new file mode 100644 index 0000000000..aef4446bee --- /dev/null +++ b/changelog.d/fixed-14371-headroom-timeout.md @@ -0,0 +1 @@ +- Classify timed-out Codex headroom probes as timeouts instead of unclassified closed-pipe or missing-response failures. diff --git a/changelog.d/fixed-14372-legacy-label-permission.md b/changelog.d/fixed-14372-legacy-label-permission.md new file mode 100644 index 0000000000..27e0e6beb0 --- /dev/null +++ b/changelog.d/fixed-14372-legacy-label-permission.md @@ -0,0 +1 @@ +- Stop repeatedly retrying optional legacy decision label definition cleanup when GitHub denies the integration permission; retry after the client restarts. diff --git a/changelog.d/fixed-14377-shadow-fence-benign.md b/changelog.d/fixed-14377-shadow-fence-benign.md new file mode 100644 index 0000000000..41bfae7456 --- /dev/null +++ b/changelog.d/fixed-14377-shadow-fence-benign.md @@ -0,0 +1 @@ +- A burst of shadow-mode "shadow fence would have denied external mutation" warnings no longer files repeated runtime-error self-reports; the warning stays in logs and enforce-mode denials remain reportable. diff --git a/changelog.d/fixed-14384-ci-poll-evidence.md b/changelog.d/fixed-14384-ci-poll-evidence.md new file mode 100644 index 0000000000..6cc7b9addb --- /dev/null +++ b/changelog.d/fixed-14384-ci-poll-evidence.md @@ -0,0 +1 @@ +- CI-poll self-reports now describe observed repeated CI-status commands and distinguish repeated reads from total reads, rather than claiming the agent was polling its own PR when PR ownership is unknown (#14384). diff --git a/changelog.d/fixed-14388-review-fetch-outage.md b/changelog.d/fixed-14388-review-fetch-outage.md new file mode 100644 index 0000000000..6a3920b515 --- /dev/null +++ b/changelog.d/fixed-14388-review-fetch-outage.md @@ -0,0 +1 @@ +- Transient GitHub failures fetching PR review decisions no longer trigger repeated unclassified fleet defect reports; review decisions remain unknown until a successful poll. diff --git a/changelog.d/fixed-14389-startup-probe-backfill.md b/changelog.d/fixed-14389-startup-probe-backfill.md new file mode 100644 index 0000000000..c38ff9f5a1 --- /dev/null +++ b/changelog.d/fixed-14389-startup-probe-backfill.md @@ -0,0 +1 @@ +- Spoke self-upgrades now raise stale startup probe thresholds to the hosted 10-minute budget in the same rollout, preventing older deployments from retaining the 160-second limit after upgrades (#14389). diff --git a/changelog.d/fixed-14390.md b/changelog.d/fixed-14390.md new file mode 100644 index 0000000000..1b4a94f7dd --- /dev/null +++ b/changelog.d/fixed-14390.md @@ -0,0 +1 @@ +- fix(entrypoint): keep the .codex/.gemini re-own off PID 1's startup path (#14390) diff --git a/changelog.d/fixed-14406-dirname-stub.md b/changelog.d/fixed-14406-dirname-stub.md new file mode 100644 index 0000000000..f59ca5efda --- /dev/null +++ b/changelog.d/fixed-14406-dirname-stub.md @@ -0,0 +1 @@ +- hive-open-pr.sh stub-PATH test now provides dirname, which the script's SCRIPT_DIR line requires, so it passes on runner images that do not resolve it implicitly ([#14406](https://github.com/hivecommons/hive/issues/14406)). diff --git a/changelog.d/fixed-14409-chat-agents-intent.md b/changelog.d/fixed-14409-chat-agents-intent.md new file mode 100644 index 0000000000..9216023f45 --- /dev/null +++ b/changelog.d/fixed-14409-chat-agents-intent.md @@ -0,0 +1 @@ +- Hive Chat now answers agent questions instead of dumping the roster (#14409). `/agents stuck`, `/agents scanner status` and "Which agent completed last?" all used to return the identical 16-line agent list with a bare `BLOCKED` label and no reason. `/agents stuck|blocked|paused|running|idle|failed` now filter the roster, blocked agents carry the hive's recorded WHY (rate-limit cap, inference auth, policy stand-down, crash loop, failed starts) and since-when, `/agents [status|why|last|history]` / `/agents status ` / "scanner status" return only that agent's card (state, block reason, current task, last completed turn and outcome, last kick, last PR/issue, restarts; unknown names list the valid agents), and natural-language questions — "which agent completed last", "what is scanner doing", "why is reviewer-2 blocked", "when did scanner last run", "which agents are idle" — go through a table-driven intent matcher. Unmatched agent questions reach the configured chat responder with the agent state attached rather than the roster. Bare `/agents` keeps the roster but appends the block reason inline. The reason and timestamp are exposed on `/api/status` `agents[]` as `blockedReason` / `blockedSince` (plus `kickOutcomeAt`) so chat and the Agents page agree. diff --git a/changelog.d/fixed-14418-self-upgrade-deployment-read.md b/changelog.d/fixed-14418-self-upgrade-deployment-read.md new file mode 100644 index 0000000000..3db9d6d33d --- /dev/null +++ b/changelog.d/fixed-14418-self-upgrade-deployment-read.md @@ -0,0 +1 @@ +- Spoke image self-upgrades reuse a single Deployment read for container images and startup probe backfill, avoiding a redundant Kubernetes API GET ([#14418](https://github.com/hivecommons/hive/issues/14418)). diff --git a/changelog.d/fixed-14419-spoke-probe-backfill.md b/changelog.d/fixed-14419-spoke-probe-backfill.md new file mode 100644 index 0000000000..7c9742b4e8 --- /dev/null +++ b/changelog.d/fixed-14419-spoke-probe-backfill.md @@ -0,0 +1 @@ +- Backfill stale spoke startup probes when the Deployment already matches the requested image, without repeatedly restarting an in-flight rollout. diff --git a/changelog.d/fixed-14420-local-pr-close.md b/changelog.d/fixed-14420-local-pr-close.md new file mode 100644 index 0000000000..333db10ef1 --- /dev/null +++ b/changelog.d/fixed-14420-local-pr-close.md @@ -0,0 +1,2 @@ +- Immediately remove PRs merged or closed by the hive from the stage tracker and governor snapshot, without waiting for a webhook or sweep ([#14420](https://github.com/hivecommons/hive/issues/14420)). +- Retry the pinned Pi coding-agent npm install in `Dockerfile.contributor` up to three times with a 30s delay, so a registry tarball that briefly 404s while a release propagates no longer fails the contributor image build. diff --git a/changelog.d/fixed-14422-kick-held-selfreport.md b/changelog.d/fixed-14422-kick-held-selfreport.md new file mode 100644 index 0000000000..ddcd4ab428 --- /dev/null +++ b/changelog.d/fixed-14422-kick-held-selfreport.md @@ -0,0 +1 @@ +- Stop reporting the expected post-restart kick hold (`failed to send kick` with `kick held after restart`, #7363) as an `unclassified_error_repeated` self-report; other kick send failures still report (#14422). diff --git a/changelog.d/fixed-14425-self-managed-startup-budget.md b/changelog.d/fixed-14425-self-managed-startup-budget.md new file mode 100644 index 0000000000..3926517b9e --- /dev/null +++ b/changelog.d/fixed-14425-self-managed-startup-budget.md @@ -0,0 +1 @@ +- Self-managed Kubernetes deployments and the manual provisioning template now allow the same ten-minute startup budget as hosted spokes, avoiding premature restarts during slow first boots. diff --git a/changelog.d/fixed-14437-prs-changes-requested.md b/changelog.d/fixed-14437-prs-changes-requested.md new file mode 100644 index 0000000000..848937536e --- /dev/null +++ b/changelog.d/fixed-14437-prs-changes-requested.md @@ -0,0 +1 @@ +- Hive Chat `/prs waiting on review` no longer lists pull requests whose review decision is changes requested; those wait on their author, not on a reviewer ([#14437](https://github.com/hivecommons/hive/issues/14437)). diff --git a/changelog.d/fixed-14464.md b/changelog.d/fixed-14464.md new file mode 100644 index 0000000000..eac3c66f7b --- /dev/null +++ b/changelog.d/fixed-14464.md @@ -0,0 +1 @@ +- cmd/hive: the sidecar supervision boot test no longer hangs the test shard; waits are bounded. diff --git a/changelog.d/fixed-14466.md b/changelog.d/fixed-14466.md new file mode 100644 index 0000000000..8efb2fe226 --- /dev/null +++ b/changelog.d/fixed-14466.md @@ -0,0 +1 @@ +- The adopted-PR stall escalation to `stage/085-needs-decision` ("adopted PR sits with nothing the hive can push") is now logged at INFO instead of WARN. It is designed behaviour (one stall note per PR), and the WARN made the `unclassified_error_repeated` classifier file fleet self-reports when several contributor PRs stalled together ([#14466](https://github.com/hivecommons/hive/issues/14466)). diff --git a/changelog.d/fixed-14468.md b/changelog.d/fixed-14468.md new file mode 100644 index 0000000000..72f551081b --- /dev/null +++ b/changelog.d/fixed-14468.md @@ -0,0 +1 @@ +- crux: the Makefile test note no longer points at a non-existent AGENTS.md; the tmux rule is stated inline with a link to hive's AGENTS.md ([#14468](https://github.com/hivecommons/hive/issues/14468)). diff --git a/changelog.d/fixed-14473-review-backlog-scope.md b/changelog.d/fixed-14473-review-backlog-scope.md new file mode 100644 index 0000000000..6b26e04470 --- /dev/null +++ b/changelog.d/fixed-14473-review-backlog-scope.md @@ -0,0 +1 @@ +- Review backlog findings in files untouched by the reviewed PR are actionable immediately instead of waiting for that PR to close or merge. diff --git a/changelog.d/fixed-14478.md b/changelog.d/fixed-14478.md new file mode 100644 index 0000000000..95e0bca2f2 --- /dev/null +++ b/changelog.d/fixed-14478.md @@ -0,0 +1 @@ +- A relay resuming a task the hub holds no lease for (e.g. after a hub restart or lease expiry) is still told to re-ready, but the rejection is no longer logged as a repeated warning ([#14478](https://github.com/hivecommons/hive/issues/14478)). diff --git a/changelog.d/fixed-14500-adopted-stale-red-recovery.md b/changelog.d/fixed-14500-adopted-stale-red-recovery.md new file mode 100644 index 0000000000..1c0cff3dbc --- /dev/null +++ b/changelog.d/fixed-14500-adopted-stale-red-recovery.md @@ -0,0 +1 @@ +- Adopted contributor PRs that were red only because of a shared-CI or base-branch flake fixed on the base since — their heads 15–22 commits behind `v5` with CI never re-run (hivecommons/hive#14430, #14440, #14456 after #14406 was fixed by #14415) — were escalated to `stage/085-needs-decision` as "nothing the hive can push". The 074 escape now tries one recovery first when the head's failing check is green on the base tip: it merges the base into the head (GitHub update-branch, also on a maintainer-editable fork) when the head is behind, or re-runs the failed workflow jobs when it is current, at most once per (head, base tip) and three times per adoption, recorded on the stage record; an escalation already in place for that head is released, and 085 waits until the attempt was made and the head is still red. One `adopted PR recovery` log line names the outcome (`update-branch`, `rerun-failed-jobs`, `skipped:`) (hivecommons/hive#14500). diff --git a/changelog.d/fixed-14509-classifier-runtime-alert.md b/changelog.d/fixed-14509-classifier-runtime-alert.md new file mode 100644 index 0000000000..c973001e0b --- /dev/null +++ b/changelog.d/fixed-14509-classifier-runtime-alert.md @@ -0,0 +1 @@ +- Report runtime semantic classifier failures that withhold kicks in the dashboard ([#14509](https://github.com/hivecommons/hive/issues/14509)). diff --git a/changelog.d/fixed-14515.md b/changelog.d/fixed-14515.md new file mode 100644 index 0000000000..db2211fbd1 --- /dev/null +++ b/changelog.d/fixed-14515.md @@ -0,0 +1 @@ +- fix(agent): document pacing-seam override rule (#14515) diff --git a/changelog.d/fixed-14526.md b/changelog.d/fixed-14526.md new file mode 100644 index 0000000000..8122e05526 --- /dev/null +++ b/changelog.d/fixed-14526.md @@ -0,0 +1 @@ +- Fleet report: the pr-request watcher's repeated `head branch is not on the remote` ERROR is now classified as `pr_request_head_unpushed` (medium, component `github-relay`) instead of `unclassified_error_repeated` ([#14526](https://github.com/hivecommons/hive/issues/14526)). diff --git a/changelog.d/fixed-14537-held-pr-expected-head.md b/changelog.d/fixed-14537-held-pr-expected-head.md new file mode 100644 index 0000000000..9b37781344 --- /dev/null +++ b/changelog.d/fixed-14537-held-pr-expected-head.md @@ -0,0 +1 @@ +- The hold guard's merge-from-base sync for held PRs is now pinned to the head it enumerated (`expected_head_sha`), so a contributor push landing mid-tick makes GitHub refuse the update instead of stacking the hive's base merge on unreviewed commits. A stale-head refusal is logged and skipped without a conflict notice, leaving the guard's baseline in place for the next drift check ([#14537](https://github.com/hivecommons/hive/issues/14537)). diff --git a/changelog.d/fixed-14578-sentinel-gate-live-label.md b/changelog.d/fixed-14578-sentinel-gate-live-label.md new file mode 100644 index 0000000000..647ae93767 --- /dev/null +++ b/changelog.d/fixed-14578-sentinel-gate-live-label.md @@ -0,0 +1 @@ +- The activity-monitor merge gate now follows the live `sentinel-alert` label instead of the head's append-only sentinel evidence, so removing the label clears the "Sentinel alert · needs human" pill on the next tick; linked issues resolve their title and URL when the configured repo is a short name; and a trusted merger's approving review on a manual-merge lane (workflow path, tests removed, human-merge path) now routes the PR to `stage/090-ready-for-human-merge` with an "Awaiting human merge — approved by @login; click Merge" caption and comment instead of re-posting an open-ended needs-decision ([#14578](https://github.com/hivecommons/hive/issues/14578)). diff --git a/changelog.d/fixed-adoption-rebase-fallback.md b/changelog.d/fixed-adoption-rebase-fallback.md new file mode 100644 index 0000000000..be2edb3a73 --- /dev/null +++ b/changelog.d/fixed-adoption-rebase-fallback.md @@ -0,0 +1 @@ +- Adopted contributor PRs now supersede automatically when confirmed push-access loss or squash sign-off attribution prevents in-place repair (not on API errors or unrelated base workflow drift), and idle rebase work falls through to a hive-owned branch instead of parking for a human; fixer instructions preserve contributor authorship and sign-offs (#13975). diff --git a/changelog.d/fixed-adoption-settle-any-stage.md b/changelog.d/fixed-adoption-settle-any-stage.md new file mode 100644 index 0000000000..66e088a2f1 --- /dev/null +++ b/changelog.d/fixed-adoption-settle-any-stage.md @@ -0,0 +1 @@ +- Adopted fork PRs are now closed (with a note linking the superseding PR) when the superseder merges, regardless of which stage the original has since moved to; when hive cannot close the original it is marked superseded so the activity monitor drops the row (#14471). diff --git a/changelog.d/fixed-adoption-skips-repo-maintainers.md b/changelog.d/fixed-adoption-skips-repo-maintainers.md index d3bfca9e77..ec83bb6e75 100644 --- a/changelog.d/fixed-adoption-skips-repo-maintainers.md +++ b/changelog.d/fixed-adoption-skips-repo-maintainers.md @@ -1 +1 @@ -- Contributor adoption (stage `074` and its re-home / DCO re-home) no longer takes over a PR whose author has admin or maintain permission on the repository: a repo admin's PR was adopted as "contributor-offline" 16 minutes after its nudge and then closed in favour of a hive copy. The permission is read once per repository and author per hour, a failed read does not adopt, and an existing in-place adoption of such an author is released. +- Contributor adoption (stage `074` and its re-home / DCO re-home) no longer takes over a PR whose author has admin or maintain permission on the repository (backport of [#14305](https://github.com/hivecommons/hive/pull/14305), [#14301](https://github.com/hivecommons/hive/issues/14301)). The permission is read once per repository and author per 10 minutes; a failed read is not cached and defers, neither adopting nor releasing; an existing adoption of such an author is released, closing a still-open hive superseder with a note first. diff --git a/changelog.d/fixed-api-hourly-cap-selfreport.md b/changelog.d/fixed-api-hourly-cap-selfreport.md new file mode 100644 index 0000000000..5b713e3d9d --- /dev/null +++ b/changelog.d/fixed-api-hourly-cap-selfreport.md @@ -0,0 +1 @@ +- Agent GitHub API hourly cap now warns once per agent per window instead of on every rejected request (#14118) diff --git a/changelog.d/fixed-app-authored-lane-attribution.md b/changelog.d/fixed-app-authored-lane-attribution.md new file mode 100644 index 0000000000..bb449826e0 --- /dev/null +++ b/changelog.d/fixed-app-authored-lane-attribution.md @@ -0,0 +1 @@ +- **Trusted-author tier hands App-authored PRs off under its own reason** — the sweep logged `reason=contributor-lane:app-authored tier=trusted-author` and recorded a trusted-author stage observation carrying the contributor lane's reason, which read as if the contributor lane had routed the App's own PRs. The tier now skips them as `trusted-author:app-authored`; `contributor-lane:app-authored` is emitted by the contributor lane only. Stage derivation treats both as no-opinion hand-offs, so the self-authored lane's `review-gate:no-review-bundle` keeps owning the stage; merge gating is unchanged ([#14482](https://github.com/hivecommons/hive/issues/14482)). diff --git a/changelog.d/fixed-chat-grounded-answers.md b/changelog.d/fixed-chat-grounded-answers.md new file mode 100644 index 0000000000..10652b41b5 --- /dev/null +++ b/changelog.d/fixed-chat-grounded-answers.md @@ -0,0 +1 @@ +- Hive Chat now answers the question it was asked instead of a canned dump. The server used to tokenise every message into a bag of words and match on keywords alone, so `/agents stuck`, `/agents scanner status` and "Which agent completed last?" all returned the full agent roster and `/prs waiting on review` returned per-repo PR counts, with the arguments silently discarded. A new router keeps the arguments: a bare `/agents`, `/prs`, `/beads` or `/status` still prints its summary, while a command with arguments or a plain-words question is answered by one read-only model turn over a compact, scoped snapshot of the hive state the dashboard already holds (agents with blockers, last kick and last completed action; open PRs with stage labels, review state, CI and age; actionable issues; items awaiting a decision; governor; alerts; version and last upgrade), instructed to say "I don't have that data" rather than guess and to cite the items it used. The turn runs on the governor reviewer endpoint with a timeout and in-flight cap, writes one `chat_grounded_answer` audit line, and can never trigger an action. When grounding is unavailable a scoped question gets the command's usage, never the bare dump. New setting `chat.grounded_answers.enabled` (default on) and `chat.grounded_answers.model`. diff --git a/changelog.d/fixed-ci-poll-kick-baseline.md b/changelog.d/fixed-ci-poll-kick-baseline.md new file mode 100644 index 0000000000..f962a5fce9 --- /dev/null +++ b/changelog.d/fixed-ci-poll-kick-baseline.md @@ -0,0 +1 @@ +- Prevent repeated scanner sweeps from being reported as CI polling by counting repeated reads within each kick, excluding earlier kicks' reads ([#14061](https://github.com/hivecommons/hive/issues/14061)). diff --git a/changelog.d/fixed-ci-poll-log-diagnosis.md b/changelog.d/fixed-ci-poll-log-diagnosis.md new file mode 100644 index 0000000000..f08c4c538b --- /dev/null +++ b/changelog.d/fixed-ci-poll-log-diagnosis.md @@ -0,0 +1 @@ +- CI failure log reads (`gh run view --log` and `--log-failed`) no longer trigger stop-polling nudges or fleet reports while agents diagnose failed jobs. diff --git a/changelog.d/fixed-ci-poll-wall-clock-sweep.md b/changelog.d/fixed-ci-poll-wall-clock-sweep.md new file mode 100644 index 0000000000..8d81df557e --- /dev/null +++ b/changelog.d/fixed-ci-poll-wall-clock-sweep.md @@ -0,0 +1 @@ +- Stop the CI-poll wall-clock nudge from interrupting a long scanner sweep that reads each PR once; a lone poll or a repeated read past the wall clock is still nudged ([#14066](https://github.com/hivecommons/hive/issues/14066)). diff --git a/changelog.d/fixed-claim-ledger-on-pr-open.md b/changelog.d/fixed-claim-ledger-on-pr-open.md new file mode 100644 index 0000000000..44e57bd549 --- /dev/null +++ b/changelog.d/fixed-claim-ledger-on-pr-open.md @@ -0,0 +1 @@ +- A PR hive opens now records its issue claims in the duplicate-PR claim ledger immediately, so a contributor can no longer be admitted to an issue that PR already covers in the window before the next claim scan (#14472). diff --git a/changelog.d/fixed-close-on-merge-cross-repo-pr.md b/changelog.d/fixed-close-on-merge-cross-repo-pr.md new file mode 100644 index 0000000000..380017cd3f --- /dev/null +++ b/changelog.d/fixed-close-on-merge-cross-repo-pr.md @@ -0,0 +1 @@ +- Close-on-merge backfill now reads a pull request that cross-references an issue from another repo in that PR's own repo, instead of looking up the same number in the issue's repo and logging a 404 warning every pass. diff --git a/changelog.d/fixed-contribution-feedback.md b/changelog.d/fixed-contribution-feedback.md new file mode 100644 index 0000000000..217892ab35 --- /dev/null +++ b/changelog.d/fixed-contribution-feedback.md @@ -0,0 +1 @@ +- The Commons contribution editor reports partial staging failures honestly, prevents overlapping batch actions, and explains runtime uncertainty, offering status, selection provenance and exclusion recovery. diff --git a/changelog.d/fixed-contribution-tui-review-viewport.md b/changelog.d/fixed-contribution-tui-review-viewport.md new file mode 100644 index 0000000000..c5f389c197 --- /dev/null +++ b/changelog.d/fixed-contribution-tui-review-viewport.md @@ -0,0 +1 @@ +- Keep TUI contribution reviews within terminal height, with wrapped target identities, keyboard scrolling, persistent Apply/Back hints, and Apply paused when the terminal is too small to review. diff --git a/changelog.d/fixed-contributor-bob-2.md b/changelog.d/fixed-contributor-bob-2.md new file mode 100644 index 0000000000..12999d95a1 --- /dev/null +++ b/changelog.d/fixed-contributor-bob-2.md @@ -0,0 +1 @@ +- Contributor images now select Bob 1.x/2.x launch flags at runtime and accept both `BOB_API_KEY` and legacy `BOBSHELL_API_KEY` credentials, matching hub-side agents (#14334). diff --git a/changelog.d/fixed-contributor-tier-limit-selfreports.md b/changelog.d/fixed-contributor-tier-limit-selfreports.md new file mode 100644 index 0000000000..3ac21e808d --- /dev/null +++ b/changelog.d/fixed-contributor-tier-limit-selfreports.md @@ -0,0 +1 @@ +- Expected contributor concurrency, hourly, and daily tier-limit refusals no longer generate repeated runtime-error self-reports; the warnings and task limits remain intact (#14414). diff --git a/changelog.d/fixed-crux-publish-module-doc-links.md b/changelog.d/fixed-crux-publish-module-doc-links.md new file mode 100644 index 0000000000..80f7528310 --- /dev/null +++ b/changelog.d/fixed-crux-publish-module-doc-links.md @@ -0,0 +1 @@ +- `crux-publish.sh` now rewrites relative `../docs/` links in the published `docs/MODULE.md` to absolute hive URLs pinned to the synced hive commit, fixing dead links in hivecommons/crux. diff --git a/changelog.d/fixed-crux-pushbroker-finished-at.md b/changelog.d/fixed-crux-pushbroker-finished-at.md new file mode 100644 index 0000000000..942e2d15aa --- /dev/null +++ b/changelog.d/fixed-crux-pushbroker-finished-at.md @@ -0,0 +1 @@ +- crux/pushbroker: successful pushes now record `finished_at` in the audit result (previously only failures did) ([#14529](https://github.com/hivecommons/hive/issues/14529)). diff --git a/changelog.d/fixed-crux-serve-loopback.md b/changelog.d/fixed-crux-serve-loopback.md new file mode 100644 index 0000000000..e63e874572 --- /dev/null +++ b/changelog.d/fixed-crux-serve-loopback.md @@ -0,0 +1 @@ +- `crux serve --listen` now refuses a non-loopback bind (`0.0.0.0`, `::`, `:port`, or a remote address) even when a bearer token is set, and `crux/api.ParseAddress` refuses `http://` or bare `host:port` for a non-loopback host: the TCP listener is plain http, so the token would otherwise cross the network in cleartext. Expose the API remotely through a TLS-terminating proxy that dials the loopback listener, or reach it with `https://`. The shared rule is `api.LoopbackHost`. diff --git a/changelog.d/fixed-delegated-supersede-adoption.md b/changelog.d/fixed-delegated-supersede-adoption.md new file mode 100644 index 0000000000..91be1a4725 --- /dev/null +++ b/changelog.d/fixed-delegated-supersede-adoption.md @@ -0,0 +1 @@ +- Delegated decisions to supersede an offline contributor PR now queue the existing adoption engine immediately, preserving contributor commits, sign-off, credit, and issue closing references. diff --git a/changelog.d/fixed-evidence-reopen-card.md b/changelog.d/fixed-evidence-reopen-card.md new file mode 100644 index 0000000000..8655feffab --- /dev/null +++ b/changelog.d/fixed-evidence-reopen-card.md @@ -0,0 +1 @@ +- Evidence: the "Issue reopen rate · rolling 30 days" card now uses the shared panel chrome (collapse toggle, ⓘ explainer, n= chip of distinct issues closed, grade pill) and a compact, overflow-contained table instead of unstyled browser defaults that bled into the next card. diff --git a/changelog.d/fixed-hive-merge-evicts-tracker-row.md b/changelog.d/fixed-hive-merge-evicts-tracker-row.md new file mode 100644 index 0000000000..32b53b70a3 --- /dev/null +++ b/changelog.d/fixed-hive-merge-evicts-tracker-row.md @@ -0,0 +1 @@ +- Fixed the stage tracker and the governor's cached actionable snapshot keeping a pull request the hive had merged or closed itself listed as open until GitHub's `closed` webhook came back round through the hub, the closed-row sweep ran on the next status publish, or the next full enumeration replaced the snapshot. On a hosted hive in webhook cadence hivecommons/docs#469 was merged by the automerge sweep at 18:25 and still read "Sentinel alert - needs human" an hour later, with the governor re-evaluating it as merge-eligible seven more times from the cached snapshot while the hub relay was dropping deliveries. Every hive merge path records a `pr_merged` / `pr_closed` audit event; that record now applies the same closed snapshot a webhook delivery would, so the row leaves both stores the moment the hive performs the write, with no GitHub call and no dependence on the webhook round-trip (hivecommons/hive#14420). diff --git a/changelog.d/fixed-hub-robots-noindex.md b/changelog.d/fixed-hub-robots-noindex.md new file mode 100644 index 0000000000..73aa1637e9 --- /dev/null +++ b/changelog.d/fixed-hub-robots-noindex.md @@ -0,0 +1 @@ +- Search engines no longer crawl or index the hub's auth-gated and API routes. Google Search Console had been reporting hub pages as "Blocked due to unauthorized request (401)" / "access forbidden (403)" because `robots.txt` said `Allow: /` and hosted hives' `/robots.txt` 302'd to the hub login. `GET /robots.txt` on the public hub now disallows every gated route group (`/api/`, `/login`, `/dashboard`, `/metrics`, `/openapi.json`, …) while keeping the landing pages and sitemap crawlable, any other host gets `Disallow: /`, every 401/403/login-redirect and every `/api/*` response carries `X-Robots-Tag: noindex, nofollow`, the login pages carry a `noindex` meta tag, and hosted spoke dashboards serve `/robots.txt` (`Disallow: /`) ahead of the auth gate and mark all their responses `noindex`. diff --git a/changelog.d/fixed-ioscan-held-pr-kick-repeat.md b/changelog.d/fixed-ioscan-held-pr-kick-repeat.md new file mode 100644 index 0000000000..0287c2153e --- /dev/null +++ b/changelog.d/fixed-ioscan-held-pr-kick-repeat.md @@ -0,0 +1 @@ +- An agent whose held-PR coordination kick is blocked by ioscan fail-closed now logs the WARN once per agent and demotes repeats to Debug, matching the main fail-closed kick path, instead of warning on every tick and tripping an `unclassified_error_repeated` runtime self-report ([#14442](https://github.com/hivecommons/hive/issues/14442)). diff --git a/changelog.d/fixed-issue-request-no-consolidate.md b/changelog.d/fixed-issue-request-no-consolidate.md new file mode 100644 index 0000000000..da940fe3e4 --- /dev/null +++ b/changelog.d/fixed-issue-request-no-consolidate.md @@ -0,0 +1 @@ +- `hive-open-issue --no-consolidate ` lets a caller file a separate issue instead of being folded into an open one; the watcher records the reason as `consolidation_skipped_reason` (#14360) diff --git a/changelog.d/fixed-mergepr-forge-audit-path.md b/changelog.d/fixed-mergepr-forge-audit-path.md new file mode 100644 index 0000000000..1fff4b5d2e --- /dev/null +++ b/changelog.d/fixed-mergepr-forge-audit-path.md @@ -0,0 +1 @@ +- Merges the GitHub forge adapter performs are now audited as `path=forge` instead of being attributed to the merge relay (`path=relay`), and the dashboard's PR throughput counts them as hive merges. diff --git a/changelog.d/fixed-pr-request-missing-head-report.md b/changelog.d/fixed-pr-request-missing-head-report.md new file mode 100644 index 0000000000..55c803b58d --- /dev/null +++ b/changelog.d/fixed-pr-request-missing-head-report.md @@ -0,0 +1 @@ +- Avoid duplicate unclassified fleet reports when a PR request head branch has not been pushed. diff --git a/changelog.d/fixed-primer-bytecap-selfreport.md b/changelog.d/fixed-primer-bytecap-selfreport.md new file mode 100644 index 0000000000..58c60a21de --- /dev/null +++ b/changelog.d/fixed-primer-bytecap-selfreport.md @@ -0,0 +1 @@ +- fleet report: the knowledge primer's byte-cap warning is an operator configuration signal, not a repeated runtime defect, and no longer files an `unclassified_error_repeated` self-report (#14173). diff --git a/changelog.d/fixed-proxy-blocked-log-noise-14204.md b/changelog.d/fixed-proxy-blocked-log-noise-14204.md new file mode 100644 index 0000000000..d9b56c6f10 --- /dev/null +++ b/changelog.d/fixed-proxy-blocked-log-noise-14204.md @@ -0,0 +1 @@ +- Repeated blocked proxy requests (e.g. Copilot-internal endpoints) are now logged once at WARN instead of on every attempt, ending false-positive self-reports (#14204) diff --git a/changelog.d/fixed-regressed-badge-only-on-real-regressions.md b/changelog.d/fixed-regressed-badge-only-on-real-regressions.md new file mode 100644 index 0000000000..5decb84698 --- /dev/null +++ b/changelog.d/fixed-regressed-badge-only-on-real-regressions.md @@ -0,0 +1 @@ +- The stage tracker's `regressed` chip now fires only on a genuine fall: a lapsed claim (020 → 010/015), the claim's PR-evidence flicker (030 → 020), the agent returning to its covering PR (040 → 030) and a 040 → 010 after the covering PR merged partially are no longer booked as regressions; the marker clears on the first forward hop instead of waiting for the item to climb back to the stage it fell from; and on a hive-driven row with nothing waiting on a person it draws as a muted `stage ↓` pill (tooltip: the hive is handling it) outside **Needs attention**, keeping red for human-gated rows ([#14570](https://github.com/hivecommons/hive/issues/14570)). diff --git a/changelog.d/fixed-released-human-claim-label.md b/changelog.d/fixed-released-human-claim-label.md new file mode 100644 index 0000000000..f3cd977a1d --- /dev/null +++ b/changelog.d/fixed-released-human-claim-label.md @@ -0,0 +1 @@ +- New agent claims clear stale `hive/claimed-by-human` labels when trusted claim and release comments confirm the prior human claim was released ([#14231](https://github.com/hivecommons/hive/issues/14231)). diff --git a/changelog.d/fixed-review-backlog-fold-criteria.md b/changelog.d/fixed-review-backlog-fold-criteria.md new file mode 100644 index 0000000000..eb37ef9dc2 --- /dev/null +++ b/changelog.d/fixed-review-backlog-fold-criteria.md @@ -0,0 +1 @@ +- The review backlog no longer folds a finding into an open issue just because both cite the same file: it folds only when the category (review perspective) matches and the finding cites the same symbol or a nearby line; otherwise it files a new issue cross-linked with "Same file as #N" ([#13906](https://github.com/hivecommons/hive/issues/13906)). The same check now applies to an open issue matched only by its "Review backlog ()" title prefix, and scanner-filed issues without a "Review backlog (…)" title or "Perspective:" line no longer receive folded review findings (retiring the [#11578](https://github.com/hivecommons/hive/issues/11578) behaviour). diff --git a/changelog.d/fixed-review-evidence-cross-process-lock.md b/changelog.d/fixed-review-evidence-cross-process-lock.md new file mode 100644 index 0000000000..f6ed090cba --- /dev/null +++ b/changelog.d/fixed-review-evidence-cross-process-lock.md @@ -0,0 +1 @@ +- review evidence: upserts to a PR's evidence bundle now take a cross-process file lock so hub and agent writers no longer lose verdicts (#13911). diff --git a/changelog.d/fixed-rootless-probe-storage.md b/changelog.d/fixed-rootless-probe-storage.md new file mode 100644 index 0000000000..723c614b26 --- /dev/null +++ b/changelog.d/fixed-rootless-probe-storage.md @@ -0,0 +1 @@ +- The rootless Podman qualification probe now keeps its temporary image store under `/var/tmp` to avoid quota-limited `/tmp` storage and preserves image-pull diagnostics when a prerequisite fails. diff --git a/changelog.d/fixed-scoped-token-installation-unset.md b/changelog.d/fixed-scoped-token-installation-unset.md new file mode 100644 index 0000000000..8a366229bd --- /dev/null +++ b/changelog.d/fixed-scoped-token-installation-unset.md @@ -0,0 +1 @@ +- Per-agent scoped-token mint with no GitHub App installation ID now logs one actionable warning per agent instead of every tick, and is reported as the `github_app_not_installed` fleet-report class rather than `unclassified_error_repeated` (#14186) diff --git a/changelog.d/fixed-self-upgrade-admission-fallback.md b/changelog.d/fixed-self-upgrade-admission-fallback.md new file mode 100644 index 0000000000..9f14fbaef8 --- /dev/null +++ b/changelog.d/fixed-self-upgrade-admission-fallback.md @@ -0,0 +1 @@ +- A spoke whose self-upgrade `PATCH` on its own Deployment is rejected or timed out by a cluster admission webhook (a failing Kyverno webhook on vllm-d blocked every patch) no longer reports the upgrade as **failed** when a plain restart will land it anyway. The patch deadline is raised from 10s to 45s (above the 10s webhook default, so the apiserver can return the real webhook error) and retried once; when the patch still fails and the Deployment tracks a floating tag that the kubelet re-pulls (`imagePullPolicy: Always` or unknown), the spoke exits immediately with a distinct exit code and reports **pending-restart**, so the hub keeps the in-flight latch instead of counting a failed attempt. The hub's stuck-upgrade alert and the orphan-sweep give-up message now carry the spoke's classified cause ("cluster admission webhook (e.g. Kyverno) is rejecting or timing out Deployment patches — a cluster admin must fix or exempt the webhook") instead of the misleading "Deployment tracks a floating tag" guess ([#14385](https://github.com/hivecommons/hive/issues/14385)). diff --git a/changelog.d/fixed-selfreport-api-noise-14191.md b/changelog.d/fixed-selfreport-api-noise-14191.md new file mode 100644 index 0000000000..7949eb2504 --- /dev/null +++ b/changelog.d/fixed-selfreport-api-noise-14191.md @@ -0,0 +1 @@ +- The hub no longer re-requests a channel commit date from GitHub on every render after a failed lookup (e.g. HTTP 403 rate limit); it backs off for 10 minutes per commit, so rate-limited hubs stop repeating the "channel commit date: fetch failed" warning. diff --git a/changelog.d/fixed-selfreport-ci-poll-nudge-commands.md b/changelog.d/fixed-selfreport-ci-poll-nudge-commands.md new file mode 100644 index 0000000000..62efb67bf1 --- /dev/null +++ b/changelog.d/fixed-selfreport-ci-poll-nudge-commands.md @@ -0,0 +1 @@ +- The `ci_poll_nudge` self-report now lists the repeated CI-status commands (with counts) in its diagnostics, so the fleet issue is triageable instead of showing only an idle pane summary (#14289). diff --git a/changelog.d/fixed-selfreport-classify-login-ioscan-startup-kick.md b/changelog.d/fixed-selfreport-classify-login-ioscan-startup-kick.md new file mode 100644 index 0000000000..d8ff75beed --- /dev/null +++ b/changelog.d/fixed-selfreport-classify-login-ioscan-startup-kick.md @@ -0,0 +1,2 @@ +- Classify the login-required, ioscan classifier fail-closed and startup-kick-dropped WARNs as named fleet-report classes instead of `unclassified_error_repeated` (#14262, #14263, #14266) +- Classify transient GitHub-fetch, label-provisioning and agent-CLI restart-loop WARNs as named fleet-report classes (`github_fetch_transient`, `label_provisioning_failed`, `agent_cli_restart_loop`), report `ioscan_classifier_unavailable` on first observation, and tighten the `agent_pane_timeout` keywords (#14258, #14259, #14260, #14277, #14278, #14286) diff --git a/changelog.d/fixed-selfreport-refile-loop.md b/changelog.d/fixed-selfreport-refile-loop.md new file mode 100644 index 0000000000..102288bf50 --- /dev/null +++ b/changelog.d/fixed-selfreport-refile-loop.md @@ -0,0 +1 @@ +- Self-reports no longer loop between a spoke and whoever closes them: a runtime-error fingerprint names the defect (class + normalised message) instead of the build, a fingerprint closed upstream is not re-filed by the same hive while it keeps observing the condition (the 7-day cooldown now runs from the last observation, on the direct path and through the hub relay), "Cleared on" comments are paced to once per hive per 24h and withheld under a closed issue, and the catch-all `unclassified_error_repeated` class joins only issues whose title matches the message rather than any open issue of that class. diff --git a/changelog.d/fixed-stable-promotion-status-coherent-record.md b/changelog.d/fixed-stable-promotion-status-coherent-record.md new file mode 100644 index 0000000000..747906f040 --- /dev/null +++ b/changelog.d/fixed-stable-promotion-status-coherent-record.md @@ -0,0 +1 @@ +- `GET /api/hub/release/stable-promotion` no longer mixes a fresh stable SHA with the digest and commit date of the previous stable build, and reports the stable commit time as `built_at` (it was published as `promoted_at`, which misread a commit time as the tag's promotion time). diff --git a/changelog.d/fixed-stage-tracker-poll-failure-chip.md b/changelog.d/fixed-stage-tracker-poll-failure-chip.md new file mode 100644 index 0000000000..bfe5c33d0b --- /dev/null +++ b/changelog.d/fixed-stage-tracker-poll-failure-chip.md @@ -0,0 +1 @@ +- Activity tracker: the "updated … ago" chip now reports when its 30s poll is failing (`poll failing (HTTP 504)` / `(unreachable)`), goes amber at once and demotes `fresh` to `cache`, instead of counting up "updated 8m ago · fresh" from the last good payload while the server is unreachable (#14416). diff --git a/changelog.d/fixed-stage040-covered-label-double-annotation.md b/changelog.d/fixed-stage040-covered-label-double-annotation.md new file mode 100644 index 0000000000..ba24d3e1f8 --- /dev/null +++ b/changelog.d/fixed-stage040-covered-label-double-annotation.md @@ -0,0 +1 @@ +- Issue stage sweep writes `stage/040-covered-by-pr` again for issues an open PR covers: the second in-cycle `annotateIssueWithClaim` pass no longer re-grades the hive's synthesised covered label as GitHub's, which had left issues with an open `Fixes #N` PR stuck at 010 (#14397). diff --git a/changelog.d/fixed-stagelabels-case-insensitive.md b/changelog.d/fixed-stagelabels-case-insensitive.md new file mode 100644 index 0000000000..a2076f9dc9 --- /dev/null +++ b/changelog.d/fixed-stagelabels-case-insensitive.md @@ -0,0 +1 @@ +- stagelabels: Owned and Plan now match existing stage labels case-insensitively, consistent with Current (#13909) diff --git a/changelog.d/fixed-stale-snapshot-drop-selfreport.md b/changelog.d/fixed-stale-snapshot-drop-selfreport.md new file mode 100644 index 0000000000..a07435536e --- /dev/null +++ b/changelog.d/fixed-stale-snapshot-drop-selfreport.md @@ -0,0 +1 @@ +- Demote the stale status snapshot drop to Info; warn only after repeated consecutive drops (#14117) diff --git a/changelog.d/fixed-startup-kick-interrupted.md b/changelog.d/fixed-startup-kick-interrupted.md new file mode 100644 index 0000000000..2303b4ba71 --- /dev/null +++ b/changelog.d/fixed-startup-kick-interrupted.md @@ -0,0 +1 @@ +- Classify repeated startup kicks dropped during an agent restart or stop as `startup_kick_interrupted` instead of an unclassified runtime error. diff --git a/changelog.d/fixed-tracker-closed-rows-persist.md b/changelog.d/fixed-tracker-closed-rows-persist.md new file mode 100644 index 0000000000..3b24129a92 --- /dev/null +++ b/changelog.d/fixed-tracker-closed-rows-persist.md @@ -0,0 +1 @@ +- A closed issue or PR no longer stays on the Hive activity monitor when the confirming re-read a webhook delivery queues is refused (critical API budget, rate-limit caution window) or fails: the tracker now applies the payload's own `state` on every `issues`, `issue_comment` and `pull_request*` delivery, so a closed or merged item leaves on whichever action arrives (`labeled`, `unlabeled`, a comment — not only `closed`), its eviction is handed to the governor and remembered for replay like a re-read, and an open re-read older than the listed copy can neither rewrite the row nor un-remember a close (hivecommons/hive#14159 and #14064 sat at "Triage needed" / "Needs human decision" twenty minutes after closing, through the close, a bot comment and the stage-label sweep's unlabel). As a backstop for deliveries that never arrived, on each status publish it lists each repo's items closed since its last look every `dashboard.stage_tracker.closed_sweep_interval_s` (default `300`, one list call per repo, negative disables) and evicts any still shown. diff --git a/changelog.d/fixed-tracker-since-and-count.md b/changelog.d/fixed-tracker-since-and-count.md new file mode 100644 index 0000000000..5aeaee9c87 --- /dev/null +++ b/changelog.d/fixed-tracker-since-and-count.md @@ -0,0 +1 @@ +- **Hive activity monitor: open-since age and one row count.** A plain *Open* chevron (no stage label yet, not on hold) now reads "Open · since 3d" ("since 14h" under two days), taken from the same instant the row's `since` column draws, with the ISO time in the chevron tooltip and a subtle red tint past two days (`STAGE_TRACKER_OPEN_STALE_DAYS`); *Open · hold* is unchanged, and an Open item carrying the `blocked` label gets its own "Open · blocked" word (⛔, hold's tint) that keeps the age — "Open · blocked · since 7h" — with the blocker (or the label) named in the chevron tooltip; hold wins when both are present. The sidebar badge and the list header no longer disagree: both are the **row count** of the default view from one shared function (the badge used to count open issues + PRs while the "of N" line counted rows, so a PR drawn on its issue's row made the badge read higher). That gap is now a footnote on the "showing" line's tooltip and the badge — "2 PRs shown on their issue's row" — and each such PR is led by a ↳ glyph whose link tooltip explains the shared row (and names the other issue when two list the same PR). A PR whose issue a filter hides still renders as its own row. diff --git a/changelog.d/fixed-write-counter-read-errors.md b/changelog.d/fixed-write-counter-read-errors.md new file mode 100644 index 0000000000..c73b6d127c --- /dev/null +++ b/changelog.d/fixed-write-counter-read-errors.md @@ -0,0 +1 @@ +- Preserve durable write counter totals and retry pending counts after transient file read errors ([#14531](https://github.com/hivecommons/hive/issues/14531)). diff --git a/changelog.d/security-13900-evidence-verification.md b/changelog.d/security-13900-evidence-verification.md new file mode 100644 index 0000000000..337d0acfa4 --- /dev/null +++ b/changelog.d/security-13900-evidence-verification.md @@ -0,0 +1 @@ +- Dashboard review-evidence downloads, listings, aggregates, tracker annotations, heartbeats, and evidence-derived metrics now verify bundle signatures against the configured Ed25519 key before using on-disk data; unsigned mode still verifies content hashes, while an unavailable configured key or tampered bundle fails closed ([#13900](https://github.com/hivecommons/hive/issues/13900)). diff --git a/changelog.d/security-13998-prefer-contributor.md b/changelog.d/security-13998-prefer-contributor.md new file mode 100644 index 0000000000..6839a2416a --- /dev/null +++ b/changelog.d/security-13998-prefer-contributor.md @@ -0,0 +1 @@ +- Keep the hive PR open and parked when a clean competing contributor PR references the same issue; delegated decisions now recommend a human adequacy comparison instead of automatically closing the hive's work ([#13998](https://github.com/hivecommons/hive/issues/13998)). diff --git a/changelog.d/security-14055-supersede-workflows.md b/changelog.d/security-14055-supersede-workflows.md new file mode 100644 index 0000000000..f1b63810b6 --- /dev/null +++ b/changelog.d/security-14055-supersede-workflows.md @@ -0,0 +1 @@ +- Refuse to publish contributor workflow/action edits into upstream adoption branches, and revalidate head evidence before every supersession. The refusal is recorded once per head: the PR gets one note and moves to needs-decision instead of being re-checked every sweep. diff --git a/changelog.d/security-14082-plain-forward-guard.md b/changelog.d/security-14082-plain-forward-guard.md new file mode 100644 index 0000000000..4528e189ad --- /dev/null +++ b/changelog.d/security-14082-plain-forward-guard.md @@ -0,0 +1 @@ +- The proxy's plain-HTTP forward path now identifies its caller and refuses destinations that are not globally routable — loopback, link-local (cloud instance metadata), RFC 1918, CGNAT and multicast — checked at dial time so a hostname resolving inward is refused too; set `HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true` to allow them (#14082) diff --git a/changelog.d/security-14114-forge-proxy-write-gates.md b/changelog.d/security-14114-forge-proxy-write-gates.md new file mode 100644 index 0000000000..41b2d4a3fb --- /dev/null +++ b/changelog.d/security-14114-forge-proxy-write-gates.md @@ -0,0 +1 @@ +- The proxy write-gate rule table mirrors GitHub posture for GitLab and Gitea/Forgejo: direct MR/PR creation and merge are hard-denied for every agent level (forcing the hive-open-pr and hive-merge request-file paths as the bot identity), reads and issues/comments/labels follow the ACMM ladder, and unknown routes fail closed ([#14114](https://github.com/hivecommons/hive/issues/14114), part of [#14096](https://github.com/hivecommons/hive/issues/14096)). diff --git a/changelog.d/security-14167-delegated-head-checks.md b/changelog.d/security-14167-delegated-head-checks.md new file mode 100644 index 0000000000..8c441058b5 --- /dev/null +++ b/changelog.d/security-14167-delegated-head-checks.md @@ -0,0 +1 @@ +- "Let hive decide" (`POST …/items/{number}/decide-delegated`) no longer reads a pull request head as CI-green from `mergeable_state` alone. GitHub reports `blocked` both for a missing required review and for red required checks, and `unknown`/`behind` say nothing about the head, so the delegate could approve a PR whose required checks were failing or had never run and record it as a "green head" decision. The gatherer now settles those states from the head's combined status and check runs — no red, nothing pending, at least one green — the same way it already judged a competing PR, and such heads get the `fix-then-merge` direction instead of an approval ([#14167](https://github.com/hivecommons/hive/issues/14167)). diff --git a/changelog.d/security-14238-claim-effect-workers.md b/changelog.d/security-14238-claim-effect-workers.md new file mode 100644 index 0000000000..cc7412702e --- /dev/null +++ b/changelog.d/security-14238-claim-effect-workers.md @@ -0,0 +1 @@ +- Limit GitHub claim-effect processing to one worker per active issue so claim/release churn no longer accumulates waiting goroutines; queued effects still run in order without dropping release cleanup. diff --git a/changelog.d/security-14348-callback-loopback.md b/changelog.d/security-14348-callback-loopback.md new file mode 100644 index 0000000000..7e65b2b56a --- /dev/null +++ b/changelog.d/security-14348-callback-loopback.md @@ -0,0 +1 @@ +- The hive capability endpoint (`callback.Listen`, bound from `$CRUX_HIVE_LISTEN`) now refuses a TCP address off loopback, the same fail-closed rule `crux serve --listen` has applied since #13967. The listener is plain http gated by `$CRUX_HIVE_TOKEN`, and it serves `POST /v1/tokens`, which mints GitHub App installation tokens — a routable bind would have carried both the bearer token and every minted token across the network in cleartext. The k8s sidecar-container option is unaffected: the containers share the pod's network namespace, so `127.0.0.1` reaches the sidecar. Remote access goes through a TLS-terminating proxy. (#14348) diff --git a/changelog.d/security-14439-inference-translator-identity.md b/changelog.d/security-14439-inference-translator-identity.md new file mode 100644 index 0000000000..5c37e4ce88 --- /dev/null +++ b/changelog.d/security-14439-inference-translator-identity.md @@ -0,0 +1 @@ +- The inference translator on `127.0.0.1:18444` now names its caller from the socket owner's UID, the way the GitHub proxy path and the decision endpoint already do, instead of from the `x-api-key` header the caller chose. That header is `sk-hive-` and the name is public inside the pod, so any local process could previously borrow a different agent's inference route — backend, model, upstream credential and usage attribution. A mapped agent UID is authoritative (a header naming someone else is logged and overridden), the hive's own control plane keeps the header name, and an unidentifiable caller is refused with 401 unless `HIVE_PROXY_ADVISORY_OK=true`. The translator server also gained a `ReadHeaderTimeout` ([#14439](https://github.com/hivecommons/hive/issues/14439)). diff --git a/changelog.d/security-14448-plain-forward-translator.md b/changelog.d/security-14448-plain-forward-translator.md new file mode 100644 index 0000000000..d8c9d149a6 --- /dev/null +++ b/changelog.d/security-14448-plain-forward-translator.md @@ -0,0 +1 @@ +- Refuse proxy-owned inference translator requests unless advisory identity is explicitly enabled, preventing forged agent identities through private plain-HTTP forwarding when `HIVE_PROXY_ALLOW_PRIVATE_FORWARD=true` (#14448). diff --git a/changelog.d/security-contributor-token-expiry.md b/changelog.d/security-contributor-token-expiry.md new file mode 100644 index 0000000000..d41aa24d6a --- /dev/null +++ b/changelog.d/security-contributor-token-expiry.md @@ -0,0 +1 @@ +- Contributor registration tokens expire after 90 days by default, support an optional inactivity cutoff, and can be inspected or revoked individually or in bulk from Management and `hivectl contribute tokens`; GitHub-authenticated `hivectl contribute login` rotates expired tokens, while legacy credentials receive persisted migration-time issuance dates (#13804). diff --git a/changelog.d/security-hold-guard-hygiene-merge-proof.md b/changelog.d/security-hold-guard-hygiene-merge-proof.md new file mode 100644 index 0000000000..7ba0d9f4e6 --- /dev/null +++ b/changelog.d/security-hold-guard-hygiene-merge-proof.md @@ -0,0 +1 @@ +- The hold guard only advances a held PR's reviewed baseline after a merge-from-base when the new head is a two-parent merge of a base-branch commit into the head the hive asked to update, so a contributor push racing the update is no longer adopted as reviewed hygiene; the base parent is matched against the live base branch, so a base that moved before the update does not turn the hive's own merge into drift ([#14534](https://github.com/hivecommons/hive/issues/14534)). diff --git a/config/backends.conf b/config/backends.conf index b2662ebb57..05c2a673b8 100644 --- a/config/backends.conf +++ b/config/backends.conf @@ -20,7 +20,7 @@ # ── Supported backends ────────────────────────────────────────────────── # BACKEND_NAME BINARY PERM_FLAG MODEL_FLAG -KNOWN_BACKENDS="claude copilot goose codex agy bob pi aider litellm opencode kilo muse omp openhands" +KNOWN_BACKENDS="claude copilot goose codex agy bob pi aider litellm opencode kilo muse omp openhands omnigent" # ── Backend → binary mapping ──────────────────────────────────────────── backend_binary() { @@ -39,6 +39,7 @@ backend_binary() { muse) echo "muse" ;; omp) echo "omp" ;; openhands) echo "openhands" ;; + omnigent) echo "omnigent" ;; # litellm runs Claude Code pointed at a LiteLLM proxy via ANTHROPIC_BASE_URL litellm) echo "claude" ;; *) echo "$1" ;; @@ -366,7 +367,7 @@ muse_local_perm_flag_shell() { # ── Backends with no confinement mechanism at all ─────────────────────── # -# goose, agy, bob, pi, aider, kilo, omp, and openhands expose no OS-level sandbox, no filesystem +# goose, agy, bob, pi, aider, kilo, omp, openhands, and omnigent expose no OS-level sandbox, no filesystem # write-allowlist, and no bash-command deny mechanism analogous to # opencode's `permission.bash` — verified against each CLI's own current # documentation, not assumed: @@ -413,6 +414,14 @@ muse_local_perm_flag_shell() { # which are GUI-server entry points with no one-shot task shape, so # there is no runtime flag hive can pass to confine `openhands # --headless`. Verified against OpenHands CLI 1.16.0 docs and source. +# - omnigent (Omnigent CLI, https://github.com/omnigent-ai/omnigent): +# meta-harness over Claude Code, Codex, Cursor, Pi, and custom agents. +# The bare CLI runs on the host with no OS-level boundary wired by hive +# unless configured within an agent bundle or server setup. Its policy +# engine is additive and application-level, not an OS sandbox wired +# directly by CLI flags from hive, so local mode treats it as +# unconfined without the per-backend opt-in +# HIVE_OMNIGENT_DANGEROUSLY_RUN_UNCONFINED=1. # # An HONEST "unconfined, opt in to proceed" is the documented contract here # (#4918's own ask: never claim confinement a backend does not have). Local @@ -429,11 +438,12 @@ unconfined_local_backend_env_var() { kilo) echo "HIVE_KILO_DANGEROUSLY_RUN_UNCONFINED" ;; omp) echo "HIVE_OMP_DANGEROUSLY_RUN_UNCONFINED" ;; openhands) echo "HIVE_OPENHANDS_DANGEROUSLY_RUN_UNCONFINED" ;; + omnigent) echo "HIVE_OMNIGENT_DANGEROUSLY_RUN_UNCONFINED" ;; *) echo "" ;; esac } -# unconfined_local_perm_flag_shell is the local-mode entry point for the eight +# unconfined_local_perm_flag_shell is the local-mode entry point for the nine # backends above. It refuses to emit a launch line at all unless the # operator has set that backend's own escape-hatch env var — there is no # confinement to fall back to, so silence is not an option (#4918's @@ -443,7 +453,7 @@ unconfined_local_perm_flag_shell() { local backend="$1" var var="$(unconfined_local_backend_env_var "$backend")" if [[ -z "$var" ]]; then - # Not one of the eight — caller error, not an operator-facing case. + # Not one of the nine — caller error, not an operator-facing case. backend_perm_flag_shell "$backend" return fi @@ -538,39 +548,32 @@ codex_default_sandbox_mode() { fi } +# Bob renamed its key in 2.x. Fill either missing alias so both versions +# work, preserving explicit values when contributors supply both names. +# Call in the parent shell so the relay and CLI inherit the exports. +# Unlike hub agents, which pin provider "harness" in Bob settings to override +# stale settings, contributors rely on Bob 2.x's default provider in a fresh +# container. Existing Bob settings can therefore select a different provider. +backend_bob_prepare_env() { + export BOB_API_KEY="${BOB_API_KEY:-${BOBSHELL_API_KEY:-}}" + export BOBSHELL_API_KEY="${BOBSHELL_API_KEY:-${BOB_API_KEY:-}}" + [[ -n "$BOB_API_KEY" ]] +} + # ── Backend → permission flag ─────────────────────────────────────────── backend_perm_flag() { case "$1" in claude) claude_family_perm_flag ;; copilot) echo "--allow-all" ;; - # bob (IBM bobshell). Mirrors the hub-side launcher (bobLaunchCmd in - # src/pkg/agent/manager.go) so relay and pod agents behave identically: - # --accept-license bob hard-errors before doing any work unless - # licenseConsent is persisted; the prompt that - # collects it is interactive and nobody is attached - # to an unattended agent loop. Idempotent. - # --auth-method api-key bob otherwise defaults to W3ID browser SSO. The - # relay is unattended, so it must use the API key - # from BOBSHELL_API_KEY. The flag is HIDDEN from - # `bob --help` (the bundle registers it then calls - # t.hide()) but is live and validated — an invalid - # value reports Choices: "sso", "api-key". - # It is also the only input that outranks the - # persisted security.auth.selectedType AND - # suppresses bob's write-back of that value, so it - # does NOT stomp a contributor's own ~/.bob config. - # --approval-mode yolo without it bob reports `Auto-approve: Off` and - # blocks forever on its first tool call. Note that - # `-y`/`--yolo` is a BOOLEAN flag in bobshell 1.0.6 - # ([boolean] [default: false]), so the former - # `-y yolo` spelling left `yolo` as a stray - # positional argument — i.e. a prompt. - # --trust bob otherwise treats the workdir as untrusted - # and gates tool availability on it. - # No --model is passed anywhere for bob: it auto-selects its own model and - # any --model makes every prompt die. See the bob case in the MODEL_FLAG - # block of bin/contributor-agent.sh. - bob) echo "--accept-license --auth-method api-key --approval-mode yolo --trust" ;; + # Match bobLaunchCmd in src/crux/agent/manager_launch.go. Only 1.x + # accepts --auth-method/--approval-mode; 2.x uses the chat subcommand. + # Unknown versions follow the current (2.x) CLI, just like pod agents. + bob) + case "$(bob --version 2>/dev/null | sed -n '1p')" in + 1.*) echo "--accept-license --auth-method api-key --approval-mode yolo --trust" ;; + *) echo "chat --accept-license --auto-approve --trust" ;; + esac + ;; codex) if is_truthy "${HIVE_CODEX_DANGEROUSLY_BYPASS_APPROVALS_AND_SANDBOX:-}"; then @@ -650,6 +653,25 @@ backend_perm_flag() { # container does not have, so without this flag an env-only credential # never reaches the agent. Verified against OpenHands CLI 1.16.0. openhands) echo "--always-approve --override-with-envs" ;; + # omnigent: the agent REPL lives under the `run` subcommand, and a bare + # `omnigent` on a TTY is rewritten upstream to `omnigent start` (spawns + # the background server and returns to the shell prompt — no agent). + # `run` is emitted here so every launch path (interactive tmux line, + # agent-launch.sh argv) produces `omnigent run [--server local] [--model X]`. + # + # `--server local` pins the session to a local Omnigent server. Without + # it, `run` falls back to the `server:` default that `omnigent login + # ` persists in ~/.omnigent/config.yaml — a REMOTE server whose + # managed runners are handed the user's GitHub token and write to GitHub + # outside the Hive container and egress proxy. Hive's proxy must stay + # authoritative on GitHub writes (#14094), so remote-server mode is + # unsupported. `local` is Omnigent's documented sentinel ("overriding + # any configured server default", `omnigent run --help`). + # + # Neither word is a sandbox: omnigent has no OS confinement on the bare + # local CLI path, so local mode still requires + # HIVE_OMNIGENT_DANGEROUSLY_RUN_UNCONFINED=1. + omnigent) echo "run --server local" ;; # muse (Muse Code) keeps its OWN sandbox on and only turns off the # approval prompt: `--approval-mode never` is the documented launch # policy ("untrusted|on-request|never", default on-request) and the diff --git a/dashboard/index.html b/dashboard/index.html index 63d14a3452..f826a1987b 100644 --- a/dashboard/index.html +++ b/dashboard/index.html @@ -3444,7 +3444,7 @@

Agents

const titles = { observe: 'Data collection only — governor unchanged', suggest: 'Experiments require your approval', - evolve: 'Fully autonomous experimentation', + evolve: 'Autonomous experimentation', }; html += ``; } diff --git a/dashboard/openapi.json b/dashboard/openapi.json index ac7a14bec9..5747f119ce 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -18797,6 +18797,44 @@ } } }, + "/api/contribute/token": { + "get": { + "tags": [ + "Contribute" + ], + "summary": "Get the caller's registration-token lifetime", + "description": "Authenticates the contributor registration token sent as `Authorization: Bearer ` and returns when it was issued, when it expires, and whether it is past 80% of the configured maximum age (`contribute.token_max_age`, default 2160h). Successful calls record token activity for the optional inactivity cutoff (`contribute.token_idle_max`). An expired token gets 401 with recovery guidance (`hivectl contribute login`, i.e. POST /api/contribute/reissue-token with GitHub authentication).", + "responses": { + "200": { + "description": "Token lifetime", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "issued_at": { + "type": "string", + "format": "date-time" + }, + "expires_at": { + "type": "string", + "format": "date-time" + }, + "rotation_due": { + "type": "boolean", + "description": "True once the token is at least 80% of the maximum age old" + } + } + } + } + } + }, + "401": { + "description": "Missing, invalid, expired or revoked registration token" + } + } + } + }, "/api/contributors/{id}/requeue": { "post": { "tags": [ @@ -18929,6 +18967,86 @@ } } }, + "/api/contributors/{id}/token/revoke": { + "post": { + "tags": [ + "Contributors" + ], + "summary": "Revoke one contributor's registration token", + "description": "Clears the contributor's registration token and disconnects their live relay sessions. Trust tier and history are preserved; the contributor recovers with `hivectl contribute login` (GitHub-authenticated reissue). Requires owner or read-write role (requireContributorWrite).", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Contributor ID or GitHub username" + } + ], + "responses": { + "200": { + "description": "Revoked", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "revoked": { + "type": "integer", + "description": "Number of registration tokens revoked" + } + } + } + } + } + }, + "403": { + "description": "Owner or read-write role required" + }, + "404": { + "description": "Contributor not found" + }, + "409": { + "description": "Token state changed concurrently; retry" + } + } + } + }, + "/api/contributors/tokens/revoke-all": { + "post": { + "tags": [ + "Contributors" + ], + "summary": "Revoke every contributor registration token", + "description": "Incident response: clears every contributor's registration token and disconnects all relays. Trust tiers are preserved; contributors recover with `hivectl contribute login`. Requires owner or read-write role (requireContributorWrite). A partial persistence failure returns 409 with the completed count so the call can be retried safely.", + "responses": { + "200": { + "description": "Revoked", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "revoked": { + "type": "integer", + "description": "Number of registration tokens revoked" + } + } + } + } + } + }, + "403": { + "description": "Owner or read-write role required" + }, + "409": { + "description": "Revocation stopped part-way; the message carries the completed count. Retry." + } + } + } + }, "/api/hives/onboard": { "post": { "tags": [ @@ -30341,6 +30459,414 @@ } } }, + "/api/repos/{owner}/{repo}/items/{number}/decide-delegated": { + "post": { + "tags": [ + "Status" + ], + "summary": "Let the hive decide one parked issue or pull request as the owner's delegate (enqueue a delegated-decision job)", + "description": "Backs the activity monitor's *Let hive decide* entry (first under Decide…, rows and selection bar). The hive decides the parked item itself as the owner's delegate under the owner-delegate policy (pkg/delegatedecision.PolicyText — decisive, reversible, keep work moving; prefer contributor PRs over hive duplicates; approve small low-risk changes and trusted-author sentinel alerts with a \"reviewed, intended\" line; supersede offline contributors now; dedupe aggressively; secrets/PAT/org-permission items are owner-only; quarantine flaky tests; accept CI hygiene gates as ratchets and design proposals with explicit defaults; never auto-close review-backlog items) and records it exactly as a human decision is recorded: a `/hive decision ` comment carrying the audit marker, the parking labels lifted, triage/accepted where it applies, and the queue approval on a released PR head (a merger/owner requester's as the queue line the auto-merge sweep parses, anyone else's as `/lgtm` + `/approve`). POST enqueues one job per item and answers 202 with it; a second POST while the job runs answers 409 with the running job; GET reads the job (30 min TTL). Rails: never merges; refuses an item wearing `hold` (nothing written); never grants membership, secrets or permissions (needs-owner + hold and one owner-action comment instead); a policy or scope change to the hive itself gets a recommendation only and keeps 085. Gated by decisions.delegate.enabled (default true) and decisions.delegate.repos..enabled (403 when off). Same authorisation as the hold toggle plus a signed-in GitHub user; audited as repo_item_decision_delegated or repo_item_decision_delegated_failed.", + "parameters": [ + { + "name": "owner", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "repo", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "number", + "in": "path", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "issue", + "pr" + ], + "description": "What the number names; defaults to issue" + }, + "reason": { + "type": "string", + "description": "The row's park reason code (why the hive stopped), read by the policy" + }, + "caption": { + "type": "string", + "description": "The row's lit-chevron caption" + }, + "sentinel_reason": { + "type": "string", + "description": "The sentinel finding summary when the PR is flagged" + } + } + } + } + } + }, + "responses": { + "202": { + "description": "Job enqueued (or already finished synchronously in tests)", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "number": { + "type": "integer" + }, + "type": { + "type": "string", + "enum": [ + "issue", + "pr" + ] + }, + "status": { + "type": "string", + "enum": [ + "running", + "done", + "failed" + ] + }, + "requester": { + "type": "string", + "description": "The signed-in user who delegated the decision; named in the recorded decision" + }, + "started_at": { + "type": "string", + "format": "date-time" + }, + "ended_at": { + "type": "string", + "format": "date-time" + }, + "decision": { + "type": "object", + "description": "The policy's structured decision once the job has run (pkg/delegatedecision.Decision)", + "properties": { + "verdict": { + "type": "string", + "enum": [ + "approve", + "direction", + "close-duplicate", + "close-superseded", + "owner-only", + "quarantine-flaky", + "accept-ratchet", + "accept-proposal", + "adopt-supersede", + "recommend-only", + "refused" + ] + }, + "rule": { + "type": "string", + "description": "The owner rule that fired (dedupe, owner-only, flaky-test, ci-ratchet, design-proposal, review-backlog, default-approve, prefer-contributor, contributor-offline, sentinel-trusted, small-low-risk, reporter-trust, fix-then-merge, default-release, hive-policy, sentinel-untrusted, rail-hold)" + }, + "text": { + "type": "string", + "description": "The `/hive decision` text recorded on the item" + }, + "rationale": { + "type": "string" + }, + "refusals": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Actions ApplyRails removed, each with the rail that removed it" + }, + "keeps_park": { + "type": "boolean", + "description": "True when the delegate only recommended and left stage/085-needs-decision in place" + }, + "actions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "target": { + "type": "integer", + "description": "close-as-duplicate / link: the item it points at" + }, + "text": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "comment-decision", + "approve-review", + "labels-add", + "labels-remove", + "close-as-duplicate", + "link", + "adopt-supersede", + "recommend-only" + ] + } + } + } + } + } + }, + "notes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "What the executor deferred or substituted (e.g. a sentinel-flagged approval left to the person who clears the label)" + }, + "error": { + "type": "string" + }, + "marker": { + "type": "string", + "description": "The audit marker written into every delegated comment: ``" + } + } + } + } + } + }, + "400": { + "description": "Invalid request (bad number or type)" + }, + "401": { + "description": "GitHub user session required" + }, + "403": { + "description": "Repository write permission required, or delegated decisions disabled by decisions.delegate.enabled / the per-repository override" + }, + "409": { + "description": "A delegated-decision job for this item is already running (the running job is returned)" + }, + "503": { + "description": "GitHub client not configured" + } + } + }, + "get": { + "tags": [ + "Status" + ], + "summary": "Read the delegated-decision job for one parked issue or pull request", + "description": "The activity monitor polls this after *Let hive decide* until status is done or failed, then refreshes the row. Same authorisation as POST.", + "parameters": [ + { + "name": "owner", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "repo", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "number", + "in": "path", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "The job", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "number": { + "type": "integer" + }, + "type": { + "type": "string", + "enum": [ + "issue", + "pr" + ] + }, + "status": { + "type": "string", + "enum": [ + "running", + "done", + "failed" + ] + }, + "requester": { + "type": "string", + "description": "The signed-in user who delegated the decision; named in the recorded decision" + }, + "started_at": { + "type": "string", + "format": "date-time" + }, + "ended_at": { + "type": "string", + "format": "date-time" + }, + "decision": { + "type": "object", + "description": "The policy's structured decision once the job has run (pkg/delegatedecision.Decision)", + "properties": { + "verdict": { + "type": "string", + "enum": [ + "approve", + "direction", + "close-duplicate", + "close-superseded", + "owner-only", + "quarantine-flaky", + "accept-ratchet", + "accept-proposal", + "adopt-supersede", + "recommend-only", + "refused" + ] + }, + "rule": { + "type": "string", + "description": "The owner rule that fired (dedupe, owner-only, flaky-test, ci-ratchet, design-proposal, review-backlog, default-approve, prefer-contributor, contributor-offline, sentinel-trusted, small-low-risk, reporter-trust, fix-then-merge, default-release, hive-policy, sentinel-untrusted, rail-hold)" + }, + "text": { + "type": "string", + "description": "The `/hive decision` text recorded on the item" + }, + "rationale": { + "type": "string" + }, + "refusals": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Actions ApplyRails removed, each with the rail that removed it" + }, + "keeps_park": { + "type": "boolean", + "description": "True when the delegate only recommended and left stage/085-needs-decision in place" + }, + "actions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "target": { + "type": "integer", + "description": "close-as-duplicate / link: the item it points at" + }, + "text": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "comment-decision", + "approve-review", + "labels-add", + "labels-remove", + "close-as-duplicate", + "link", + "adopt-supersede", + "recommend-only" + ] + } + } + } + } + } + }, + "notes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "What the executor deferred or substituted (e.g. a sentinel-flagged approval left to the person who clears the label)" + }, + "error": { + "type": "string" + }, + "marker": { + "type": "string", + "description": "The audit marker written into every delegated comment: ``" + } + } + } + } + } + }, + "401": { + "description": "GitHub user session required" + }, + "403": { + "description": "Repository write permission required" + }, + "404": { + "description": "No job for this item (none started, or expired after 30 minutes)" + } + } + } + }, "/api/repos/{owner}/{repo}/items/{number}/accept-triage": { "post": { "tags": [ diff --git a/docs/backend-setup.md b/docs/backend-setup.md index f52267628e..27ec865f5c 100644 --- a/docs/backend-setup.md +++ b/docs/backend-setup.md @@ -1,6 +1,6 @@ # Agent backend setup -Hive validates backend names in `src/pkg/config` and launches CLIs in `src/pkg/agent/manager.go`. `backend:` selects the runtime for an agent; inference backends are covered separately in [inference-backends.md](inference-backends.md). The acceptance bar for adding a new CLI backend, and which support tier it lands in, is in [backend-support-tiers.md](../src/docs/backend-support-tiers.md). That policy also defines the metering requirement for budget-gated hives and the deprecation path when upstream CLI drift breaks a backend. +Hive validates backend names in `src/pkg/config` and launches CLIs in `src/crux/agent/manager.go`. `backend:` selects the runtime for an agent; inference backends are covered separately in [inference-backends.md](inference-backends.md). The acceptance bar for adding a new CLI backend, and which support tier it lands in, is in [backend-support-tiers.md](../src/docs/backend-support-tiers.md). That policy also defines the metering requirement for budget-gated hives and the deprecation path when upstream CLI drift breaks a backend. ## CLI backends @@ -13,7 +13,7 @@ Hive validates backend names in `src/pkg/config` and launches CLIs in `src/pkg/a | `goose` | `goose` | Install [Goose](https://github.com/aaif-goose/goose) from its canonical upstream and configure provider/model (`GOOSE_PROVIDER`, `GOOSE_MODEL`, or `goose configure`). | Hive launches `goose run -s` and appends `--model` when set. | | `pi` | `goose` in the Go manager; `pi` in contributor scripts | Contributor mode requires `AGENT_MODEL=provider/model` plus that provider's official credential variable or `~/.pi/agent/auth.json`. In the server-side manager, `backendBinary("pi")` maps to Goose. | Contributor Pi supports interactive and headless (`--print --mode json`) delivery. No generic Pi key/provider variable exists. | | `pi` + Kiro | `goose` in the Go manager; `pi` in contributor scripts | Set `KIRO_API_KEY` and use the Pi Kiro provider model id shape, for example `AGENT_MODEL=kiro-api-key/claude-opus-5:high`. Hive forwards `KIRO_API_KEY` into contributor containers and the dashboard's `pi` model picker includes common `kiro-api-key/*` IDs as a non-authoritative fallback. | This is the supported Kiro path for now instead of a first-class `kiro-cli` backend: Kiro's stable unattended/trust/login CLI contract is not verified in Hive, while `pi` + `pi-kiro-api` is already the adopter-proven path. Quota headroom remains separate from the `anthropic` pool until the Kiro quota work in #8726 lands. | -| `bob` | `bob` | Provide `HIVE_BOB_API_KEY` or `/secrets/bob_api_key` for pods; contributor mode requires `BOBSHELL_API_KEY`. | Hive uses API-key auth headlessly and accepts the Bob license at launch. | +| `bob` | `bob` | Provide `HIVE_BOB_API_KEY` or `/secrets/bob_api_key` for pods; contributor mode requires `BOB_API_KEY` (legacy `BOBSHELL_API_KEY` also accepted). | Hive uses API-key auth headlessly and accepts the Bob license at launch. | | `codex` | `codex` | Install `@openai/codex` and run `codex login --device-auth` for subscription/OAuth auth. The CLI stores credentials in `CODEX_HOME/auth.json` (default `${HOME}/.codex/auth.json`); API-key mode can use `CODEX_API_KEY`/`OPENAI_API_KEY` or a populated auth file, but it is not required for subscription users. | Hive gives each agent its own `CODEX_HOME` and probes `auth.json` for OAuth tokens/API-key state (or API-key env presence). Contributor mode keeps `--ask-for-approval on-request`, grants the exact `HIVE_WORKSPACE_DIR` tree with `--add-dir`, and defaults `approvals_reviewer` to `auto_review` so an unattended task never waits on the contributor. The `--sandbox` value is resolved at launch rather than fixed (#6653): `workspace-write` wherever bubblewrap can actually create the user namespace it needs — local mode, and containers started with a relaxed seccomp profile — and `danger-full-access` only inside the contributor container when that namespace is denied, where the container is already the boundary and asking for a nested one failed every command. Override with `HIVE_CODEX_APPROVAL_POLICY`/`HIVE_CODEX_SANDBOX_MODE`/`HIVE_CODEX_APPROVALS_REVIEWER`, or set `HIVE_CODEX_DANGEROUSLY_BYPASS_APPROVALS_AND_SANDBOX=1` only when you intentionally want the old bypass posture. `AGENT_REASONING_EFFORT` is passed to Codex as `-c model_reasoning_effort="..."` on the contributor-relay path; server-launched agents use the per-agent `reasoning_effort` config field (`minimal`\|`low`\|`medium`\|`high`\|`xhigh`) the same way since #6163. | | `aider` | contributor scripts launch `aider`; the server-side Go manager does not launch it | Install Aider and configure its provider/API key normally for contributor mode. | Not supported as a server-side agent backend in this branch: config accepts the name, but `backendBinary("aider")` returns `unknown backend: aider`, so a pod agent will not start. Use contributor mode for Aider. | | `agy` | `agy` | Install the Antigravity CLI (`brew install --cask antigravity-cli`) and run `agy` once to sign in interactively with a Google account. **There is no API-key mode.** agy persists OAuth state under `~/.gemini` (`oauth_creds.json` with a refresh token, `google_accounts.json`, alongside the `antigravity-cli/` state dir) — `just contribute-hive agy` stages that whole directory into the container, though whether a staged credential actually re-authenticates an unattended agy has not been confirmed end-to-end (agy's binary also links an OS-keyring client, so some auth paths may need a running Secret Service the container does not provide). | **No OS-level sandbox of its own** (same posture as goose/bob/pi/aider — see [sandbox-isolation.md](../src/docs/sandbox-isolation.md)). **Container mode is the only mode with any host boundary** and is supported: `src/Dockerfile.contributor` installs the `agy` binary from Google's published, checksummed release tarball (`#5048`; it did not before). Local mode **refuses to launch** agy without `HIVE_AGY_DANGEROUSLY_RUN_UNCONFINED=1`. Hub-managed agy agents default to Hive's headless shim: the tmux pane stays at a shell prompt and each kick runs one agy turn through `hive agy-turn`, so the upstream interactive TUI wake loop in google-antigravity/antigravity-cli#945 is not resident between or during turns. The runner sends the prompt to agy on stdin (`--input-format stream-json`, so no argv size limit) and resumes the agent's own conversation by id (`--conversation`), recorded in a per-launch file; `-c` is not used because all agents share one agy conversation store. A restart, or `clear_on_kick`, starts a new conversation. The pane shows assistant text and one line per tool call. Set `HIVE_AGY_LAUNCH_MODE=interactive` (or `tui`) on the hive process only when you explicitly need the old attachable TUI. Both modes pass `--dangerously-skip-permissions` (same contract as `claude`, or agy blocks on a per-tool approval prompt nobody is attached to answer). When a model is configured the manager appends `--model --effort `: agy *requires* `--effort` alongside `--model` and otherwise ignores the model entirely. An unrecognised model is not fatal — agy warns and falls back to its own default. Since #6163 the effort is the agent's per-agent `reasoning_effort` (`low`\|`medium`\|`high`, settable in hive config and the dashboard - see [agent-configuration.md](../src/docs/agent-configuration.md)) when it is a value agy accepts; anything else (unset, or codex-only values like `xhigh`) falls back to `low`, the default agy itself uses. `AGENT_REASONING_EFFORT` is the contributor-relay spelling of the same control. Headless (`agy -p`) is verified on a host that has already signed in; whether it works unattended in a fresh container is unverified, which is also why agy stays out of `just contribute-k8s`'s headless-pod allowlist — a pod cannot complete the interactive sign-in even once. agy also exits `2` if the working directory does not resolve, where some other backends tolerate it. agy 1.1.22's own `--sandbox` flag is **not** a local OS boundary — see `config/backends.conf`'s "no confinement mechanism" section for why. | @@ -22,6 +22,7 @@ Hive validates backend names in `src/pkg/config` and launches CLIs in `src/pkg/a | `openhands` | `openhands` | Install the OpenHands CLI on the host: `uv tool install openhands --python 3.12` (PyPI package `openhands`, MIT; [docs.openhands.dev](https://docs.openhands.dev/openhands/usage/cli/installation)). Credentials are environment values: `LLM_API_KEY`, optional `LLM_BASE_URL` (OpenAI-compatible gateway), and the model as `AGENT_MODEL=provider/model` (any litellm model id, forwarded to the CLI as `LLM_MODEL`); alternatively run `openhands` once interactively, which persists them under `~/.openhands/settings.json` (`OPENHANDS_PERSISTENCE_DIR` overrides the directory). | **Contributor relay only; headless mode only** (`CONTRIBUTOR_MODE=headless`; no interactive-tmux wiring). Dispatches through `openhands --headless -t "" --always-approve --override-with-envs`. `--always-approve` is OpenHands' unattended auto-approve flag (headless mode forces it regardless); `--override-with-envs` is required because the CLI otherwise **ignores** `LLM_*` environment variables in favour of its persisted settings. OpenHands has no `--model` flag, so hive never passes one. **Confinement note:** the bare `openhands` binary runs the agent directly on the host with no sandbox, allowlist, or deny-list hive can wire (OpenHands' Docker sandbox exists only behind `openhands serve`/Agent Canvas, which have no one-shot task shape), so local mode **refuses to launch** without `HIVE_OPENHANDS_DANGEROUSLY_RUN_UNCONFINED=1` — the same #4918 refusal gate as goose/agy/bob/pi/aider/kilo/omp. **Not in the contributor image** (`src/Dockerfile.contributor` is Debian bookworm with Python 3.11; OpenHands needs 3.12), so container mode needs a derived image with the CLI installed (the launcher forwards `LLM_API_KEY`/`LLM_BASE_URL`/`LLM_MODEL` and mounts a staged copy of `~/.openhands` when `settings.json` exists), and `openhands` is excluded from `just contribute-k8s`'s headless-pod allowlist. Status: upstream marks the OpenHands CLI repository as no longer actively maintained in favour of Agent Canvas; the CLI is still published (1.16.0 at the time of wiring) and this backend is T3 experimental. | | `muse` | `muse` | Install Muse Code with `curl -fsSL https://dev.meta.ai/install.sh | bash` for host use. Both images install muse instead from its release manifest, pinned by version and per-arch SHA-256 (`MUSE_VERSION`/`MUSE_SHA256_*` in `src/Dockerfile` and `src/Dockerfile.contributor`) — the public installer is deliberately not used in a build, because its launcher resolves the release at runtime and verifies a checksum only when the server volunteers one. Note the binary is ~250-265 MB per arch, the largest CLI layer in either image. Authenticate with `export META_API_KEY=...`, or run `muse login` / `muse auth set --api-key-stdin`, which writes `~/.config/muse/auth.json`; muse documents `META_API_KEY` as always taking priority over the account login. Set the model with `export AGENT_MODEL=` — `GET https://api.meta.ai/v1/models` returns the catalog. **Query that endpoint from the machine that will actually run muse**: the catalog is caller-dependent. Measured 2026-09-08 with one and the same key, a workstation saw seven ids (including `muse-spark-1.3-contributor`) while a container in AWS saw only four — `muse-image-1.0`, `muse-spark-1.2-contributor`, `muse-spark-1.2`, `muse-spark-1.1`. Requesting a model the caller cannot see fails at task time with ``model `…` does not exist or you lack access``. Note also that muse does **not** reliably reject a bad model id up front: with a wholly invalid id, the prompt `hi` completed normally and exited 0 (3/3 trials) while `say ok` exited 1 — so a mistyped or unentitled model can pass a trivial smoke test and only fail on real work. Verify the id against the catalog rather than against a test run. `AGENT_REASONING_EFFORT` maps to `--reasoning-effort` (`none|minimal|low|medium|high|xhigh|max|ultra`). | **Contributor relay only; headless mode only** (`CONTRIBUTOR_MODE=headless`). muse *does* have an interactive TUI, but hive has no interactive-tmux wiring for it. Dispatches through `muse exec ""` — note muse is **sub-command-first**: its options are parsed by `exec`, not by the `muse` root, so hive's headless argv builder places the flags *after* `exec` (`flagsAfterCommand` in `bin/contributor-relay.js`). Getting this wrong is silent: `muse --approval-mode never exec "..."` prints the root help and exits 0 without running the task. **Confinement note:** unlike opencode/kilo, muse ships a **real OS-enforced sandbox** (bubblewrap/seccomp + namespaces on Linux, seatbelt on macOS) that is **on by default**, putting it in the `claude`/`codex`/`copilot` class. `backend_perm_flag` maps muse to `--approval-mode never --user-input-auto-resolve` — approval off, **sandbox intact**. It deliberately does **not** use `--yolo`, which muse documents as disabling approval *and* sandboxing. `just contribute-hive muse local` narrows it further via `muse_local_perm_flag_shell` (`--workspace $HIVE_WORKSPACE_DIR`, `--sandbox-network proxy-only`, `--no-foreign-personal-context`) and refuses to launch without `HIVE_WORKSPACE_DIR`. Escape hatch: `HIVE_MUSE_DANGEROUSLY_BYPASS_APPROVALS_AND_SANDBOX=1`. Excluded from `just contribute-k8s`'s headless-pod allowlist for now: muse authenticates from `META_API_KEY`, which — unlike agy's interactive OAuth — is exactly the shape that *should* work unattended in a fresh pod, but that has not been verified end-to-end yet. It stays out until it has been. | | `omp` | `omp` | Hub agents: install Oh My Pi in the hive image (the hub `src/Dockerfile` now carries the same pinned/checksummed `OMP_VERSION` layer as the contributor image) and sign in per agent/provider under that agent's durable `/data/home/agents//.omp/agent` state (OAuth subscription or API key; steps in [Signing an omp hub agent in](../src/docs/agent-configuration.md#signing-an-omp-hub-agent-in)). The dashboard offers `omp` as a method, discovers models with `omp models --json`, and keeps each model's `thinking` levels as the only valid `reasoning_effort` values. Contributor mode: install Oh My Pi and sign in once on the host (run `omp`, complete its provider setup, quit). omp keeps that sign-in under `~/.omp/agent` — `config.yml` plus the `agent.db` SQLite store whose `auth_credentials` rows hold each provider's OAuth record or API key; there is no `auth.json` — and container mode stages an allowlist of that directory into the throwaway CLI staging copy, keeping only the selected provider's credential rows (`AGENT_MODEL=provider/model`, otherwise the providers `config.yml`'s `modelRoles` name), the same least-privilege rule as `pi` ([#7678](https://github.com/hivecommons/hive/issues/7678)). A provider API key exported in your shell (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, …) is forwarded as well. `just contribute-setup omp` reports which providers are signed in and which the container will receive; if none is, the container opens on omp's setup wizard and the relay waits at it (a sign-in completed there is discarded with the container). The preflight verifies the binary and the stored credentials' *presence* only: no generic probe proves authentication. **OAuth refresh and the staged copy** ([#7922](https://github.com/hivecommons/hive/issues/7922)): OAuth refresh tokens are single-use, so a container that refreshed its staged copy left the host's row holding a revoked token — omp on the host then failed to refresh, and every later launch staged the dead row, which omp reports as `No API key found for ` before falling back to any provider that still lists a model (on the observed host, an unreachable local ollama) while the relay kept reporting the selected model. Two guards now cover this: `just contribute-hive omp` writes the container's newer credential row back into `~/.omp/agent/agent.db` every `HIVE_OMP_CREDENTIAL_SYNC_SECONDS` (default 300; `0` for exit-only) while the container runs and once more when it exits (`bin/omp-backend.js --sync-back` — only the selected provider's existing row, only its credential columns, only when strictly newer; `config.yml` and other rows never travel back), and `--stage`/`--describe` refuse to launch on a row omp has marked `disabled_cause`, printing the cause and pointing at `omp` → `/login` on the host. A container killed mid-run can still lose a refresh since its last timer pass; if omp on the host then reports a refresh failure, sign in again there. Two containers running from one host store at the same time still race each other's refreshes; give each its own sign-in. | **Contributor relay interactive mode only.** Hive starts ordinary `omp --model ` in its prepared tmux cwd, with no fabricated permission flag. OMP is excluded from the TTY-less K8s allowlist until its one-shot invocation, exit semantics, fresh-pod credentials, and Hive completion contract are demonstrated. OMP exposes no verified local sandbox or deny-list, so local mode refuses unless `HIVE_OMP_DANGEROUSLY_RUN_UNCONFINED=1`; container mode is the supported boundary, and the contributor image ships a pinned, checksummed `omp` (`OMP_VERSION` in `src/Dockerfile.contributor`, [#7661](https://github.com/hivecommons/hive/issues/7661)) so `just contribute-hive omp` runs without an `omp` on the host. | +| `omnigent` | `omnigent` | Install the Omnigent CLI ([github.com/omnigent-ai/omnigent](https://github.com/omnigent-ai/omnigent), Apache-2.0). Auth uses ambient provider keys (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`), Databricks configuration (`DATABRICKS_TOKEN`, `~/.databrickscfg`), or session credentials in `~/.omnigent/auth_tokens.json` / `~/.omnigent/config.yaml`. | **T3 experimental backend.** Meta-harness over Claude Code, Codex, Cursor, Pi, and custom agents. On the local launch path, Omnigent has no OS-level sandbox wired by Hive, so local mode refuses by default without `HIVE_OMNIGENT_DANGEROUSLY_RUN_UNCONFINED=1`. Container mode (`just contribute-hive omnigent`, default) provides standard container boundary isolation and stages only `~/.omnigent/auth_tokens.json` and `config.yaml` (never session history). Hive launches `omnigent run --server local`: a remote `server:` default from `omnigent login ` is not honoured, and remote-server mode is unsupported so Hive's egress/API proxy stays strictly authoritative on all GitHub writes. | ### LiteLLM flavors on /contribute @@ -56,7 +57,7 @@ own auth (`HIVE_BOB_API_KEY`/`BOBSHELL_API_KEY`, see the `bob` row above). `just contribute-k8s` runs backends in a TTY-less pod and only permits the backends in its `HEADLESS_BACKENDS` allowlist, currently `claude litellm copilot codex goose` (the `contribute-k8s` `HEADLESS_BACKENDS` -allowlist in the `Justfile`). `agy`, `opencode`, `kilo`, `muse`, `omp`, and `openhands` are +allowlist in the `Justfile`). `agy`, `opencode`, `kilo`, `muse`, `omp`, `openhands`, and `omnigent` are deliberately excluded: their credentials are not verified for unattended use in a fresh pod. OMP additionally remains interactive-only until the contributor relay's one-shot invocation, meaningful exit status, and completion contract are @@ -85,7 +86,7 @@ governor: Defaults are already wired: Hive consults `/secrets/bob_api_key`, then `/data/secrets/bob_api_key` (where the dashboard's Governor → Bob tab stores a key), then the `HIVE_BOB_API_KEY` environment variable. Use the dashboard tab when you do not have cluster Secret access; it writes the key to the PVC-backed `/data/secrets/bob_api_key` and relaunches parked bob agents. The value is injected into bob as `BOBSHELL_API_KEY`, and Hive launches bob with the hidden-but-supported `--auth-method api-key` flag plus full approval/trust flags for unattended operation. Store only the location in YAML, never the key value. -Contributor relay containers use the same bobshell package, but contributor-mode scripts expect `BOBSHELL_API_KEY` in the container environment when `AGENT_BACKEND=bob`. +Contributor relay containers use the same bobshell package, but contributor-mode scripts expect `BOB_API_KEY` (or legacy `BOBSHELL_API_KEY`) in the container environment when `AGENT_BACKEND=bob`. The dashboard **Test key** probe intentionally sends `User-Agent: bobshell`. IBM's edge has been observed to block generic Go/curl user agents with an HTML 403 before the request reaches bob auth, while the bobshell UA returns the real backend verdict. If you reproduce a key test manually, use that UA or treat a generic-UA 403 as an inconclusive edge block, not proof that the key is invalid. diff --git a/docs/design/symphony-spec-alignment.md b/docs/design/symphony-spec-alignment.md index d9d84bc293..c5e62bf4c6 100644 --- a/docs/design/symphony-spec-alignment.md +++ b/docs/design/symphony-spec-alignment.md @@ -44,8 +44,8 @@ rather than fragile line numbers. [subscription headroom](../../src/pkg/rotation/rotation.go) and [model discovery](../../src/pkg/dashboard/cli_models_codex.go); those are not a per-issue coding runner. -- **E4 — Isolated execution:** [`SandboxExecutor.Run` / `prepareWorkspace`](../../src/pkg/agent/sandbox_executor.go), - [path/cwd tests](../../src/pkg/agent/sandbox_executor_paths_test.go). +- **E4 — Isolated execution:** [`SandboxExecutor.Run` / `prepareWorkspace`](../../src/crux/agent/sandbox_executor.go), + [path/cwd tests](../../src/crux/agent/sandbox_executor_paths_test.go). This is an execution path, not a guarantee for all tmux agents. - **E5 — Safety:** [security self-assessment](../../src/docs/security-self-assessment.md), [ACMM matrix](../../src/docs/acmm-policy-matrix.md), diff --git a/docs/development.md b/docs/development.md index 6f1627ba5d..409dfca44e 100644 --- a/docs/development.md +++ b/docs/development.md @@ -23,11 +23,16 @@ Use `v5` as the PR base for ordinary Hive development (`v4` is feature-frozen an ## Build -Build every package in the Go module: +`src/` is a two-module Go workspace (`src/go.work`): the root module +`github.com/hivecommons/hive` and the nested Crux agent-runtime module +`github.com/hivecommons/hive/crux` at `src/crux/` (see +[`src/docs/design/crux.md`](../src/docs/design/crux.md)). In workspace mode +`./...` covers the **root module only**; the Crux module is addressed from +`src/` as `./crux/...`. Build every package in both modules: ```bash cd src -go build ./... +go build ./... ./crux/... ``` To build only the main Hive binary during quick iteration: @@ -39,13 +44,17 @@ go build ./cmd/hive ## Test -Run the module tests from `src/`: +Run both modules' tests from `src/`: ```bash cd src -go test ./... +go test ./... ./crux/... ``` +(`cd src/crux && go test ./...` is the same Crux run from inside the module; +`GOWORK=off` from `src/` builds the root module with the `replace` in +`src/go.mod` and is what the container images do.) + The `src/test/` package holds the inception e2e/regression suite. Those tests talk to a **live hive over the network** and sit behind the `integration` build tag, so the command above compiles the package but runs none of them — a plain @@ -70,6 +79,7 @@ Useful narrower loops: ```bash cd src go test ./pkg/... +go test ./crux/... go test ./cmd/hive ``` @@ -81,27 +91,31 @@ workflow that publishes the required `test` check — uses the same three flags: ```bash cd src -go test ./pkg/hub -short -race -count=1 -run '<1/3 slice>' # test (hub i/3) -go test ./pkg/agent -short -race -count=1 -run '<1/5 slice>' # test (agent i/5) -go test $PKGS -short -race -count=1 # test (rest i/3) +go test ./pkg/hub -short -race -count=1 -run '<1/4 slice>' # test (hub i/4) +go test ./pkg/dashboard -short -race -count=1 -run '<1/4 slice>' # test (dashboard i/4) +go test ./pkg/scheduler -short -race -count=1 -run '<1/3 slice>' # test (scheduler i/3) +go test ./crux/agent -short -race -count=1 -run '<1/5 slice>' # test (agent i/5) +go test $PKGS -short -race -count=1 # test (rest i/4) ``` -The workflow shards for wall-clock only: `pkg/hub` and `pkg/agent` are each -sliced across several jobs by test *function* (they are single packages too -slow to run whole), and everything else from -`go list ./pkg/... ./cmd/... ./internal/...` is +The workflow shards for wall-clock only: `pkg/hub`, `pkg/dashboard`, +`pkg/scheduler` and `crux/agent` are each sliced across several jobs by test +*function* (they are single packages too slow to run whole), and everything +else from +`go list ./pkg/... ./cmd/... ./internal/... ./crux/...` is partitioned into balanced buckets. Every shard also runs with `-v` and posts its slowest test functions to the job's step summary — look there before reaching for a local profile when the gate feels slow. Scheduled runs add `-shuffle=on` (the seed is in the log); the PR gate does not, so an order-dependent test shows up as a filed issue rather than as a red PR. The union of the shards is the whole of `./pkg/...`, -`./cmd/...`, and `./internal/...` (the shared test helpers in -`internal/testutil`), so what changes between your loop and the gate is the -*flags*, not the coverage. To reproduce the gate locally in one unsharded run: +`./cmd/...`, `./internal/...` (the shared test helpers in +`internal/testutil`) and `./crux/...` (the Crux module), so what changes +between your loop and the gate is the *flags*, not the coverage. To reproduce +the gate locally in one unsharded run: ```bash cd src -go test ./pkg/... ./cmd/... ./internal/... -short -race -count=1 +go test ./pkg/... ./cmd/... ./internal/... ./crux/... -short -race -count=1 ``` What each flag changes: @@ -186,11 +200,11 @@ Run `gofmt` on Go files you edit: gofmt -w path/to/file.go ``` -The v5 CI workflow runs `go vet ./...` after building the Hive binary. Reproduce that check locally from the Go module: +The v5 CI workflow runs `go vet ./... ./crux/...` after building the Hive binary. Reproduce that check locally from the Go workspace: ```bash cd src -go vet ./... +go vet ./... ./crux/... ``` diff --git a/docs/getting-started-contributing.md b/docs/getting-started-contributing.md index 5ae9cdabcf..8adfdca749 100644 --- a/docs/getting-started-contributing.md +++ b/docs/getting-started-contributing.md @@ -22,15 +22,16 @@ build/test commands, see [`docs/development.md`](development.md). For most changes you do **not** need a cluster. - **Go code:** you need Go (see [`docs/development.md`](development.md) for the - version and `make`/`just` targets). `cd src && go build ./...` compiles the - binary; that's enough to iterate on most code. + version and `make`/`just` targets). `cd src && go build ./... ./crux/...` + compiles both Go modules (the root module and the Crux agent runtime at + `src/crux/`); that's enough to iterate on most code. - **Docs:** no toolchain needed — edit Markdown and preview locally. - **Running the whole thing:** the fastest full run is Docker Compose from the repo's [Quick Start](../src/docs/quickstart-docker-compose.md) (`docker compose -f src/docker-compose.yaml up -d`). You only need Kubernetes for deployment-specific work. ## 3. Test a change without a cluster -- **Go:** `cd src && go test ./...` runs the unit suite. Most tests are +- **Go:** `cd src && go test ./... ./crux/...` runs the unit suite. Most tests are hermetic; a few need env like `kubectl` and are skipped otherwise. See the Test section of [`docs/development.md`](development.md). - **Shell scripts:** `*.test.sh` / `*.test.js` files under diff --git a/docs/goose-at-scale-upstream-proposal.md b/docs/goose-at-scale-upstream-proposal.md new file mode 100644 index 0000000000..4ccd226c22 --- /dev/null +++ b/docs/goose-at-scale-upstream-proposal.md @@ -0,0 +1,68 @@ +# Proposed upstream issue: Goose-at-scale guide + +This is the ready-to-paste text for the issue a maintainer files by hand in +[aaif-goose/goose](https://github.com/aaif-goose/goose), following its +issue-first workflow. Upstream requires a human-written issue to reach +**Ready** before any external PR, so Hive does not open it. Everything below +the rule is the issue body. + +--- + +**Title:** Docs: running Goose unattended at fleet scale + +## Summary + +Propose a guide under `documentation/docs/guides/` for running Goose +unattended: headless `goose run --no-session -t` from CI jobs and +orchestrators, with notes on confinement and exit behaviour. + +## Motivation + +Orchestrators and CI systems increasingly launch Goose as a child process, +one task per invocation, with no attached terminal. The existing +[running-tasks guide](https://github.com/aaif-goose/goose/blob/main/documentation/docs/guides/running-tasks.md) +covers the flags, but a few things that matter at scale are not documented: + +- **Exit-code semantics.** Which exit statuses mean success, a failed task, + a provider or credential error, or an interrupted run, so a caller can + report completion or failure reliably. +- **TTY-less behaviour.** What `goose run` does without an attached TTY, + including prompts, approvals, and output formatting. +- **Confinement.** `GOOSE_MODE=auto` permits unattended tool execution but is + not a sandbox. Operators need guidance on running Goose inside a container + or similar boundary, and on keeping unrelated credentials out of the task + environment. +- **Credential handling.** Passing provider keys (for example + `ANTHROPIC_API_KEY`) through the environment rather than command lines, + task prompts, or committed config. + +## Proposed page outline + +1. Overview and scope: one-shot headless runs versus interactive sessions. +2. Provider credentials and a one-shot task (`GOOSE_PROVIDER`, + `GOOSE_MODEL`, `GOOSE_MODE`, `--no-session`, `-t`). +3. Running without a TTY: expected behaviour and limits. +4. Exit codes and how a calling process should interpret them. +5. Confinement: what `GOOSE_MODE=auto` does and does not guarantee, and + container-based isolation. +6. Links to the provider and environment-variable references. + +Facts in the page would be checked against the current CLI before submission, +and open questions (such as exit codes) would be settled in this issue first. + +## Relationship to #12691 + +[aaif-goose/goose#12691](https://github.com/aaif-goose/goose/issues/12691) +asks where community integrations that run Goose should be listed. This +proposal is complementary: it covers how to run Goose unattended, not a list +of tools. It can be folded into whatever structure that thread settles on. + +## Reference implementation + +Hive maintains a guide covering the same ground, which can serve as a draft: +[goose-at-scale.md](https://github.com/hivecommons/hive/blob/v5/docs/goose-at-scale.md). + +## Offer + +Once this issue reaches Ready, a PR implementing the agreed outline will +follow. diff --git a/docs/goose-at-scale.md b/docs/goose-at-scale.md index e846da0c1e..f328c29334 100644 --- a/docs/goose-at-scale.md +++ b/docs/goose-at-scale.md @@ -78,3 +78,6 @@ sponsor must propose this page and obtain Ready status before submitting it upstream. Track that Goose documentation handoff in [Hive #10627](https://github.com/hivecommons/hive/issues/10627) and the [ecosystem landscape](../src/docs/landscape.md#agentic-ai-foundation-aaif-and-goose). + +The drafted upstream issue text is in +[goose-at-scale-upstream-proposal.md](goose-at-scale-upstream-proposal.md). diff --git a/docs/presentations/hive-hub-and-spoke.html b/docs/presentations/hive-hub-and-spoke.html index 0843904029..1bb528dcad 100644 --- a/docs/presentations/hive-hub-and-spoke.html +++ b/docs/presentations/hive-hub-and-spoke.html @@ -42,7 +42,7 @@

Hive: Hub & Spoke

AI Agent Orchestration for Open Source

-

From brainstorming to full autonomy and everything in-between.

+

From brainstorming to autonomous operation and everything in-between.

A fully customizable fleet of AI agents covering every level of project maintenance.

hive.hivecommons.dev

@@ -97,7 +97,7 @@

Agents & the Governor

ACMM Levels & the Proxy

-

The AI Codebase Maturity Model - 6 levels from brainstorming to fully autonomous.

+

The AI Codebase Maturity Model - 6 levels from brainstorming to autonomous.

The ACMM Proxy sits between agents and GitHub, enforcing level constraints and preventing rogue behavior.

diff --git a/examples/acmm/README.md b/examples/acmm/README.md index fdc3e3c0b4..44b7259b67 100644 --- a/examples/acmm/README.md +++ b/examples/acmm/README.md @@ -10,6 +10,6 @@ These Markdown fragments document the operating constraints for each AI Codebase | [l3.md](l3.md) | L3 Quality-Gated | Measured quality-lane behavior plus the reviewer on the hold-gated PR queue. | | [l4.md](l4.md) | L4 Security-Aware | Issues plus selected hold-gated PR behavior; reviewer on the hold-gated PR queue. | | [l5.md](l5.md) | L5 Semi-Autonomous | Hold-gated PR behavior for trusted lanes. | -| [l6.md](l6.md) | L6 Fully Autonomous | Auto-merge-on-green constraints. | +| [l6.md](l6.md) | L6 Autonomous | Auto-merge-on-green constraints. | See [src/docs/acmm-policy-matrix.md](../../src/docs/acmm-policy-matrix.md) for the authoritative lane-by-lane policy matrix used by the built-in packs. diff --git a/examples/acmm/l6.md b/examples/acmm/l6.md index 806dc400c2..232c0f0242 100644 --- a/examples/acmm/l6.md +++ b/examples/acmm/l6.md @@ -1,8 +1,8 @@ -# ACMM Level 6 — Fully Autonomous +# ACMM Level 6 — Autonomous **This file OVERRIDES any conflicting instruction in your agent policy.** -You are operating at **ACMM Level 6**. The project grants full autonomous operation. +You are operating at **ACMM Level 6**. The project grants autonomous operation. ## Permissions @@ -24,4 +24,4 @@ At L6, you own the full lifecycle. The operator trusts you to: ## Budget Awareness -Full autonomy does not mean unlimited spending. Respect the governor's token budget and mode signals. If budget is low, focus on high-impact work and defer low-priority items. +Autonomy does not mean unlimited spending. Respect the governor's token budget and mode signals. If budget is low, focus on high-impact work and defer low-priority items. diff --git a/examples/kubestellar/auto-qa-skill.md b/examples/kubestellar/auto-qa-skill.md index 2d6b7a861b..f997cc54e2 100644 --- a/examples/kubestellar/auto-qa-skill.md +++ b/examples/kubestellar/auto-qa-skill.md @@ -41,7 +41,7 @@ Process items in this priority order: --- -### 2. For Each Item — Full Autonomous Workflow +### 2. For Each Item — Autonomous Workflow #### 2a. Close Stalled Copilot PR (if one exists for this issue) diff --git a/pr-body.txt b/pr-body.txt deleted file mode 100644 index eb204b6c80..0000000000 --- a/pr-body.txt +++ /dev/null @@ -1,3 +0,0 @@ -Closes #11572 - -The kick's ISSUE FILTER notice rendered a deny-all "Agents may ONLY work issues carrying..." block whenever the filter was non-zero, even with an empty label allowlist (e.g. reporter-trust only). It now renders only when require_labels is non-empty. Adds table-driven test. diff --git a/src/Dockerfile b/src/Dockerfile index 44ffd97404..5fc56b08fe 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -33,7 +33,11 @@ WORKDIR /src # populated. Mount contents never become image layers, so nothing here can # ship a stale artifact — the mount is an input cache for `go`, which # verifies modules against go.sum regardless of where they come from. -COPY src/go.mod src/go.sum ./ +# src/go.work makes this a two-module workspace (root + the nested Crux +# module at src/crux, #13536): the root go.mod replaces the Crux module with +# ./crux, so its go.mod/go.sum must be present before `go mod download`. +COPY src/go.mod src/go.sum src/go.work src/go.work.sum ./ +COPY src/crux/go.mod src/crux/go.sum ./crux/ RUN --mount=type=cache,id=hive-go-mod,target=/go/pkg/mod \ go mod download @@ -163,7 +167,7 @@ COPY --from=tmux-builder /usr/local/bin/tmux /usr/local/bin/tmux # su-exec: lightweight setuid exec for UID switching from non-root. # gosu rejects SUID mode, so we use su-exec (which is designed for it) -# for the manager's dev→agent UID switches (pkg/agent/manager.go, +# for the manager's dev→agent UID switches (crux/agent/manager.go, # pkg/dashboard/workspace_cleanup.go, deploy/ttyd-tmux.sh — all invoked by the # non-root `dev` process AFTER the entrypoint's `exec gosu dev`, so a SUID # helper or CAP_SETUID is genuinely required to switch to agent UIDs at runtime). @@ -316,7 +320,7 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit @openai/codex@${CODEX_V echo "WARN: OpenAI Codex CLI install failed — skipping" # --- Layer 7: Claude Code (most frequent updates) --- -ARG CLAUDE_CODE_VERSION=2.1.309 +ARG CLAUDE_CODE_VERSION=2.1.311 # --ignore-scripts (kept, deliberately: no arbitrary postinstall runs during the # build) skips claude-code's install.cjs, which is what LINKS the platform # native binary into the package's bin/. The binary itself ships in the @@ -337,9 +341,9 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit @anthropic-ai/claude-co # Installs from the OFFICIAL aaif-goose/goose repo, pinned to a fixed release and # SHA-256-verified before extraction. Download failures are tolerated because # Goose is an optional backend; checksum mismatches must hard-fail the build. -ARG GOOSE_VERSION=1.55.0 -ARG GOOSE_SHA256_AMD64=fd3932908328fac850ca7ad2a8289c744e0c2ddaf4c5a2338709fb17a00b3a4c -ARG GOOSE_SHA256_ARM64=ea83f59bb165a3d999dd00edf69a58028747f31e404b96f0153452c6b1651188 +ARG GOOSE_VERSION=1.56.0 +ARG GOOSE_SHA256_AMD64=225a929b35b5130fb691eafdb50be236f2c0a150019b53973775938f8f19c281 +ARG GOOSE_SHA256_ARM64=26bcb7d3f55aac485a91d81cbdecd8d90e5af1acadeca6e580b4d58b64a888e4 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ case "$ARCH" in \ amd64) GOOSE_ARCH=x86_64; GOOSE_SHA256="$GOOSE_SHA256_AMD64" ;; \ @@ -385,9 +389,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ # --- Layer 8.27: OMP CLI (Oh My Pi — backend: omp) --- # Mirrored from Dockerfile.contributor so hub agents can launch backend: omp. # Downloads are optional like Goose/agy, but checksum mismatches are fatal. -ARG OMP_VERSION=18.12.1 -ARG OMP_SHA256_AMD64=16181401e616d7d895bae31fa97476f1513d4080807d60d3a1a04006191a1bb3 -ARG OMP_SHA256_ARM64=16545d7e747b31dbc66f2bcd63f24f12dc1dc7e944767b09b0df6591bf826a93 +ARG OMP_VERSION=18.13.1 +ARG OMP_SHA256_AMD64=ee2961c7da02e514a2475ee958c2fd85dbabf4d0907949ba5022edb2951f84b5 +ARG OMP_SHA256_ARM64=803db725f9e5bd177ce5274f7718081481f367829f773d984a0a9611f4713244 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64|x86_64) OMP_ASSET=omp-linux-x64; OMP_SHA256="$OMP_SHA256_AMD64" ;; \ @@ -485,7 +489,7 @@ ARG BOBSHELL_SHA256_ARM64=eff232eb1b69f34f984ddd295e6960470058ca922b1c751879c5a8 # backend "bob" passed config validation (validBackends in pkg/config) and # then failed at launch with no honest signal — a silent trap that cost real # debugging time. That trap is now closed at the launch site instead of at -# image-build time: pkg/agent/manager.go's launchInTmux resolves the backend +# image-build time: crux/agent/manager.go's launchInTmux resolves the backend # binary via exec.LookPath BEFORE any bob-specific logic runs, and on a miss # marks the agent StateFailed, records LastError, logs a warning, writes a # banner into the agent's own tmux pane, and emits an AuditAgentStartFailed @@ -669,7 +673,7 @@ RUN printf '\n# hive #4045: scrub backend-exported GitHub credentials from agent # while silently leaving CapAmb=0x0. When the bounding set LACKS it (self-hosted/rootless), the entrypoint # does NOT attempt the raise (setpriv would error) and falls back to plain # `gosu dev`: the binary execs fine (no file cap to block it) and the proxy's -# SO_MARK path degrades gracefully (src/pkg/proxy/github_proxy.go logs once and +# SO_MARK path degrades gracefully (src/crux/proxy/github_proxy.go logs once and # dials unmarked). setpriv comes from util-linux, installed in Layer 1. # --- Nous framework (strategist experiment engine) --- diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 1c216126d4..978a9ce78f 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -39,7 +39,8 @@ FROM --platform=$BUILDPLATFORM ${BASE_REGISTRY}/golang:1.27-alpine@sha256:f2c872 ARG TARGETOS ARG TARGETARCH WORKDIR /src -COPY src/go.mod src/go.sum ./ +COPY src/go.mod src/go.sum src/go.work src/go.work.sum ./ +COPY src/crux/go.mod src/crux/go.sum ./crux/ RUN --mount=type=cache,id=hive-go-mod,target=/go/pkg/mod \ go mod download COPY src/ . @@ -260,7 +261,7 @@ RUN mkdir -p /etc/apt/keyrings \ # npm cache is cleaned in the same layer it's populated (issue #2613, # Observation 2 "also minor") so the cache doesn't linger in this layer's # diff, mirroring the cleanup already done for the bobshell install below. -ARG CLAUDE_CODE_VERSION=2.1.309 +ARG CLAUDE_CODE_VERSION=2.1.311 ARG COPILOT_VERSION=1.0.88 # --ignore-scripts (kept, deliberately: no arbitrary postinstall runs during # the build) skips claude-code's install.cjs, which is what LINKS the platform @@ -386,9 +387,9 @@ RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \ # build time (sec-check issue #2919). Pin to a specific release tag and verify # the tarball before extraction. Download failures are tolerated because Goose # is an optional contributor backend; checksum mismatches are not tolerated. -ARG GOOSE_VERSION=1.55.0 -ARG GOOSE_SHA256_AMD64=fd3932908328fac850ca7ad2a8289c744e0c2ddaf4c5a2338709fb17a00b3a4c -ARG GOOSE_SHA256_ARM64=ea83f59bb165a3d999dd00edf69a58028747f31e404b96f0153452c6b1651188 +ARG GOOSE_VERSION=1.56.0 +ARG GOOSE_SHA256_AMD64=225a929b35b5130fb691eafdb50be236f2c0a150019b53973775938f8f19c281 +ARG GOOSE_SHA256_ARM64=26bcb7d3f55aac485a91d81cbdecd8d90e5af1acadeca6e580b4d58b64a888e4 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64) GOOSE_ARCH=x86_64; GOOSE_SHA256="$GOOSE_SHA256_AMD64" ;; \ @@ -482,9 +483,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ # No credential is baked in: omp reads the selected model's provider # credential from its documented environment variables (ANTHROPIC_API_KEY, # OPENAI_API_KEY, …), which the Justfile's container arm already forwards. -ARG OMP_VERSION=18.12.1 -ARG OMP_SHA256_AMD64=16181401e616d7d895bae31fa97476f1513d4080807d60d3a1a04006191a1bb3 -ARG OMP_SHA256_ARM64=16545d7e747b31dbc66f2bcd63f24f12dc1dc7e944767b09b0df6591bf826a93 +ARG OMP_VERSION=18.13.1 +ARG OMP_SHA256_AMD64=ee2961c7da02e514a2475ee958c2fd85dbabf4d0907949ba5022edb2951f84b5 +ARG OMP_SHA256_ARM64=803db725f9e5bd177ce5274f7718081481f367829f773d984a0a9611f4713244 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64) OMP_ASSET=omp-linux-x64; OMP_SHA256="$OMP_SHA256_AMD64" ;; \ @@ -508,8 +509,16 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ # npm-shrinkwrap.json, so install a pinned npm package with lifecycle scripts # disabled and fail the image build if the binary is absent. ARG PI_CODING_AGENT_VERSION=1.4.0 -RUN npm install -g --ignore-scripts --no-fund --no-audit \ - "@earendil-works/pi-coding-agent@${PI_CODING_AGENT_VERSION}" \ +# Registry tarballs can briefly return 404 while a release propagates. npm's +# fetch retries do not cover 404s, so retry the install with a bounded delay. +RUN for attempt in 1 2 3; do \ + if npm install -g --ignore-scripts --no-fund --no-audit \ + "@earendil-works/pi-coding-agent@${PI_CODING_AGENT_VERSION}"; then \ + break; \ + fi; \ + if [ "$attempt" -eq 3 ]; then exit 1; fi; \ + sleep 30; \ + done \ && npm cache clean --force \ && which pi diff --git a/src/changelog.d/added-live-run-line.md b/src/changelog.d/added-live-run-line.md new file mode 100644 index 0000000000..fc6a6839b6 --- /dev/null +++ b/src/changelog.d/added-live-run-line.md @@ -0,0 +1 @@ +- Agent cards and the transcript pane header show a live per-run line while an agent is working — `$cost · tokens · elapsed · tests ✓/✗/– · N files` — built from the token collector's existing per-agent cumulative figures (delta since the kick was delivered, so a reload does not reset it) plus the last test-runner result and `N files changed` summary visible in the pane. `GET /api/status` carries it as `agents[].run` and the governor strip gains a fleet `running N · $X this hour` tile from the new top-level `fleetRun` ([#14576](https://github.com/hivecommons/hive/issues/14576)). diff --git a/src/changelog.d/added-merge-issue-author-credit.md b/src/changelog.d/added-merge-issue-author-credit.md new file mode 100644 index 0000000000..ffcea66106 --- /dev/null +++ b/src/changelog.d/added-merge-issue-author-credit.md @@ -0,0 +1 @@ +- Every hive-performed merge (self-authored, trusted-author, contributor-lane and queue sweeps, `hive-merge` relay) now credits the human authors of the issues the PR closes: `Co-authored-by: ` trailers are appended to the squash (or merge) commit message, so a reporter whose issue an external clanker fixed shows on the contributor graph the way hive-opened PRs already did. Bots, the PR author and logins the branch already credits are skipped; branch commit bodies and `Signed-off-by` lines are preserved ahead of the trailers; the logins are recorded as `issue_authors=` on the merge audit entry and in review-evidence `merge.issue_authors`. One cached `GET /issues/{n}` per newly referenced issue. Switch: `auto_merge.credit_issue_authors` (default on) (#14581). diff --git a/src/changelog.d/changed-crux-cmd-hive-runtime-seam.md b/src/changelog.d/changed-crux-cmd-hive-runtime-seam.md new file mode 100644 index 0000000000..7fea880486 --- /dev/null +++ b/src/changelog.d/changed-crux-cmd-hive-runtime-seam.md @@ -0,0 +1 @@ +- `cmd/hive` now holds the agent runtime as `crux.Runtime` (plus `crux.Host` for the wiring it installs), constructs it through `crux/runtime.New` with the mode from `CRUX_MODE` (in-process by default; an unavailable mode fails boot with a clear error) and builds the fleet watchdog through `crux/runtime.NewWatchdogFleet`; `crux.Runtime` gains the `Authorizer` family and the remaining `Supervisor` reads it needed. No package in the root module imports `crux/agent` any more; the inbound ratchet baseline is empty. No behaviour change (#14138 stage 1). diff --git a/src/changelog.d/changed-crux-runtime-seam.md b/src/changelog.d/changed-crux-runtime-seam.md new file mode 100644 index 0000000000..709378152a --- /dev/null +++ b/src/changelog.d/changed-crux-runtime-seam.md @@ -0,0 +1 @@ +- Crux stage 1 of the sidecar plan (#14138): `crux.Runtime` grows the `Resumer`, `Supervisor`, `Restorer` and `Background` families `cmd/hive` calls, and `crux.Host` names the in-process wiring Hive installs (credential minters, resolvers, observers), kept off `Runtime` on purpose. The hive-provided capability types (`AppTokenMinter`, `AgentMintIssuer`, `LinearCredential`, `PRCreator`, `SpanStarter`), `ResumeHandle` and the `ErrResume*` sentinels now live in the `crux` root (aliased by `crux/agent`); `ResolveReasoningEffort` moved to `crux/config`. New `crux/config.Mode` (`inprocess` default, `sidecar` reserved; `CRUX_MODE`) and `crux/runtime.New(cfg, opts...)`, which returns the in-process runtime or a clear "not yet available" error for `sidecar`. `crux/cli.RuntimeVerbs` and its parity test tie every CLI verb to a `Runtime` method. No behaviour change. diff --git a/src/changelog.d/fixed-upgrade-drain-visible-to-hub.md b/src/changelog.d/fixed-upgrade-drain-visible-to-hub.md new file mode 100644 index 0000000000..d56c6b95fd --- /dev/null +++ b/src/changelog.d/fixed-upgrade-drain-visible-to-hub.md @@ -0,0 +1 @@ +- The hub no longer abandons a hub-managed upgrade that the spoke is deliberately holding while in-flight agent kicks drain. The spoke now reports the drain (target, kicks in flight, time waited and remaining) in its heartbeat; the orphan sweep keeps the attempt latched for as long as the spoke's own deadline (plus the usual grace) allows, the timeline and stuck-upgrade alert say the spoke is draining rather than guessing at an immutable tag, and when the retry budget is finally exhausted the failure names the kick that never settled instead of blaming a floating image tag. Before this, the 20-minute orphan window versus the 45-minute default drain made every daily-mode upgrade on a hive with a long turn in flight look like a floating-tag failure after three days. diff --git a/src/cmd/crux/definitions.go b/src/cmd/crux/definitions.go index e4383fa499..ea6c565222 100644 --- a/src/cmd/crux/definitions.go +++ b/src/cmd/crux/definitions.go @@ -4,35 +4,32 @@ import ( "fmt" "os" "path/filepath" - "regexp" "strings" "gopkg.in/yaml.v3" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/defs" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux" "github.com/hivecommons/hive/pkg/defsrc" ) // hiveLoader is the cli.DefinitionLoader backed by Hive's own config loader // and pkg/defsrc. It lives in the process boundary rather than pkg/crux/cli // because both of those packages sit outside the Crux boundary -// (src/docs/design/crux.md, "Configuration leaf"). +// (src/docs/design/crux.md, "Configuration leaf"), which is also why +// cmd/crux itself is a host package outside the Crux module set: it is to +// the crux binary what cmd/hive is to the hive binary, and the outbound +// ratchet (internal/testutil/crux_outbound_ratchet_test.go) measures the +// ring without it. type hiveLoader struct{} -// definitionNamePattern is the dashboard import handler's agent-name rule -// (kickTemplatePattern on `.md`): lower-case words joined by hyphens -// or underscores, so a definition cannot name a path. -var definitionNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) - -// maxDefinitionNameLen matches the dashboard import limit. -const maxDefinitionNameLen = 64 - -// Defaults a portable AgentDefinition is merged over; the same values the -// dashboard's agentConfigFromDefinition fills in for an import. +// The name rule and the defaults a portable AgentDefinition is merged over +// are the module loader's (crux/defs), so both crux binaries agree. const ( - defaultDefinitionBackend = "copilot" - defaultDefinitionRestartStrategy = "immediate" + maxDefinitionNameLen = defs.MaxNameLen + defaultDefinitionBackend = defs.DefaultBackend + defaultDefinitionRestartStrategy = defs.DefaultRestartStrategy ) // defaultPoliciesLocalDir is cmd/hive's default policies checkout. @@ -116,16 +113,7 @@ func isPortableDefinition(data []byte) bool { } func validateDefinitionName(name string) error { - if name == "" { - return fmt.Errorf("agent name is empty") - } - if len(name) > maxDefinitionNameLen { - return fmt.Errorf("agent name %q is longer than %d characters", name, maxDefinitionNameLen) - } - if !definitionNamePattern.MatchString(name) { - return fmt.Errorf("agent name %q contains invalid characters", name) - } - return nil + return defs.ValidateName(name) } // projectContextFromConfig builds the ProjectContext cmd/hive builds in diff --git a/src/cmd/crux/main.go b/src/cmd/crux/main.go index 6262189f34..e9f9b961a4 100644 --- a/src/cmd/crux/main.go +++ b/src/cmd/crux/main.go @@ -1,32 +1,34 @@ -// Command crux runs, observes and controls agents through crux.Runtime -// (hivecommons/hive#13536; src/docs/crux-cli.md). All behaviour lives in -// pkg/crux/cli; this package is the process boundary: signals, exit code and -// the Hive-backed definition loader (definitions.go) the CLI programs -// against through cli.DefinitionLoader. +// Command crux is the Hive-hosted build of the crux binary: it runs, +// observes and controls agents through crux.Runtime (hivecommons/hive#13536; +// src/docs/crux-cli.md), serves that runtime over the Crux API (`crux serve`; +// src/docs/crux-serve.md) and, with --server or CRUX_SERVER, drives such a +// server instead of an in-process runtime. All behaviour lives in crux/cli; +// this package is a thin host in the root module whose one job is the +// Hive-backed definition loader (definitions.go: hive.yaml through pkg/config, +// the portable AgentDefinition through pkg/defsrc) the CLI programs against +// through cli.DefinitionLoader. The same binary built from the nested module +// alone is crux/cmd/crux, which wires crux/defs (plain definition files) +// instead; verbs and flags are identical. package main import ( "context" "io" "os" - "os/signal" - "syscall" - "github.com/hivecommons/hive/pkg/crux/cli" + "github.com/hivecommons/hive/crux/cli" ) -func main() { - os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr, signalContext)) -} +// version is what --version reports; set with -ldflags "-X main.version=…". +var version = cli.DefaultVersion -// signalContext ends on SIGINT or SIGTERM so a foreground `crux run` stops -// its agent before exiting. -func signalContext() (context.Context, context.CancelFunc) { - return signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) +func main() { + os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr, cli.SignalContext)) } func run(args []string, stdin io.Reader, stdout, stderr io.Writer, ctxFn func() (context.Context, context.CancelFunc)) int { ctx, stop := ctxFn() defer stop() - return cli.Main(ctx, args, stdin, stdout, stderr, hiveLoader{}) + env := &cli.Env{Stdin: stdin, Stdout: stdout, Stderr: stderr, Loader: hiveLoader{}, Version: version} + return env.Run(ctx, args) } diff --git a/src/cmd/crux/main_test.go b/src/cmd/crux/main_test.go index 8e3ca1d154..bb7fe83c62 100644 --- a/src/cmd/crux/main_test.go +++ b/src/cmd/crux/main_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/cli" + "github.com/hivecommons/hive/crux/cli" ) func TestRunPrintsUsageWithoutACommand(t *testing.T) { @@ -35,8 +35,12 @@ func TestRunRejectsUnknownCommand(t *testing.T) { } } -func TestSignalContextIsCancellable(t *testing.T) { - ctx, stop := signalContext() - stop() - <-ctx.Done() +func TestRunReportsVersion(t *testing.T) { + var stdout bytes.Buffer + code := run([]string{"--version"}, strings.NewReader(""), &stdout, &bytes.Buffer{}, func() (context.Context, context.CancelFunc) { + return context.WithCancel(context.Background()) + }) + if code != cli.ExitOK || stdout.String() != "crux "+cli.DefaultVersion+"\n" { + t.Fatalf("code %d, stdout %q", code, stdout.String()) + } } diff --git a/src/cmd/hive-backup/main.go b/src/cmd/hive-backup/main.go index 12934766f1..696a4776ed 100644 --- a/src/cmd/hive-backup/main.go +++ b/src/cmd/hive-backup/main.go @@ -33,8 +33,8 @@ import ( "log/slog" "os" + "github.com/hivecommons/hive/crux/logscrub" "github.com/hivecommons/hive/pkg/hubbackup" - "github.com/hivecommons/hive/pkg/logscrub" "github.com/hivecommons/hive/pkg/spokebackup" ) diff --git a/src/cmd/hive/agent_activity_seams_test.go b/src/cmd/hive/agent_activity_seams_test.go index d214a0c77b..639695c2e3 100644 --- a/src/cmd/hive/agent_activity_seams_test.go +++ b/src/cmd/hive/agent_activity_seams_test.go @@ -4,9 +4,10 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux" "github.com/hivecommons/hive/pkg/governor" ) @@ -74,7 +75,7 @@ func TestIsHiveAgentAuthorRecognizesEffectiveAppBotAuthor(t *testing.T) { // leave all three empty so the hub reads no-signal exactly like a legacy spoke. func TestAgentActivityForBackendAuthOnlyCarriesFailures(t *testing.T) { cfg := activityTestConfig() - mgr := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) since := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC) failing := &crux.Process{BackendAuth: crux.BackendAuthState{ @@ -106,7 +107,7 @@ func TestAgentActivityForBackendAuthOnlyCarriesFailures(t *testing.T) { // never-started process is asserted in TestAgentActivityForRidesPauseProvenance). func TestAgentActivityForCarriesStartedAt(t *testing.T) { cfg := activityTestConfig() - mgr := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) started := time.Date(2026, 9, 10, 6, 30, 0, 0, time.UTC) proc := &crux.Process{StartedAt: &started} act := agentActivityFor(mgr, cfg, governor.State{}, "busy", "scanner", proc, nil) @@ -120,7 +121,7 @@ func TestAgentActivityForCarriesStartedAt(t *testing.T) { // crash-loop detection sees the same numbers the spoke recorded. func TestAgentActivityForCarriesRestartTelemetry(t *testing.T) { cfg := activityTestConfig() - mgr := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) lastAt := time.Now().Add(-time.Hour).UTC().Truncate(time.Second) mgr.SeedRestartTelemetry("scanner", 5, []crux.RestartEvent{{At: lastAt, Reason: "pane stall"}}, "seed fallback") diff --git a/src/cmd/hive/agent_activity_test.go b/src/cmd/hive/agent_activity_test.go index ee2bd83fec..c8cbfe0e96 100644 --- a/src/cmd/hive/agent_activity_test.go +++ b/src/cmd/hive/agent_activity_test.go @@ -4,9 +4,10 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux" "github.com/hivecommons/hive/pkg/governor" spoke "github.com/hivecommons/hive/pkg/hub/spoke" ) @@ -33,7 +34,7 @@ func activityTestConfig() *config.Config { // quiesce into a multi-day incident investigation. func TestAgentActivityForRidesPauseProvenance(t *testing.T) { cfg := activityTestConfig() - mgr := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) pausedAt := time.Date(2026, 8, 20, 9, 0, 0, 0, time.UTC) lastPane := time.Date(2026, 8, 20, 8, 55, 0, 0, time.UTC) proc := &crux.Process{ @@ -76,7 +77,7 @@ func TestAgentActivityForRidesPauseProvenance(t *testing.T) { // would make every idle on-demand agent read as divergent fleet-side. func TestAgentActivityForExpectedActiveHonorsOnDemandPack(t *testing.T) { cfg := activityTestConfig() - mgr := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) proc := &crux.Process{} scheduled := spoke.AgentActivityFor(mgr, cfg, governor.State{}, "busy", "scanner", proc, nil) @@ -103,7 +104,7 @@ func TestAgentActivityForExpectedActiveHonorsOnDemandPack(t *testing.T) { // panicking or guessing — the hub reads all-false as UNKNOWN, never as a // divergence verdict. func TestAgentActivityForNilConfig(t *testing.T) { - mgr := agent.NewManager(map[string]config.AgentConfig{}, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(map[string]config.AgentConfig{}, restoreTestLogger(), crux.ProjectContext{}) act := spoke.AgentActivityFor(mgr, nil, governor.State{}, "busy", "scanner", &crux.Process{}, nil) if act.ExpectedActive || act.Enabled { t.Errorf("nil config: ExpectedActive=%v Enabled=%v, want both false", act.ExpectedActive, act.Enabled) @@ -115,7 +116,7 @@ func TestAgentActivityForNilConfig(t *testing.T) { // capabilities for an unknown agent would fabricate a divergence signal. func TestAgentActivityForUnknownAgentCapabilitiesStayFalse(t *testing.T) { cfg := activityTestConfig() - mgr := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) act := spoke.AgentActivityFor(mgr, cfg, governor.State{}, "busy", "ghost", &crux.Process{}, nil) if act.CanOpenIssue || act.CanOpenPR || act.CanMerge { t.Errorf("unknown agent capabilities = (%v, %v, %v), want all false (UNKNOWN)", diff --git a/src/cmd/hive/agent_kicker_test.go b/src/cmd/hive/agent_kicker_test.go index 106a64d597..63e2948573 100644 --- a/src/cmd/hive/agent_kicker_test.go +++ b/src/cmd/hive/agent_kicker_test.go @@ -4,7 +4,9 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/planning" ) @@ -18,7 +20,7 @@ import ( var _ planning.Kicker = agentKicker{} func TestAgentKickerUnknownAgentError(t *testing.T) { - mgr := agent.NewManager(map[string]config.AgentConfig{}, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(map[string]config.AgentConfig{}, restoreTestLogger(), crux.ProjectContext{}) k := agentKicker{mgr: mgr} err := k.Kick("ghost", "replan kick") @@ -31,9 +33,9 @@ func TestAgentKickerUnknownAgentError(t *testing.T) { } func TestAgentKickerNotRunningAgentError(t *testing.T) { - mgr := agent.NewManager(map[string]config.AgentConfig{ + mgr := cruxtest.NewRuntime(map[string]config.AgentConfig{ "idle-agent": {Enabled: true}, - }, restoreTestLogger(), agent.ProjectContext{}) + }, restoreTestLogger(), crux.ProjectContext{}) k := agentKicker{mgr: mgr} err := k.Kick("idle-agent", "replan kick") diff --git a/src/cmd/hive/agent_repo_scope_test.go b/src/cmd/hive/agent_repo_scope_test.go index 836ea1175e..8314822863 100644 --- a/src/cmd/hive/agent_repo_scope_test.go +++ b/src/cmd/hive/agent_repo_scope_test.go @@ -12,8 +12,8 @@ import ( "path/filepath" "testing" + "github.com/hivecommons/hive/crux/proxy" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/proxy" ) func TestAgentRepoScopeRefusal_UsesReposFromAgentSpec(t *testing.T) { diff --git a/src/cmd/hive/agent_runtime.go b/src/cmd/hive/agent_runtime.go new file mode 100644 index 0000000000..dc2227ee0a --- /dev/null +++ b/src/cmd/hive/agent_runtime.go @@ -0,0 +1,44 @@ +package main + +import ( + "fmt" + "log/slog" + + "github.com/hivecommons/hive/crux" + cruxconfig "github.com/hivecommons/hive/crux/config" + cruxruntime "github.com/hivecommons/hive/crux/runtime" +) + +// hostedRuntime is the runtime as the boot sequence sees it: the crux.Runtime +// families plus crux.Host, the in-process wiring Hive installs (credential +// minters, launch-time resolvers, observers). Only the construction site and +// the installers that call Set* need it; everything else takes crux.Runtime +// or a narrower family. In sidecar mode (#14138 stage 3) the Host half is +// sidecarHost: observers fed from the runtime's event stream, capabilities +// served back over the hive callback endpoint. +type hostedRuntime interface { + crux.Runtime + crux.Host +} + +// newHostedRuntime builds the runtime selected by CRUX_MODE (in-process by +// default) and returns it with the Host wiring the boot sequence relies on: +// the in-process manager is a Host itself; the sidecar client is wrapped in +// sidecarHost. Anything else is a boot error, not a silent downgrade. +func newHostedRuntime(cfg cruxconfig.RuntimeConfig, opts ...cruxruntime.Option) (hostedRuntime, error) { + rt, err := cruxruntime.New(cfg, opts...) + if err != nil { + return nil, err + } + return hostRuntime(cfg, rt, nil) +} + +func hostRuntime(cfg cruxconfig.RuntimeConfig, rt crux.Runtime, logger *slog.Logger) (hostedRuntime, error) { + if hr, ok := rt.(hostedRuntime); ok { + return hr, nil + } + if cfg.Mode == cruxconfig.ModeSidecar { + return newSidecarHost(rt, logger), nil + } + return nil, fmt.Errorf("crux runtime mode %q (%T): %w", cfg.Mode, rt, cruxruntime.ErrNotInProcess) +} diff --git a/src/cmd/hive/agent_runtime_sidecar.go b/src/cmd/hive/agent_runtime_sidecar.go new file mode 100644 index 0000000000..d5a479b7fb --- /dev/null +++ b/src/cmd/hive/agent_runtime_sidecar.go @@ -0,0 +1,594 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "sync" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/api/callback" + cruxconfig "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/effects" + "github.com/hivecommons/hive/crux/pushbroker" + + "github.com/hivecommons/hive/pkg/cruxhost" + "github.com/hivecommons/hive/pkg/fleetreport" + "github.com/hivecommons/hive/pkg/ioscan" + "github.com/hivecommons/hive/pkg/tokens" +) + +// eventObservable is the sidecar client's event subscription +// (crux/api/client.Runtime.SetEventObserver): every event on the runtime's +// /v1/events stream, including the crux.Host observer events the server +// relays (#14138 stage 3). +type eventObservable interface { + SetEventObserver(fn func(api.Event)) +} + +// sidecarHost adapts the sidecar runtime client to hostedRuntime so the boot +// sequence is the same in both modes. The Host observers are real: they are +// fed from the event stream. The capability setters are no-ops because the +// runtime reaches those capabilities through the hive callback endpoint +// (cruxhost over callback.Listen, started by bootAgents). The policy setters +// the callback carries (#14138 stage 4) record their value here and the +// endpoint serves it as the GET /v1/policy snapshot the runtime polls; the +// inference routes, mutation boundary, mint issuer and Bob resolvers are +// stored and served by their own callback families. Only the span starter +// is in-process-only, logged once. +type sidecarHost struct { + crux.Runtime + logger *slog.Logger + + mu sync.Mutex + installed bool + kickOutcome func(agentName string, outcome crux.KickOutcome) + pause func(crux.PauseTransitionEvent) + relaunch func(agentName string) + persistPause func(name string, paused bool) + prompt func(agent, trigger, prompt string) + sandboxAudit func(agent, action, detail string) + audit crux.AuditSink + conditions crux.ConditionReporter + noted map[string]bool + + // Policy setter values, served to the runtime by policy(). + buildInfo func() (string, string) + gatewayCheck func(string) bool + explainMode func() string + autoMerge func(string) bool + kickLogSource func() string + sandboxConfig *cruxconfig.AgentSandboxConfig + deadOwner *bool + + // Call-shaped setters, served to the runtime by the inference-route and + // effect callback families (#14138 stage 4). + setRoute func(agentName, backend, model string) + clearRoute func(agentName string) + mutation effects.Boundary + // Credential writers served over the callback (#14138 stage 4): the + // mint issuer behind POST /v1/tokens/mint and the Bob resolvers behind + // GET /v1/bob/credential. App tokens need no field: the callback's + // token family mints them from b.appAuth directly. + mint crux.AgentMintIssuer + bobKey func() string + bobKeySource func() string +} + +var _ hostedRuntime = (*sidecarHost)(nil) + +func newSidecarHost(rt crux.Runtime, logger *slog.Logger) *sidecarHost { + if logger == nil { + logger = slog.Default() + } + return &sidecarHost{Runtime: rt, logger: logger, noted: map[string]bool{}} +} + +// setLogger replaces the construction-time logger with the boot's. +func (h *sidecarHost) setLogger(l *slog.Logger) { + if l == nil { + return + } + h.mu.Lock() + defer h.mu.Unlock() + h.logger = l +} + +// ---- observers: fed from the event stream ---- + +// install subscribes to the event stream once, on the first observer. +func (h *sidecarHost) install() { + if h.installed { + return + } + src, ok := h.Runtime.(eventObservable) + if !ok { + h.logger.Warn("sidecar runtime has no event stream; host observers will not fire", "runtime", typeName(h.Runtime)) + h.installed = true + return + } + h.installed = true + src.SetEventObserver(h.dispatch) +} + +func (h *sidecarHost) dispatch(ev api.Event) { + h.mu.Lock() + kick, pause, relaunch, persist, prompt, sandbox, audit, conds := h.kickOutcome, h.pause, h.relaunch, h.persistPause, h.prompt, h.sandboxAudit, h.audit, h.conditions + h.mu.Unlock() + decode := func(v any) bool { + if err := json.Unmarshal(ev.Data, v); err != nil { + h.logger.Warn("sidecar runtime: undecodable host event", "type", ev.Type, "agent", ev.Agent, "error", err) + return false + } + return true + } + switch ev.Type { + case api.EventKickOutcome: + var o api.KickOutcome + if kick != nil && decode(&o) { + kick(ev.Agent, crux.KickOutcome{Kind: o.Kind, Reason: o.Reason, At: o.At, KickAt: o.KickAt}) + } + case api.EventPauseTransition: + var p api.PauseTransition + if pause != nil && decode(&p) { + pause(crux.PauseTransitionEvent{ + Agent: ev.Agent, Paused: p.Paused, Trigger: p.Trigger, Reason: p.Reason, By: p.By, At: p.At, + Causation: crux.PauseCausation{Depth: p.Causation.Depth, HookName: p.Causation.HookName, OriginTransition: p.Causation.OriginTransition}, + }) + } + case api.EventRelaunch: + if relaunch != nil { + relaunch(ev.Agent) + } + case api.EventPausePersist: + var p api.PausePersist + if persist != nil && decode(&p) { + persist(ev.Agent, p.Paused) + } + case api.EventPrompt: + var p api.Prompt + if prompt != nil && decode(&p) { + prompt(ev.Agent, p.Trigger, p.Prompt) + } + case api.EventSandboxAudit: + var a api.SandboxAudit + if sandbox != nil && decode(&a) { + sandbox(ev.Agent, a.Action, a.Detail) + } + case api.EventAudit: + var a api.Audit + if audit != nil && decode(&a) { + audit.Record(a.Actor, a.Action, ev.Agent, a.Fields) + } + case api.EventCondition: + var c api.ConditionEvent + if conds != nil && decode(&c) { + if c.Action == api.ConditionClear { + conds.Clear(c.Class, c.Key) + } else { + conds.Observe(c.Class, c.Key, c.Message, c.Diagnostics) + } + } + } +} + +func (h *sidecarHost) SetKickOutcomeObserver(fn func(agentName string, outcome crux.KickOutcome)) { + h.mu.Lock() + defer h.mu.Unlock() + h.kickOutcome = fn + h.install() +} + +func (h *sidecarHost) SetPauseTransitionObserver(fn func(crux.PauseTransitionEvent)) { + h.mu.Lock() + defer h.mu.Unlock() + h.pause = fn + h.install() +} + +func (h *sidecarHost) SetRelaunchObserver(fn func(agentName string)) { + h.mu.Lock() + defer h.mu.Unlock() + h.relaunch = fn + h.install() +} + +func (h *sidecarHost) SetPersistPauseCallback(fn func(name string, paused bool)) { + h.mu.Lock() + defer h.mu.Unlock() + h.persistPause = fn + h.install() +} + +func (h *sidecarHost) SetRecordPromptCallback(fn func(agent, trigger, prompt string)) { + h.mu.Lock() + defer h.mu.Unlock() + h.prompt = fn + h.install() +} + +func (h *sidecarHost) SetSandboxAuditCallback(fn func(agent, action, detail string)) { + h.mu.Lock() + defer h.mu.Unlock() + h.sandboxAudit = fn + h.install() +} + +func (h *sidecarHost) SetAuditSink(sink crux.AuditSink) { + h.mu.Lock() + defer h.mu.Unlock() + h.audit = sink + h.install() +} + +func (h *sidecarHost) SetConditionReporter(r crux.ConditionReporter) { + h.mu.Lock() + defer h.mu.Unlock() + h.conditions = r + h.install() +} + +// ---- capabilities: served to the runtime over the hive callback ---- + +// SetAppAuth needs no storage: the sidecar's App token writer mints over +// POST /v1/tokens, whose scoper is hive's AppAuth itself. The mint issuer +// and Bob resolvers are stored and served by the agent_mint and bob +// callback families. +func (h *sidecarHost) SetAppAuth(crux.AppTokenMinter) { h.callbackCarried("SetAppAuth") } +func (h *sidecarHost) SetAgentMint(issuer crux.AgentMintIssuer) { + h.setPolicy("SetAgentMint", func() { h.mint = issuer }) +} +func (h *sidecarHost) SetBobAPIKeyResolver(fn func() string) { + h.setPolicy("SetBobAPIKeyResolver", func() { h.bobKey = fn }) +} +func (h *sidecarHost) SetBobKeySourceResolver(fn func() string) { + h.setPolicy("SetBobKeySourceResolver", func() { h.bobKeySource = fn }) +} + +// Enabled and MintAgentToken are cruxhost.AgentMintIssuer over the stored +// issuer: mint off (or never set, as in-process when the mint is not +// configured) is an empty token. +func (h *sidecarHost) Enabled() bool { + h.mu.Lock() + m := h.mint + h.mu.Unlock() + return m != nil && m.Enabled() +} + +func (h *sidecarHost) MintAgentToken(agentName, tier string) (string, error) { + h.mu.Lock() + m := h.mint + h.mu.Unlock() + if m == nil || !m.Enabled() { + return "", nil + } + return m.MintAgentToken(agentName, tier) +} + +// bobCredential is cruxhost.BobResolver over the stored resolvers. +func (h *sidecarHost) bobCredential() (apiKey, source string) { + h.mu.Lock() + key, src := h.bobKey, h.bobKeySource + h.mu.Unlock() + if key != nil { + apiKey = key() + } + if src != nil { + source = src() + } + return apiKey, source +} +func (h *sidecarHost) SetLinearCredentialResolver(func() crux.LinearCredential) { + h.callbackCarried("SetLinearCredentialResolver") +} +func (h *sidecarHost) SetSandboxPushMinter(pushbroker.TokenMinter) { + h.callbackCarried("SetSandboxPushMinter") +} +func (h *sidecarHost) SetSandboxPRClient(crux.PRCreator) { h.callbackCarried("SetSandboxPRClient") } + +// ---- policy: carried to the runtime as the callback's policy snapshot ---- + +func (h *sidecarHost) SetBuildInfo(fn func() (string, string)) { + h.setPolicy("SetBuildInfo", func() { h.buildInfo = fn }) +} +func (h *sidecarHost) SetGatewayBackendChecker(fn func(string) bool) { + h.setPolicy("SetGatewayBackendChecker", func() { h.gatewayCheck = fn }) +} +func (h *sidecarHost) SetExplainModeDefaultResolver(fn func() string) { + h.setPolicy("SetExplainModeDefaultResolver", func() { h.explainMode = fn }) +} +func (h *sidecarHost) SetAdvisorEnabledResolver(func(string) bool) { + h.inProcessOnly("SetAdvisorEnabledResolver") +} +func (h *sidecarHost) SetAdvisorRolesResolver(func() map[string]cruxconfig.ModelRole) { + h.inProcessOnly("SetAdvisorRolesResolver") +} +func (h *sidecarHost) SetRepeatFailureEnabledResolver(func() bool) { + h.inProcessOnly("SetRepeatFailureEnabledResolver") +} +func (h *sidecarHost) SetRepoAutoMergeEnabledResolver(fn func(string) bool) { + h.setPolicy("SetRepoAutoMergeEnabledResolver", func() { h.autoMerge = fn }) +} +func (h *sidecarHost) SetKickLogSourceResolver(fn func() string) { + h.setPolicy("SetKickLogSourceResolver", func() { h.kickLogSource = fn }) +} +func (h *sidecarHost) SetSandboxConfig(cfg cruxconfig.AgentSandboxConfig) { + h.setPolicy("SetSandboxConfig", func() { h.sandboxConfig = &cfg }) +} +func (h *sidecarHost) SetDeadSessionRecoveryOwner(ownedElsewhere bool) { + h.setPolicy("SetDeadSessionRecoveryOwner", func() { h.deadOwner = &ownedElsewhere }) +} + +// The inference routes configure hive's proxy: the sidecar manager pushes +// each agent's route over PUT/DELETE /v1/inference/routes/{agent} and the +// stored closures land it in the proxy. The mutation boundary is reached +// two-phase over POST /v1/effects; hive's journaling boundary runs here. +func (h *sidecarHost) SetInferenceCallbacks(setRoute func(agentName, backend, model string), clearRoute func(agentName string)) { + h.setPolicy("SetInferenceCallbacks", func() { h.setRoute, h.clearRoute = setRoute, clearRoute }) +} + +func (h *sidecarHost) SetSandboxMutationBoundary(boundary effects.Boundary) { + h.setPolicy("SetSandboxMutationBoundary", func() { h.mutation = boundary }) +} + +// SetSpanStarter is in-process-only: a span is a Go value wrapping a call +// in the same address space. The sidecar's manager starts its spans on the +// OpenTelemetry global tracer (a no-op without an exporter configured in +// the runtime's own process); trajectory events do not depend on it. +func (h *sidecarHost) SetSpanStarter(crux.SpanStarter) { h.inProcessOnly("SetSpanStarter") } + +// SetInferenceRoute and ClearInferenceRoute are cruxhost.InferenceRouter: +// the callback's end of SetInferenceCallbacks. A route that arrives before +// boot has installed the closures is dropped and logged; the runtime's +// periodic resync pushes it again. +func (h *sidecarHost) SetInferenceRoute(agentName, backend, model string) { + h.mu.Lock() + fn := h.setRoute + h.mu.Unlock() + if fn == nil { + h.logger.Warn("sidecar runtime: inference route arrived before the proxy callbacks were installed", "agent", agentName) + return + } + fn(agentName, backend, model) +} + +func (h *sidecarHost) ClearInferenceRoute(agentName string) { + h.mu.Lock() + fn := h.clearRoute + h.mu.Unlock() + if fn != nil { + fn(agentName) + } +} + +// Execute is effects.Boundary: hive's journaling boundary once boot has +// set it, the nil boundary (the effect runs unjournaled, as in-process) +// before. +func (h *sidecarHost) Execute(ctx context.Context, claim effects.Claim, effect func(context.Context) (effects.Result, error)) (effects.Result, error) { + h.mu.Lock() + b := h.mutation + h.mu.Unlock() + return effects.Execute(ctx, b, claim, effect) +} + +func (h *sidecarHost) setPolicy(setter string, store func()) { + h.mu.Lock() + store() + h.mu.Unlock() + h.callbackCarried(setter) +} + +// policy is the GET /v1/policy snapshot: one field per policy setter boot +// has called, resolved now. gateways lists the configured gateway names; +// the stored checker decides which of them the runtime may route to. +func (h *sidecarHost) policy(gateways func() []string) callback.Policy { + h.mu.Lock() + build, gw, explain, kick, sandbox, owner := h.buildInfo, h.gatewayCheck, h.explainMode, h.kickLogSource, h.sandboxConfig, h.deadOwner + h.mu.Unlock() + var p callback.Policy + if build != nil { + v, c := build() + p.Build = &callback.BuildInfo{Version: v, Commit: c} + } + if explain != nil { + v := explain() + p.ExplainModeDefault = &v + } + if kick != nil { + v := kick() + p.KickLogSource = &v + } + if gw != nil { + names := []string{} + if gateways != nil { + for _, name := range gateways() { + if gw(name) { + names = append(names, name) + } + } + } + p.GatewayBackends = &names + } + if owner != nil { + v := *owner + p.DeadSessionRecoveryOwnedElsewhere = &v + } + if sandbox != nil { + if b, err := json.Marshal(sandbox); err == nil { + p.Sandbox = b + } + } + return p +} + +// repoAutoMergeEnabled answers the per-repo auto-merge call with the stored +// resolver, default-on (the manager's nil-resolver behaviour) until boot +// sets one. +func (h *sidecarHost) repoAutoMergeEnabled(repo string) bool { + h.mu.Lock() + fn := h.autoMerge + h.mu.Unlock() + if fn == nil { + return true + } + return fn(repo) +} + +func (h *sidecarHost) callbackCarried(setter string) { + h.note(setter, "sidecar runtime: setter is served by the hive callback endpoint instead") +} + +func (h *sidecarHost) inProcessOnly(setter string) { + h.note(setter, "sidecar runtime: setter is in-process-only; the runtime uses its own in-process equivalent") +} + +func (h *sidecarHost) note(setter, msg string) { + h.mu.Lock() + defer h.mu.Unlock() + if h.noted[setter] { + return + } + h.noted[setter] = true + h.logger.Info(msg, "setter", setter) +} + +func typeName(v any) string { + if v == nil { + return "" + } + return fmt.Sprintf("%T", v) +} + +// ---- hive side of the callback: the capabilities hive serves the runtime ---- + +// hiveCallbackCapabilities is the cruxhost capability set hive serves a +// sidecar runtime: the same values the in-process wiring hands the manager +// directly. Values that are not yet constructed when bootAgents runs (the +// Linear resolver, from bootSupervision) are read lazily; nil ones report +// not_configured and the runtime treats them as the nil seam. +func (b *boot) hiveCallbackCapabilities() *cruxhost.Capabilities { + deps := cruxhost.Deps{ + REST: githubRESTAccounting{}, + Canary: ioscan.DefaultCanaries, + Text: ioscan.ToolApproveScanner{}, + Conditions: fleetreport.DefaultReporter{}, + Linear: func() (string, string) { + if b.linearCredentialResolver == nil { + return "", "" + } + c := b.linearCredentialResolver() + return c.AccessToken, c.APIKey + }, + } + if b.appAuth != nil { + deps.Tokens = b.appAuth + } + // The usage ledger is built by bootProxy, after this; the capability is + // served from the start and records into the ledger once it exists. + deps.Usage = lateUsageSink{get: b.inferenceSinkRef, logger: b.logger, once: &sync.Once{}} + if b.ghClient != nil { + deps.PullRequests = b.ghClient + } + if sh, ok := b.agentMgr.(*sidecarHost); ok { + deps.Policy = func() callback.Policy { return sh.policy(b.gatewayNames) } + deps.RepoAutoMerge = sh.repoAutoMergeEnabled + deps.Inference = sh + deps.Mutation = sh + deps.Mint = sh + deps.Bob = sh.bobCredential + } + return cruxhost.New(deps) +} + +// setInferenceSink publishes the proxy's usage ledger for the callback. +func (b *boot) setInferenceSink(s *tokens.InferenceSink) { + b.inferenceSinkMu.Lock() + defer b.inferenceSinkMu.Unlock() + b.inferenceSink = s +} + +// inferenceSinkRef is the ledger as a cruxhost.UsageSink, nil until bootProxy +// built it (a typed-nil pointer must not escape into the interface). +func (b *boot) inferenceSinkRef() cruxhost.UsageSink { + b.inferenceSinkMu.Lock() + defer b.inferenceSinkMu.Unlock() + if b.inferenceSink == nil { + return nil + } + return b.inferenceSink +} + +// lateUsageSink is cruxhost.UsageSink over a ledger that may not exist yet: +// a record arriving before bootProxy is dropped and logged once. The sidecar +// runtime's agents cannot launch before boot completes, so in practice the +// window is empty; the guard is for the ordering, not a steady state. +type lateUsageSink struct { + get func() cruxhost.UsageSink + logger *slog.Logger + once *sync.Once +} + +func (l lateUsageSink) sink() cruxhost.UsageSink { + s := l.get() + if s == nil && l.once != nil { + l.once.Do(func() { + if l.logger != nil { + l.logger.Warn("sidecar runtime: inference usage arrived before the proxy's usage ledger was built; record dropped") + } + }) + } + return s +} + +func (l lateUsageSink) Record(agent, model string, in, out int64) { + if s := l.sink(); s != nil { + s.Record(agent, model, in, out) + } +} + +func (l lateUsageSink) RecordCopilot(agent, model string, in, out int64) { + if s := l.sink(); s != nil { + s.RecordCopilot(agent, model, in, out) + } +} + +func (l lateUsageSink) RecordCopilotWithCache(agent, model string, in, out, cacheRead, cacheCreation int64) { + if s := l.sink(); s != nil { + s.RecordCopilotWithCache(agent, model, in, out, cacheRead, cacheCreation) + } +} + +// gatewayNames lists the configured model gateway names off the live config. +func (b *boot) gatewayNames() []string { + if b.cfg == nil { + return nil + } + var names []string + for _, gw := range b.cfg.Governor.ResolvedGateways() { + if gw.Name != "" { + names = append(names, gw.Name) + } + } + return names +} + +// startHiveCallback serves caps on the callback socket ($CRUX_HIVE_SOCKET, +// default next to the runtime socket) and, when $CRUX_HIVE_LISTEN is set, +// on TCP with the $CRUX_HIVE_TOKEN bearer. The server runs until ctx ends. +func startHiveCallback(ctx context.Context, caps callback.Capabilities, logger *slog.Logger) error { + token := os.Getenv(callback.TokenEnvVar) + ls, err := callback.Listen(callback.ListenConfig{Socket: callback.DefaultSocket(), Listen: os.Getenv(callback.ListenEnvVar), Token: token}) + if err != nil { + return err + } + logger.Info("hive callback endpoint serving", "addrs", ls.Addrs()) + go func() { + if err := callback.NewServer(caps, token).Serve(ctx, ls); err != nil { + logger.Error("hive callback endpoint stopped", "error", err) + } + }() + return nil +} diff --git a/src/cmd/hive/agent_runtime_sidecar_test.go b/src/cmd/hive/agent_runtime_sidecar_test.go new file mode 100644 index 0000000000..a29aec76b9 --- /dev/null +++ b/src/cmd/hive/agent_runtime_sidecar_test.go @@ -0,0 +1,598 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/api/callback" + cruxconfig "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/cruxtest" + "github.com/hivecommons/hive/crux/effects" + cruxruntime "github.com/hivecommons/hive/crux/runtime" + "github.com/hivecommons/hive/pkg/tokens" +) + +// sidecarRuntimeFake is a crux.Runtime with the sidecar client's event +// subscription and no crux.Host: what cruxruntime.New returns in sidecar +// mode, as the boot sequence sees it. +type sidecarRuntimeFake struct { + crux.Runtime + mu sync.Mutex + observer func(api.Event) +} + +func newSidecarRuntimeFake() *sidecarRuntimeFake { + return &sidecarRuntimeFake{Runtime: cruxtest.NewRecorder()} +} + +func (f *sidecarRuntimeFake) SetEventObserver(fn func(api.Event)) { + f.mu.Lock() + defer f.mu.Unlock() + f.observer = fn +} + +func (f *sidecarRuntimeFake) emit(t *testing.T, typ, agent string, data any) { + t.Helper() + var raw json.RawMessage + if data != nil { + b, err := json.Marshal(data) + if err != nil { + t.Fatal(err) + } + raw = b + } + f.mu.Lock() + fn := f.observer + f.mu.Unlock() + if fn == nil { + t.Fatal("no event observer installed") + } + fn(api.Event{Time: time.Now(), Type: typ, Agent: agent, Data: raw}) +} + +type recordingConditions struct { + mu sync.Mutex + observed []string + cleared []string +} + +func (r *recordingConditions) Observe(class, key, message, diagnostics string) { + r.mu.Lock() + defer r.mu.Unlock() + r.observed = append(r.observed, class+"/"+key+":"+message+":"+diagnostics) +} + +func (r *recordingConditions) Clear(class, key string) { + r.mu.Lock() + defer r.mu.Unlock() + r.cleared = append(r.cleared, class+"/"+key) +} + +type recordingAudit struct { + mu sync.Mutex + entries []string +} + +func (r *recordingAudit) Record(actor, action, agentName string, fields map[string]any) { + r.mu.Lock() + defer r.mu.Unlock() + r.entries = append(r.entries, actor+"|"+action+"|"+agentName+"|"+fields["outcome"].(string)) +} + +// hostRuntime hands an in-process Host straight through, wraps a sidecar +// runtime, and refuses anything else: a Host-less runtime in in-process mode +// is a boot error, not a silent downgrade. +func TestHostRuntime_SelectsByModeAndShape(t *testing.T) { + rec := cruxtest.NewRecorder() + hr, err := hostRuntime(cruxconfig.RuntimeConfig{Mode: cruxconfig.ModeInProcess}, rec, nil) + if err != nil || hr != crux.Runtime(rec) { + t.Fatalf("in-process Host: got (%T, %v), want the recorder itself", hr, err) + } + + sc := newSidecarRuntimeFake() + hr, err = hostRuntime(cruxconfig.RuntimeConfig{Mode: cruxconfig.ModeSidecar}, sc, nil) + if err != nil { + t.Fatal(err) + } + if _, ok := hr.(*sidecarHost); !ok { + t.Fatalf("sidecar runtime wrapped as %T, want *sidecarHost", hr) + } + + _, err = hostRuntime(cruxconfig.RuntimeConfig{Mode: cruxconfig.ModeInProcess}, sc, nil) + if !errors.Is(err, cruxruntime.ErrNotInProcess) { + t.Fatalf("Host-less in-process runtime: err = %v, want ErrNotInProcess", err) + } +} + +// Every Host observer the boot sequence installs fires from the matching +// event on the runtime's stream, with the payload mapped back to the crux +// types; the subscription is installed once, on the first observer. +func TestSidecarHost_ObserversFedFromEventStream(t *testing.T) { + sc := newSidecarRuntimeFake() + var log bytes.Buffer + h := newSidecarHost(sc, slog.New(slog.NewTextHandler(&log, nil))) + + var ( + mu sync.Mutex + kicks []string + pauses []crux.PauseTransitionEvent + relaunch []string + persist []string + prompts []string + sandbox []string + ) + h.SetKickOutcomeObserver(func(agent string, o crux.KickOutcome) { + mu.Lock() + defer mu.Unlock() + kicks = append(kicks, agent+":"+o.Kind+":"+o.Reason) + }) + if sc.observer == nil { + t.Fatal("first observer did not subscribe to the event stream") + } + h.SetPauseTransitionObserver(func(ev crux.PauseTransitionEvent) { + mu.Lock() + defer mu.Unlock() + pauses = append(pauses, ev) + }) + h.SetRelaunchObserver(func(agent string) { + mu.Lock() + defer mu.Unlock() + relaunch = append(relaunch, agent) + }) + h.SetPersistPauseCallback(func(name string, paused bool) { + mu.Lock() + defer mu.Unlock() + persist = append(persist, name+":"+map[bool]string{true: "paused", false: "resumed"}[paused]) + }) + h.SetRecordPromptCallback(func(agent, trigger, prompt string) { + mu.Lock() + defer mu.Unlock() + prompts = append(prompts, agent+":"+trigger+":"+prompt) + }) + h.SetSandboxAuditCallback(func(agent, action, detail string) { + mu.Lock() + defer mu.Unlock() + sandbox = append(sandbox, agent+":"+action+":"+detail) + }) + audit := &recordingAudit{} + h.SetAuditSink(audit) + conds := &recordingConditions{} + h.SetConditionReporter(conds) + + at := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + sc.emit(t, api.EventKickOutcome, "scanner", api.KickOutcome{Kind: "question", Reason: "asked", At: at, KickAt: at}) + sc.emit(t, api.EventPauseTransition, "scanner", api.PauseTransition{Paused: true, Trigger: "hook", Reason: "budget", By: "system", At: at, + Causation: api.PauseCausation{Depth: 2, HookName: "pre-kick", OriginTransition: "pause"}}) + sc.emit(t, api.EventRelaunch, "scanner", nil) + sc.emit(t, api.EventPausePersist, "scanner", api.PausePersist{Paused: true}) + sc.emit(t, api.EventPrompt, "scanner", api.Prompt{Trigger: "schedule", Prompt: "go"}) + sc.emit(t, api.EventSandboxAudit, "scanner", api.SandboxAudit{Action: "push", Detail: "main"}) + sc.emit(t, api.EventAudit, "scanner", api.Audit{Actor: "alice", Action: "restart", Fields: map[string]any{"outcome": "ok"}}) + sc.emit(t, api.EventCondition, "scanner", api.ConditionEvent{Action: api.ConditionObserve, Class: "stall", Key: "scanner", Message: "m", Diagnostics: "d"}) + sc.emit(t, api.EventCondition, "scanner", api.ConditionEvent{Action: api.ConditionClear, Class: "stall", Key: "scanner"}) + // Unknown types and undecodable payloads are logged, never fatal. + sc.emit(t, "agent.state", "scanner", nil) + sc.observer(api.Event{Type: api.EventPrompt, Agent: "scanner", Data: json.RawMessage(`{bad`)}) + + mu.Lock() + defer mu.Unlock() + if got := strings.Join(kicks, ","); got != "scanner:question:asked" { + t.Errorf("kick outcomes = %q", got) + } + if len(pauses) != 1 || pauses[0].Agent != "scanner" || !pauses[0].Paused || pauses[0].Causation.Depth != 2 || + pauses[0].Causation.HookName != "pre-kick" || pauses[0].Causation.OriginTransition != "pause" || !pauses[0].At.Equal(at) { + t.Errorf("pause transitions = %+v", pauses) + } + if got := strings.Join(relaunch, ","); got != "scanner" { + t.Errorf("relaunches = %q", got) + } + if got := strings.Join(persist, ","); got != "scanner:paused" { + t.Errorf("persisted pauses = %q", got) + } + if got := strings.Join(prompts, ","); got != "scanner:schedule:go" { + t.Errorf("prompts = %q", got) + } + if got := strings.Join(sandbox, ","); got != "scanner:push:main" { + t.Errorf("sandbox audits = %q", got) + } + audit.mu.Lock() + if got := strings.Join(audit.entries, ","); got != "alice|restart|scanner|ok" { + t.Errorf("audit entries = %q", got) + } + audit.mu.Unlock() + conds.mu.Lock() + if strings.Join(conds.observed, ",") != "stall/scanner:m:d" || strings.Join(conds.cleared, ",") != "stall/scanner" { + t.Errorf("conditions observed=%v cleared=%v", conds.observed, conds.cleared) + } + conds.mu.Unlock() + if !strings.Contains(log.String(), "undecodable host event") { + t.Errorf("bad payload not logged:\n%s", log.String()) + } +} + +// Observers set to nil stop firing; a runtime without an event stream warns +// once rather than panicking. +func TestSidecarHost_NilObserverAndNoStream(t *testing.T) { + sc := newSidecarRuntimeFake() + h := newSidecarHost(sc, nil) + fired := 0 + h.SetRelaunchObserver(func(string) { fired++ }) + h.SetRelaunchObserver(nil) + sc.emit(t, api.EventRelaunch, "scanner", nil) + if fired != 0 { + t.Fatalf("nil observer fired %d times", fired) + } + + var log bytes.Buffer + plain := newSidecarHost(struct{ crux.Runtime }{cruxtest.NewRecorder()}, slog.New(slog.NewTextHandler(&log, nil))) + plain.SetRelaunchObserver(func(string) {}) + plain.SetKickOutcomeObserver(nil) + if n := strings.Count(log.String(), "no event stream"); n != 1 { + t.Fatalf("no-stream warning logged %d times, want 1:\n%s", n, log.String()) + } +} + +// Capability and policy setters say once per setter how they reach a +// sidecar runtime: capabilities and the carried policy setters over the +// callback endpoint; the span starter in-process-only. +func TestSidecarHost_CapabilityAndPolicySettersLogOnce(t *testing.T) { + var log bytes.Buffer + h := newSidecarHost(newSidecarRuntimeFake(), slog.New(slog.NewTextHandler(&log, nil))) + h.SetAppAuth(nil) + h.SetAppAuth(nil) + h.SetAgentMint(nil) + h.SetBobAPIKeyResolver(nil) + h.SetBobKeySourceResolver(nil) + h.SetLinearCredentialResolver(nil) + h.SetSandboxPushMinter(nil) + h.SetSandboxPRClient(nil) + h.SetBuildInfo(nil) + h.SetGatewayBackendChecker(nil) + h.SetExplainModeDefaultResolver(nil) + h.SetRepoAutoMergeEnabledResolver(nil) + h.SetKickLogSourceResolver(nil) + h.SetInferenceCallbacks(nil, nil) + h.SetSandboxConfig(cruxconfig.AgentSandboxConfig{}) + h.SetSandboxMutationBoundary(nil) + h.SetDeadSessionRecoveryOwner(true) + h.SetSpanStarter(nil) + out := log.String() + if n := strings.Count(out, "setter=SetAppAuth"); n != 1 { + t.Fatalf("SetAppAuth noted %d times, want once:\n%s", n, out) + } + if strings.Count(out, "served by the hive callback") != 16 { + t.Errorf("callback-carried setters = %d, want 16:\n%s", strings.Count(out, "served by the hive callback"), out) + } + if strings.Count(out, "not carried") != 0 { + t.Errorf("not-carried setters = %d, want 0:\n%s", strings.Count(out, "not carried"), out) + } + if strings.Count(out, "in-process-only") != 1 || !strings.Contains(out, "setter=SetSpanStarter") { + t.Errorf("in-process-only setters:\n%s", out) + } +} + +type sidecarMintIssuer struct { + enabled bool + calls []string +} + +func (f *sidecarMintIssuer) Enabled() bool { return f.enabled } +func (f *sidecarMintIssuer) MintAgentToken(agent, tier string) (string, error) { + f.calls = append(f.calls, agent+"@"+tier) + return "mint-" + agent, nil +} + +// The stored mint issuer and Bob resolvers are what the agent_mint and bob +// callback families serve: unset or disabled is the empty credential, as +// in-process when neither is configured. +func TestSidecarHost_MintAndBobAreServed(t *testing.T) { + h := newSidecarHost(newSidecarRuntimeFake(), slog.New(slog.NewTextHandler(io.Discard, nil))) + if h.Enabled() { + t.Fatal("mint enabled before SetAgentMint") + } + if tok, err := h.MintAgentToken("a", "t"); tok != "" || err != nil { + t.Fatalf("unset mint = %q, %v", tok, err) + } + if key, src := h.bobCredential(); key != "" || src != "" { + t.Fatalf("unset bob = %q/%q", key, src) + } + off := &sidecarMintIssuer{} + h.SetAgentMint(off) + if tok, err := h.MintAgentToken("a", "t"); tok != "" || err != nil || len(off.calls) != 0 || h.Enabled() { + t.Fatalf("disabled mint = %q, %v, calls %v", tok, err, off.calls) + } + on := &sidecarMintIssuer{enabled: true} + h.SetAgentMint(on) + if tok, err := h.MintAgentToken("a", "trusted"); tok != "mint-a" || err != nil || strings.Join(on.calls, ",") != "a@trusted" || !h.Enabled() { + t.Fatalf("mint = %q, %v, calls %v", tok, err, on.calls) + } + h.SetBobAPIKeyResolver(func() string { return "bob-key" }) + h.SetBobKeySourceResolver(func() string { return "env:BOB_API_KEY" }) + if key, src := h.bobCredential(); key != "bob-key" || src != "env:BOB_API_KEY" { + t.Fatalf("bob = %q/%q", key, src) + } + caps := (&boot{agentMgr: h}).hiveCallbackCapabilities() + if !caps.Configured(callback.CapabilityAgentMint) || !caps.Configured(callback.CapabilityBob) { + t.Fatal("capabilities not served") + } + resp, err := caps.MintAgentToken(context.Background(), callback.AgentMintRequest{Agent: "b", Tier: "t"}) + if err != nil || resp.Token != "mint-b" { + t.Fatalf("caps mint = %+v, %v", resp, err) + } + cred, err := caps.BobCredential(context.Background()) + if err != nil || cred.APIKey != "bob-key" { + t.Fatalf("caps bob = %+v, %v", cred, err) + } +} + +// The inference-route closures and the mutation boundary boot installs are +// what the callback's route and effect families reach; before boot sets +// them a route is dropped (and logged once per agent) and an effect runs +// unjournaled, the nil boundary in-process also tolerates. +func TestSidecarHost_InferenceRoutesAndMutationBoundary(t *testing.T) { + var log bytes.Buffer + h := newSidecarHost(newSidecarRuntimeFake(), slog.New(slog.NewTextHandler(&log, nil))) + ctx := context.Background() + h.SetInferenceRoute("a", "litellm", "m") + h.ClearInferenceRoute("a") + if !strings.Contains(log.String(), "before the proxy callbacks were installed") { + t.Fatalf("early route not logged:\n%s", log.String()) + } + ran := 0 + claim := effects.Claim{Repo: "o/r", Kind: effects.KindBranchPush, Target: "b"} + if _, err := h.Execute(ctx, claim, func(context.Context) (effects.Result, error) { ran++; return effects.Result{}, nil }); err != nil || ran != 1 { + t.Fatalf("effect before boundary = %v ran=%d", err, ran) + } + + var routes []string + h.SetInferenceCallbacks(func(agent, backend, model string) { routes = append(routes, "set "+agent+" "+backend+" "+model) }, func(agent string) { routes = append(routes, "clear "+agent) }) + h.SetInferenceRoute("a", "litellm", "m") + h.ClearInferenceRoute("a") + if strings.Join(routes, ";") != "set a litellm m;clear a" { + t.Fatalf("routes = %v", routes) + } + h.SetSandboxMutationBoundary(denyBoundary{}) + if _, err := h.Execute(ctx, claim, func(context.Context) (effects.Result, error) { ran++; return effects.Result{}, nil }); !errors.Is(err, effects.ErrDenied) || ran != 1 { + t.Fatalf("effect through boundary = %v ran=%d", err, ran) + } + // Served to the runtime through the callback capability set. + b := &boot{agentMgr: h} + caps := b.hiveCallbackCapabilities() + if !caps.Configured(callback.CapabilityInferenceRoutes) || !caps.Configured(callback.CapabilityEffects) { + t.Fatal("route/effect capabilities not served from the sidecar host") + } + if err := caps.SetInferenceRoute(ctx, "b", callback.InferenceRoute{Backend: "gw"}); err != nil || routes[len(routes)-1] != "set b gw " { + t.Fatalf("route via caps = %v %v", err, routes) + } + if _, err := caps.BeginEffect(ctx, callback.EffectClaim{Repo: "o/r", Kind: "k", Target: "t"}); !errors.Is(err, callback.ErrEffectDenied) { + t.Fatalf("effect via caps = %v", err) + } +} + +type denyBoundary struct{} + +func (denyBoundary) Execute(context.Context, effects.Claim, func(context.Context) (effects.Result, error)) (effects.Result, error) { + return effects.Result{}, effects.ErrDenied +} + +// The policy snapshot carries exactly the setters boot has called, resolved +// at read time, and follows later calls (a config reload re-sets them). +func TestSidecarHost_PolicySnapshot(t *testing.T) { + h := newSidecarHost(newSidecarRuntimeFake(), slog.New(slog.DiscardHandler)) + if p := h.policy(nil); p.Build != nil || p.ExplainModeDefault != nil || p.KickLogSource != nil || p.GatewayBackends != nil || + p.DeadSessionRecoveryOwnedElsewhere != nil || p.Sandbox != nil { + t.Fatalf("policy before any setter = %+v", p) + } + if !h.repoAutoMergeEnabled("o/r") { + t.Fatal("auto-merge before the resolver is set must be default-on") + } + explain := "on" + h.SetBuildInfo(func() (string, string) { return "v5.3", "deadbeef" }) + h.SetExplainModeDefaultResolver(func() string { return explain }) + h.SetKickLogSourceResolver(func() string { return "transcript" }) + h.SetGatewayBackendChecker(func(b string) bool { return b == "gw-a" }) + h.SetRepoAutoMergeEnabledResolver(func(repo string) bool { return repo == "o/yes" }) + h.SetDeadSessionRecoveryOwner(true) + h.SetSandboxConfig(cruxconfig.AgentSandboxConfig{Enabled: true, Image: "sandbox:1"}) + + p := h.policy(func() []string { return []string{"gw-a", "gw-b"} }) + if p.Build == nil || p.Build.Version != "v5.3" || p.Build.Commit != "deadbeef" { + t.Errorf("build = %+v", p.Build) + } + if p.ExplainModeDefault == nil || *p.ExplainModeDefault != "on" || p.KickLogSource == nil || *p.KickLogSource != "transcript" { + t.Errorf("explain/kick = %v/%v", p.ExplainModeDefault, p.KickLogSource) + } + if p.GatewayBackends == nil || len(*p.GatewayBackends) != 1 || (*p.GatewayBackends)[0] != "gw-a" { + t.Errorf("gateways = %v", p.GatewayBackends) + } + if p.DeadSessionRecoveryOwnedElsewhere == nil || !*p.DeadSessionRecoveryOwnedElsewhere { + t.Errorf("dead-session owner = %v", p.DeadSessionRecoveryOwnedElsewhere) + } + var sb cruxconfig.AgentSandboxConfig + if err := json.Unmarshal(p.Sandbox, &sb); err != nil || !sb.Enabled || sb.Image != "sandbox:1" { + t.Errorf("sandbox = %s (%v)", p.Sandbox, err) + } + if !h.repoAutoMergeEnabled("o/yes") || h.repoAutoMergeEnabled("o/no") { + t.Error("auto-merge resolver not used") + } + + explain = "off" + h.SetDeadSessionRecoveryOwner(false) + p = h.policy(nil) + if *p.ExplainModeDefault != "off" || *p.DeadSessionRecoveryOwnedElsewhere || p.GatewayBackends == nil || len(*p.GatewayBackends) != 0 { + t.Errorf("policy after re-set = %+v", p) + } +} + +// In sidecar mode bootAgents serves the hive callback endpoint before the +// rest of the wiring, with the capability set built from the boot; the +// in-process path never touches it. A callback that cannot bind exits. +func TestBootAgentsWith_SidecarServesHiveCallback(t *testing.T) { + t.Setenv(cruxconfig.ModeEnvVar, string(cruxconfig.ModeSidecar)) + f := newBootAgentsFake() + sc := newSidecarRuntimeFake() + f.deps.newRuntime = func(cfg cruxconfig.RuntimeConfig, _ ...cruxruntime.Option) (hostedRuntime, error) { + return hostRuntime(cfg, sc, nil) + } + var served callback.Capabilities + f.deps.startHiveCallback = func(_ context.Context, caps callback.Capabilities, _ *slog.Logger) error { + served = caps + return nil + } + cfg := bootAgentsConfig() + b := newBootAgentsBoot(t, f, cfg) + + b.bootAgentsWith(f.deps) + + if len(f.exitCodes) != 0 { + t.Fatalf("exit codes = %v", f.exitCodes) + } + if served == nil { + t.Fatal("hive callback endpoint not served in sidecar mode") + } + if f.loopsMgr != b.agentMgr { + t.Fatal("agent loops not started on the sidecar runtime") + } + if sc.observer == nil { + t.Fatal("boot installed no Host observer on the sidecar runtime") + } + if !strings.Contains(f.log.String(), "setter=SetSpanStarter") { + t.Fatalf("policy setter not noted:\n%s", f.log.String()) + } + // The Linear resolver is read lazily: bootSupervision sets it later. + if cred, err := served.LinearCredential(context.Background()); err != nil || !cred.Empty() { + t.Fatalf("Linear before bootSupervision = (%v, %v), want configured and empty", cred, err) + } + b.linearCredentialResolver = func() crux.LinearCredential { return crux.LinearCredential{AccessToken: "tok", APIKey: "key"} } + if cred, err := served.LinearCredential(context.Background()); err != nil || cred.AccessToken != "tok" || cred.APIKey != "key" { + t.Fatalf("Linear after bootSupervision = (%v, %v)", cred, err) + } + // The policy setters boot called are served as the policy snapshot. + ps, ok := served.(callback.PolicySource) + if !ok { + t.Fatalf("served capabilities %T have no policy half", served) + } + pol, err := ps.Policy(context.Background()) + if err != nil { + t.Fatalf("Policy: %v", err) + } + if pol.Build == nil || pol.ExplainModeDefault == nil || pol.GatewayBackends == nil || !pol.RepoAutoMerge || pol.Sandbox == nil { + t.Fatalf("boot policy snapshot missing setters: %+v", pol) + } +} + +func TestBootAgentsWith_SidecarCallbackBindFailureExits(t *testing.T) { + t.Setenv(cruxconfig.ModeEnvVar, string(cruxconfig.ModeSidecar)) + f := newBootAgentsFake() + f.deps.newRuntime = func(cfg cruxconfig.RuntimeConfig, _ ...cruxruntime.Option) (hostedRuntime, error) { + return hostRuntime(cfg, newSidecarRuntimeFake(), nil) + } + f.deps.startHiveCallback = func(context.Context, callback.Capabilities, *slog.Logger) error { + return errors.New("address in use") + } + b := newBootAgentsBoot(t, f, bootAgentsConfig()) + + b.bootAgentsWith(f.deps) + + if len(f.exitCodes) != 1 || f.exitCodes[0] != 1 { + t.Fatalf("exit codes = %v, want [1]", f.exitCodes) + } + if f.loopsMgr != nil { + t.Fatal("wiring installed after the callback endpoint failed to bind") + } + if !strings.Contains(f.log.String(), "hive callback endpoint unavailable") { + t.Fatalf("log missing the bind error:\n%s", f.log.String()) + } +} + +func TestBootAgentsWith_InProcessNeverServesHiveCallback(t *testing.T) { + f := newBootAgentsFake() + f.deps.startHiveCallback = func(context.Context, callback.Capabilities, *slog.Logger) error { + t.Fatal("hive callback served in in-process mode") + return nil + } + b := newBootAgentsBoot(t, f, bootAgentsConfig()) + b.bootAgentsWith(f.deps) + if len(f.exitCodes) != 0 || b.agentMgr == nil { + t.Fatalf("in-process boot broken: exits=%v mgr=%v", f.exitCodes, b.agentMgr) + } +} + +// The real startHiveCallback binds the socket named by $CRUX_HIVE_SOCKET and +// serves until the context ends. +func TestStartHiveCallback_BindsConfiguredSocket(t *testing.T) { + dir, err := os.MkdirTemp("", "hcb") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + sock := filepath.Join(dir, "hive.sock") + t.Setenv(callback.SocketEnvVar, sock) + t.Setenv(callback.ListenEnvVar, "") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var log bytes.Buffer + caps := (&boot{}).hiveCallbackCapabilities() + if err := startHiveCallback(ctx, caps, slog.New(slog.NewTextHandler(&log, nil))); err != nil { + t.Fatal(err) + } + c, err := callback.NewClient(sock, "") + if err != nil { + t.Fatal(err) + } + if cred, err := c.LinearCredential(ctx); err != nil || !cred.Empty() { + t.Fatalf("empty boot Linear = (%v, %v), want empty", cred, err) + } + if !strings.Contains(log.String(), "hive callback endpoint serving") { + t.Fatalf("serving not logged:\n%s", log.String()) + } + if err := startHiveCallback(ctx, caps, slog.Default()); err == nil { + t.Fatal("second bind on the same socket succeeded") + } +} + +// The callback's usage capability is served from boot and records into the +// proxy's ledger once bootProxy has built it; a record before that is +// dropped and logged once, never a typed-nil dereference. +func TestHiveCallbackCapabilities_UsageReachesTheInferenceSink(t *testing.T) { + var log bytes.Buffer + b := &boot{logger: slog.New(slog.NewTextHandler(&log, nil))} + caps := b.hiveCallbackCapabilities() + if !caps.Configured(callback.CapabilityUsage) { + t.Fatal("usage must be configured from boot") + } + ctx := context.Background() + for i := 0; i < 2; i++ { + if err := caps.RecordUsage(ctx, callback.UsageRecord{Agent: "a", Model: "m", Kind: callback.UsageKindGateway, InputTokens: 1, OutputTokens: 2}); err != nil { + t.Fatalf("record before proxy: %v", err) + } + } + if n := strings.Count(log.String(), "record dropped"); n != 1 { + t.Fatalf("early-record warning logged %d times, want once:\n%s", n, log.String()) + } + dir := t.TempDir() + b.setInferenceSink(tokens.NewInferenceSink(dir, b.logger)) + rec := callback.UsageRecord{Agent: "a", Model: "m", InputTokens: 10, OutputTokens: 20} + for _, kind := range []string{callback.UsageKindGateway, callback.UsageKindCopilot, callback.UsageKindCopilotCache} { + rec.Kind = kind + if err := caps.RecordUsage(ctx, rec); err != nil { + t.Fatalf("record %s: %v", kind, err) + } + } + files, _ := filepath.Glob(filepath.Join(dir, "*a.jsonl")) + if len(files) == 0 { + t.Fatalf("no ledger file written under %s", dir) + } +} diff --git a/src/cmd/hive/agent_runtime_supervise.go b/src/cmd/hive/agent_runtime_supervise.go new file mode 100644 index 0000000000..7aad99df44 --- /dev/null +++ b/src/cmd/hive/agent_runtime_supervise.go @@ -0,0 +1,171 @@ +package main + +import ( + "context" + "errors" + "log/slog" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api" + cruxconfig "github.com/hivecommons/hive/crux/config" + cruxruntime "github.com/hivecommons/hive/crux/runtime" + "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/fleetreport" +) + +// Failure semantics for a sidecar runtime (hivecommons/hive#14138 stage 4, +// documented in docs/crux-serve.md): +// +// - The governor keeps its loop; nothing in hive blocks on the socket. +// - Every agent-dependent call fails with crux.ErrRuntimeUnavailable and +// observer reads answer zero (crux/api/client). Kicks are REFUSED, not +// queued: the governor re-derives what is due on its next tick, so a +// replay buffer would only duplicate work after a long outage. +// - The hive raises the crux_runtime_unavailable fleet-report condition on +// the first failed probe and clears it on the first success; the +// dashboard renders it like every other runtime condition and +// /api/health carries crux_runtime. +// - Under CRUX_SUPERVISE=child (the default) hive spawns `crux serve` +// itself and restarts it with backoff; under external it only +// reconnects. + +// cruxRuntimeProbeInterval is how often the watcher checks the socket when +// no other call has done so. +const cruxRuntimeProbeInterval = 15 * time.Second + +// cruxRuntimeConditionKey scopes the fleet-report condition; one runtime +// per hive. +const cruxRuntimeConditionKey = "sidecar" + +// runtimeProber is the reachability surface of crux/api/client.Runtime, +// promoted through sidecarHost's embedded crux.Runtime. +type runtimeProber interface { + Status() api.Status + Client() *api.Client +} + +// cruxRuntimeSupervision is the stage-4 wiring bootAgentsWith installs for +// a sidecar runtime: the optional child supervisor and the probe loop. +type cruxRuntimeSupervision struct { + cfg cruxconfig.RuntimeConfig + prober runtimeProber + supervisor *cruxruntime.Supervisor + logger *slog.Logger + interval time.Duration + observe func(class, key, message, diagnostics string) + clear func(class, key string) +} + +// newCruxRuntimeSupervision prepares supervision for rt under cfg. It +// returns nil for a runtime that is not a sidecar client (in-process has +// nothing to supervise). +func newCruxRuntimeSupervision(cfg cruxconfig.RuntimeConfig, rt crux.Runtime, logger *slog.Logger) *cruxRuntimeSupervision { + if cfg.Mode != cruxconfig.ModeSidecar { + return nil + } + p, ok := rt.(runtimeProber) + if !ok { + // sidecarHost embeds the client as an interface, so the probe + // surface is not promoted; look through it. + sh, isHost := rt.(*sidecarHost) + if !isHost { + return nil + } + if p, ok = sh.Runtime.(runtimeProber); !ok { + return nil + } + } + s := &cruxRuntimeSupervision{cfg: cfg, prober: p, logger: logger, interval: cruxRuntimeProbeInterval, observe: fleetreport.Observe, clear: fleetreport.Clear} + if cfg.Supervise == cruxconfig.SuperviseChild { + s.supervisor = &cruxruntime.Supervisor{Config: cfg, Logger: logger} + } + return s +} + +// start runs the supervisor (when configured) and the probe loop until ctx +// ends. A missing binary under child supervision is loud but not fatal: the +// hive falls back to reconnect-only, exactly as external supervision. +func (s *cruxRuntimeSupervision) start(ctx context.Context) { + if s.supervisor != nil { + go func() { + err := s.supervisor.Run(ctx) + if errors.Is(err, cruxruntime.ErrBinaryNotFound) { + s.logger.Error("crux serve child supervision disabled: binary not found; set "+cruxconfig.BinaryEnvVar+" or "+cruxconfig.SuperviseEnvVar+"=external", + "binary", s.cfg.Binary, "error", err) + } else if err != nil { + s.logger.Error("crux serve supervisor stopped", "error", err) + } + }() + } + go s.watch(ctx) +} + +// watch probes the runtime and keeps the fleet-report condition in step +// with the client's reachability. +func (s *cruxRuntimeSupervision) watch(ctx context.Context) { + t := time.NewTicker(s.interval) + defer t.Stop() + down := false + for { + s.probe(ctx) + down = s.reconcile(down) + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + +// probe makes one health call so the client's Status reflects the socket +// even when no agent traffic has exercised it. +func (s *cruxRuntimeSupervision) probe(ctx context.Context) { + pctx, cancel := context.WithTimeout(ctx, s.interval) + defer cancel() + _, _ = s.prober.Client().Health(pctx) +} + +// reconcile raises or clears the condition on a transition and returns the +// new down state. +func (s *cruxRuntimeSupervision) reconcile(wasDown bool) bool { + st := s.prober.Status() + switch { + case !st.Reachable && !wasDown: + s.logger.Error("crux runtime unavailable: crux server unreachable; kicks refused until it returns", + "server", s.cfg.Server, "supervise", s.cfg.Supervise, "error", st.LastError) + s.observe(fleetreport.ClassCruxRuntimeUnavailable, cruxRuntimeConditionKey, + "crux runtime unavailable: crux server unreachable at "+s.cfg.Server, "supervise="+string(s.cfg.Supervise)+" error="+st.LastError) + case st.Reachable && wasDown: + s.logger.Info("crux runtime reachable again", "server", s.cfg.Server, "outage", time.Since(st.Since).Round(time.Second)) + s.clear(fleetreport.ClassCruxRuntimeUnavailable, cruxRuntimeConditionKey) + } + return !st.Reachable +} + +// health is the /api/health crux_runtime body. +func (s *cruxRuntimeSupervision) health() dashboard.CruxRuntimeHealth { + st := s.prober.Status() + h := dashboard.CruxRuntimeHealth{ + Mode: string(s.cfg.Mode), + Reachable: st.Reachable, + LastError: st.LastError, + Since: st.Since, + Supervise: string(s.cfg.Supervise), + } + if s.supervisor != nil { + c := s.supervisor.State() + h.Child = &dashboard.CruxRuntimeChild{Running: c.Running, Ready: c.Ready, PID: c.PID, Starts: c.Starts, LastExit: c.LastExit, Binary: c.Binary} + } + return h +} + +// cruxRuntimeHealthFunc is the Dependencies.CruxRuntimeHealth hook: nil for +// in-process (always reachable, nothing to report), else the supervision's +// view. +func cruxRuntimeHealthFunc(s *cruxRuntimeSupervision) func() dashboard.CruxRuntimeHealth { + if s == nil { + return nil + } + return s.health +} diff --git a/src/cmd/hive/agent_runtime_supervise_test.go b/src/cmd/hive/agent_runtime_supervise_test.go new file mode 100644 index 0000000000..b0805cbdf8 --- /dev/null +++ b/src/cmd/hive/agent_runtime_supervise_test.go @@ -0,0 +1,246 @@ +package main + +import ( + "context" + "errors" + "log/slog" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/api/client" + cruxconfig "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/cruxtest" + cruxruntime "github.com/hivecommons/hive/crux/runtime" + "github.com/hivecommons/hive/crux/serve" + "github.com/hivecommons/hive/internal/testutil" + "github.com/hivecommons/hive/pkg/fleetreport" +) + +// conditionLog records fleet-report Observe/Clear calls. +type conditionLog struct { + mu sync.Mutex + raised []string + cleared []string +} + +func (c *conditionLog) observe(class, key, _, _ string) { + c.mu.Lock() + defer c.mu.Unlock() + c.raised = append(c.raised, class+"/"+key) +} + +func (c *conditionLog) clear(class, key string) { + c.mu.Lock() + defer c.mu.Unlock() + c.cleared = append(c.cleared, class+"/"+key) +} + +func (c *conditionLog) counts() (int, int) { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.raised), len(c.cleared) +} + +// liveServe is a `crux serve` over a recorder, reachable at URL. +func liveServe(t *testing.T) (*cruxtest.Recorder, *httptest.Server) { + t.Helper() + rec := cruxtest.NewRecorder() + srv := serve.New(rec) + srv.Token = "tok" + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(func() { + // httptest.Server.Close waits for in-flight requests; a runtime + // holding a long-lived request open would hang it until the go test + // deadline. Drop the connections first and bound the wait. + ts.CloseClientConnections() + done := make(chan struct{}) + go func() { ts.Close(); close(done) }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Error("live crux serve did not shut down within 10s") + } + }) + return rec, ts +} + +func newSupervisionFor(t *testing.T, server string, supervise cruxconfig.Supervision) (*cruxRuntimeSupervision, *conditionLog, *syncLogBuffer) { + t.Helper() + cfg := cruxconfig.RuntimeConfig{Mode: cruxconfig.ModeSidecar, Server: server, Token: "tok", Supervise: supervise, Binary: filepath.Join(t.TempDir(), "no-such-crux")} + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + rt, err := client.New(cfg.Server, cfg.Token) + if err != nil { + t.Fatal(err) + } + rt.Timeout = 2 * time.Second + // The supervisor logs from a background goroutine; a bare bytes.Buffer + // races with the test's log.String() reads under -race. + var log syncLogBuffer + logger := slog.New(slog.NewTextHandler(&log, nil)) + hr, err := hostRuntime(cfg, rt, logger) + if err != nil { + t.Fatal(err) + } + s := newCruxRuntimeSupervision(cfg, hr, logger) + if s == nil { + t.Fatal("no supervision for a sidecar client") + } + cl := &conditionLog{} + s.observe, s.clear = cl.observe, cl.clear + s.interval = 20 * time.Millisecond + return s, cl, &log +} + +func TestCruxRuntimeSupervision_NilOutsideSidecar(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stderr, nil)) + if s := newCruxRuntimeSupervision(cruxconfig.RuntimeConfig{Mode: cruxconfig.ModeInProcess}, cruxtest.NewRecorder(), logger); s != nil { + t.Fatal("in-process must not be supervised") + } + // A sidecar runtime without the client's probe surface (test fakes) is + // left alone rather than panicking. + cfg := cruxconfig.RuntimeConfig{Mode: cruxconfig.ModeSidecar, Server: "127.0.0.1:1", Supervise: cruxconfig.SuperviseChild} + if s := newCruxRuntimeSupervision(cfg, newSidecarHost(newSidecarRuntimeFake(), logger), logger); s != nil { + t.Fatal("non-prober runtime supervised") + } + if cruxRuntimeHealthFunc(nil) != nil { + t.Fatal("health hook for nil supervision") + } +} + +func TestCruxRuntimeSupervision_ConditionFollowsReachability(t *testing.T) { + _, ts := liveServe(t) + s, cl, log := newSupervisionFor(t, ts.URL, cruxconfig.SuperviseExternal) + if s.supervisor != nil { + t.Fatal("external supervision spawned a child supervisor") + } + ctx := context.Background() + + s.probe(ctx) + if down := s.reconcile(false); down { + t.Fatal("live server reported down") + } + h := s.health() + if !h.Reachable || h.Mode != "sidecar" || h.Supervise != "external" || h.Child != nil { + t.Fatalf("health up = %+v", h) + } + + ts.Close() + s.probe(ctx) + if down := s.reconcile(false); !down { + t.Fatal("closed server reported up") + } + // Staying down does not re-raise. + s.probe(ctx) + s.reconcile(true) + if r, c := cl.counts(); r != 1 || c != 0 || cl.raised[0] != fleetreport.ClassCruxRuntimeUnavailable+"/"+cruxRuntimeConditionKey { + t.Fatalf("conditions after outage = %v / %v", cl.raised, cl.cleared) + } + h = s.health() + if h.Reachable || h.LastError == "" || h.Since.IsZero() { + t.Fatalf("health down = %+v", h) + } + if !strings.Contains(log.String(), "kicks refused until it returns") { + t.Fatalf("outage not logged:\n%s", log.String()) + } + // Kicks through the hosted runtime are refused with the sentinel. + if err := s.prober.(crux.Runtime).SendKick("a", "m"); !errors.Is(err, crux.ErrRuntimeUnavailable) { + t.Fatalf("SendKick during outage = %v", err) + } + // A transition back up clears exactly once. + s2, cl2, _ := newSupervisionFor(t, liveServeURL(t), cruxconfig.SuperviseExternal) + s2.probe(ctx) + if down := s2.reconcile(true); down { + t.Fatal("recovered server still down") + } + if r, c := cl2.counts(); r != 0 || c != 1 { + t.Fatalf("conditions after recovery = %v / %v", cl2.raised, cl2.cleared) + } +} + +func liveServeURL(t *testing.T) string { + t.Helper() + _, ts := liveServe(t) + return ts.URL +} + +func TestCruxRuntimeSupervision_StartWatchesAndReportsMissingBinary(t *testing.T) { + s, cl, log := newSupervisionFor(t, "127.0.0.1:1", cruxconfig.SuperviseChild) + if s.supervisor == nil { + t.Fatal("child supervision without a supervisor") + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s.start(ctx) + testutil.Eventually(t, 5*time.Second, func() bool { + r, _ := cl.counts() + return r == 1 && strings.Contains(log.String(), "child supervision disabled: binary not found") + }, "watcher did not raise the condition / supervisor did not report the missing binary:\n%s", log.String()) + h := s.health() + if h.Reachable || h.Supervise != "child" || h.Child == nil || h.Child.Running { + t.Fatalf("health = %+v child=%+v", h, h.Child) + } + cancel() +} + +func TestBootAgentsWith_SidecarStartsSupervision(t *testing.T) { + t.Setenv(cruxconfig.ModeEnvVar, string(cruxconfig.ModeSidecar)) + t.Setenv(cruxconfig.SuperviseEnvVar, string(cruxconfig.SuperviseExternal)) + _, ts := liveServe(t) + t.Setenv(cruxconfig.ServerEnvVar, ts.URL) + t.Setenv(cruxconfig.ServeTokenEnvVar, "tok") + f := newBootAgentsFake() + f.deps.newRuntime = func(cfg cruxconfig.RuntimeConfig, opts ...cruxruntime.Option) (hostedRuntime, error) { + hr, err := newHostedRuntime(cfg, opts...) + if err == nil { + // Cleanups run LIFO, so this unsubscribes (and joins the stream + // goroutine) before liveServe's httptest server is closed. + t.Cleanup(func() { + if sh, ok := hr.(*sidecarHost); ok { + if src, ok := sh.Runtime.(eventObservable); ok { + src.SetEventObserver(nil) + } + } + }) + } + return hr, err + } + f.deps.startHiveCallback = func(context.Context, callback.Capabilities, *slog.Logger) error { return nil } + b := newBootAgentsBoot(t, f, bootAgentsConfig()) + + b.bootAgentsWith(f.deps) + + if len(f.exitCodes) != 0 { + t.Fatalf("exit codes = %v", f.exitCodes) + } + if f.supervised == nil || b.cruxSupervision != f.supervised { + t.Fatal("sidecar boot did not hand its supervision to the seam") + } + if f.supervised.supervisor != nil || f.supervised.cfg.Supervise != cruxconfig.SuperviseExternal { + t.Fatalf("CRUX_SUPERVISE=external not honoured: %+v", f.supervised.cfg) + } + hf := cruxRuntimeHealthFunc(b.cruxSupervision) + if hf == nil { + t.Fatal("no /api/health crux_runtime hook") + } + if h := hf(); h.Mode != "sidecar" || !h.Reachable { + t.Fatalf("health = %+v", h) + } +} + +func TestBootAgentsWith_InProcessHasNoSupervision(t *testing.T) { + f := newBootAgentsFake() + b := newBootAgentsBoot(t, f, bootAgentsConfig()) + b.bootAgentsWith(f.deps) + if f.supervised != nil || b.cruxSupervision != nil || cruxRuntimeHealthFunc(b.cruxSupervision) != nil { + t.Fatal("in-process boot started sidecar supervision") + } +} diff --git a/src/cmd/hive/approvalhook.go b/src/cmd/hive/approvalhook.go index 5a93338536..a0e1a5cf76 100644 --- a/src/cmd/hive/approvalhook.go +++ b/src/cmd/hive/approvalhook.go @@ -5,15 +5,15 @@ import ( "errors" "fmt" + "github.com/hivecommons/hive/crux/toolapprove" "github.com/hivecommons/hive/pkg/hooks" - "github.com/hivecommons/hive/pkg/toolapprove" ) // hookApprovalAdapter satisfies hooks.ApprovalQueue over toolapprove.Inbox. type hookApprovalAdapter struct { inbox *toolapprove.Inbox // acmmLevel is the hive's level, recorded on the queued item so the panel - // can flag an approval parked on a full-autonomy hive as a probable policy + // can flag an approval parked on an L6 (Autonomous) hive as a probable policy // bug (throughput-contract requirement 4) exactly as it does for the // sweep-produced ones. acmmLevel int diff --git a/src/cmd/hive/approvalhook_test.go b/src/cmd/hive/approvalhook_test.go index 5ea94b790c..8c628a7dd0 100644 --- a/src/cmd/hive/approvalhook_test.go +++ b/src/cmd/hive/approvalhook_test.go @@ -8,8 +8,8 @@ import ( "path/filepath" "testing" + "github.com/hivecommons/hive/crux/toolapprove" "github.com/hivecommons/hive/pkg/hooks" - "github.com/hivecommons/hive/pkg/toolapprove" ) func hookApprovalRequest() hooks.ApprovalRequest { diff --git a/src/cmd/hive/approvalparity_test.go b/src/cmd/hive/approvalparity_test.go index 848cefb601..7e4a7fa890 100644 --- a/src/cmd/hive/approvalparity_test.go +++ b/src/cmd/hive/approvalparity_test.go @@ -14,8 +14,8 @@ import ( "context" "testing" + "github.com/hivecommons/hive/crux/toolapprove" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/toolapprove" ) func parityDesk(t *testing.T, cfg *config.Config) *toolapprove.Desk { diff --git a/src/cmd/hive/approvalwire.go b/src/cmd/hive/approvalwire.go index c02ffee13c..ea9e02115b 100644 --- a/src/cmd/hive/approvalwire.go +++ b/src/cmd/hive/approvalwire.go @@ -5,11 +5,16 @@ import ( "errors" "log/slog" + "github.com/hivecommons/hive/crux/toolapprove" "github.com/hivecommons/hive/pkg/config" ghpkg "github.com/hivecommons/hive/pkg/github" - "github.com/hivecommons/hive/pkg/toolapprove" + "github.com/hivecommons/hive/pkg/ioscan" ) +// The tool-approval desk scans payload text through pkg/ioscan, wired here +// because pkg/toolapprove sits inside the Crux ring and cannot import it. +func init() { toolapprove.SetDefaultTextScanner(ioscan.ToolApproveScanner{}) } + // buildApprovalDesk constructs the desk and its durable operator inbox from // config. // diff --git a/src/cmd/hive/approvalwire_test.go b/src/cmd/hive/approvalwire_test.go index 03feed11bc..aa995d54b3 100644 --- a/src/cmd/hive/approvalwire_test.go +++ b/src/cmd/hive/approvalwire_test.go @@ -4,8 +4,8 @@ import ( "context" "testing" + "github.com/hivecommons/hive/crux/toolapprove" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/toolapprove" ) func TestApprovalDeskAllowsLegacyOperationNilDesk(t *testing.T) { diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index 184df8eba9..fb86759dbc 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -8,8 +8,12 @@ import ( "sync/atomic" "time" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/proxy" + "github.com/hivecommons/hive/crux/toolapprove" + "github.com/hivecommons/hive/crux/trajectory" + "github.com/hivecommons/hive/crux/watchdog" "github.com/hivecommons/hive/pkg/advisory" - "github.com/hivecommons/hive/pkg/agent" "github.com/hivecommons/hive/pkg/beads" "github.com/hivecommons/hive/pkg/claims" "github.com/hivecommons/hive/pkg/config" @@ -31,8 +35,8 @@ import ( "github.com/hivecommons/hive/pkg/mergelane" "github.com/hivecommons/hive/pkg/notify" "github.com/hivecommons/hive/pkg/planning" + "github.com/hivecommons/hive/pkg/priorart" "github.com/hivecommons/hive/pkg/promptsrc" - "github.com/hivecommons/hive/pkg/proxy" "github.com/hivecommons/hive/pkg/questionclose" "github.com/hivecommons/hive/pkg/retro" "github.com/hivecommons/hive/pkg/review/eventdispatch" @@ -41,9 +45,6 @@ import ( "github.com/hivecommons/hive/pkg/snapshot" "github.com/hivecommons/hive/pkg/startrec" "github.com/hivecommons/hive/pkg/tokens" - "github.com/hivecommons/hive/pkg/toolapprove" - "github.com/hivecommons/hive/pkg/trajectory" - "github.com/hivecommons/hive/pkg/watchdog" ) // boot is the state main() used to keep in ~40 local variables, made explicit @@ -85,6 +86,9 @@ type boot struct { // sweep on the current client. adoptGitHubClient swaps both on every // rebuild (#9621); see github_client_provider.go. ghClientLive atomic.Pointer[githubClientRef] + // priorArtSearcher is the prior-art gate's Searcher, shared by the + // scheduler's park and the issue relay's filing verdict (#13879). + priorArtSearcher atomic.Pointer[priorart.Searcher] // forgeState is the per-repo GitHub state cache the governor work // source reads from when `forgestate.enabled` is on (#13375). Built @@ -117,7 +121,7 @@ type boot struct { repoTargetIssueMessage func() string advisoryIssues map[string]int advisoryStore *advisory.Store - agentMgr *agent.Manager + agentMgr hostedRuntime approvalDesk *toolapprove.Desk approvalInbox *toolapprove.Inbox dashChat *dashchat.Bot @@ -186,11 +190,11 @@ type boot struct { githubAppState github.AppAuthState githubAppRequired bool policyDirPath string - projectCtx agent.ProjectContext + projectCtx crux.ProjectContext mutationStats *effects.Recorder // bootAgents - agentMinter agent.AgentMintIssuer + agentMinter crux.AgentMintIssuer // bootState saved *snapshot.PersistedState @@ -213,13 +217,18 @@ type boot struct { // bootSupervision quotaReadingPublisher *rotation.Manager - linearCredentialResolver func() agent.LinearCredential + linearCredentialResolver func() crux.LinearCredential // bootWatchers configWatcher *config.Watcher // bootProxy githubProxy *proxy.GitHubProxy + // inferenceSink is the one per-agent inference usage ledger; the proxy + // records into it in-process and the hive callback's usage capability + // records into it for a sidecar runtime (#14138 stage 4). + inferenceSink *tokens.InferenceSink + inferenceSinkMu sync.Mutex // bootHeartbeat reporterName string @@ -238,6 +247,7 @@ type boot struct { lastSupersessionSweep time.Time lastSharedCIRerunSweep time.Time lastRedMainSweep time.Time + lastBlockedIssuesSweep time.Time lastIssueUnparkSweep time.Time lastClosedIssueSweep time.Time // lastClarityResumeSweep gates runClarityGateResumeSweepIfDue @@ -280,6 +290,9 @@ type boot struct { // read and sweepTrigger their webhook trigger over it (#13377). forgeView *github.ForgeStateView sweepTrigger *automerge.SweepTrigger + // cruxSupervision is the sidecar runtime's child supervisor and probe + // loop (hivecommons/hive#14138 stage 4); nil in-process. + cruxSupervision *cruxRuntimeSupervision } // deferStack stands in for the `defer` statements that used to sit in diff --git a/src/cmd/hive/boot_agent_dashboard_token_test.go b/src/cmd/hive/boot_agent_dashboard_token_test.go index f82aeaa240..dad79972dd 100644 --- a/src/cmd/hive/boot_agent_dashboard_token_test.go +++ b/src/cmd/hive/boot_agent_dashboard_token_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/taskmcp" + "github.com/hivecommons/hive/crux/taskmcp" ) // #8348: the task MCP URL and the project context the boot hands to the agent diff --git a/src/cmd/hive/boot_agents_deps.go b/src/cmd/hive/boot_agents_deps.go index 4bb3aaaf8c..98fd2d8914 100644 --- a/src/cmd/hive/boot_agents_deps.go +++ b/src/cmd/hive/boot_agents_deps.go @@ -3,8 +3,14 @@ package main import ( "context" "log/slog" + "os" + + "github.com/hivecommons/hive/crux/api/callback" + + "github.com/hivecommons/hive/crux" + cruxconfig "github.com/hivecommons/hive/crux/config" + cruxruntime "github.com/hivecommons/hive/crux/runtime" - "github.com/hivecommons/hive/pkg/agent" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/github/automerge" @@ -14,7 +20,7 @@ import ( // bootAgentsDeps carries the effects of bootAgents that outlive the call // (#7571, step 2): every goroutine the phase starts and the two collaborators // that touch /data or a signing key. The manager itself is constructed for -// real — agent.NewManager is pure — so a test can assert on the wiring the +// real — in-process construction is pure — so a test can assert on the wiring the // phase installs on it (resolvers, hooks, App auth, minter) while the loops // that would otherwise poll tmux, token caches and request directories for // the rest of the test binary's life are recorded instead of started. @@ -23,7 +29,7 @@ type bootAgentsDeps struct { // watchdog and the Copilot session refresh. All three are unconditional // (see the comments at the call site for why each must not gate on App // state at boot). - startAgentLoops func(ctx context.Context, mgr *agent.Manager) + startAgentLoops func(ctx context.Context, mgr crux.Runtime) // prepareRequestDirs creates the agent-facing request queues; // startTokenAccessAudit folds the per-UID wrapper events into the audit // log. Both run regardless of App state. @@ -39,9 +45,24 @@ type bootAgentsDeps struct { // its channel closes once the sweep loop has exited. startSelfAuthoredSweep func(ctx context.Context, c *github.Client, maxMerges int, acmmAllowed bool, acmmLevel *int, opts automerge.Options) <-chan struct{} // buildMinter is buildAgentMinter; only consulted when mint.enabled. - buildMinter func(cfg *config.Config, logger *slog.Logger) (agent.AgentMintIssuer, error) - // startPermissionsWatcher is agent.StartPermissionsWatcher. + buildMinter func(cfg *config.Config, logger *slog.Logger) (crux.AgentMintIssuer, error) + // startPermissionsWatcher is cruxruntime.StartPermissionsWatcher. startPermissionsWatcher func(logger *slog.Logger) + // newRuntime is newHostedRuntime: the runtime selected by CRUX_MODE. + // Fakes keep the real in-process runtime (construction is pure) and + // substitute only when asserting the unavailable-mode exit. + newRuntime func(cfg cruxconfig.RuntimeConfig, opts ...cruxruntime.Option) (hostedRuntime, error) + // startHiveCallback serves the hive capabilities to a sidecar runtime + // over the callback endpoint (crux/api/callback). Only called when + // CRUX_MODE=sidecar; fakes record the capability set. + startHiveCallback func(ctx context.Context, caps callback.Capabilities, logger *slog.Logger) error + // superviseRuntime starts the sidecar's child supervisor and probe + // loop (agent_runtime_supervise.go; hivecommons/hive#14138 stage 4). + // Only called when CRUX_MODE=sidecar; fakes record it. + superviseRuntime func(ctx context.Context, s *cruxRuntimeSupervision) + // exit terminates the process when no runtime can be built. os.Exit in + // production; fakes record it. + exit func(code int) } // requestRelays is the authorization the five request watchers are started @@ -57,13 +78,16 @@ type requestRelays struct { // the relays run on (the auto-merge engine's RelayMergeRefusal). nil // installs no gate. mergePolicy func(*github.Client) github.MergeRequestPolicyGate - pushBranch github.PushBranchRequestAuthorizer - logger *slog.Logger + // forgeMerge routes a relayed merge for a repo on GitLab/Gitea through + // that forge's merger. nil keeps every repo on the GitHub path. + forgeMerge github.ForgeMergeSelector + pushBranch github.PushBranchRequestAuthorizer + logger *slog.Logger } func defaultBootAgentsDeps() bootAgentsDeps { return bootAgentsDeps{ - startAgentLoops: func(ctx context.Context, mgr *agent.Manager) { + startAgentLoops: func(ctx context.Context, mgr crux.Runtime) { go mgr.StartAgentTokenRefresh(ctx) go mgr.StartCredentialWatchdog(ctx) go mgr.StartCopilotSessionRefresh(ctx) @@ -77,6 +101,9 @@ func defaultBootAgentsDeps() bootAgentsDeps { if r.mergePolicy != nil { c.SetMergeRequestPolicyGate(r.mergePolicy(c)) } + if r.forgeMerge != nil { + c.SetForgeMergeSelector(r.forgeMerge) + } return joinDone( startRequestWatchers(ctx, requestwatch.New(c, r.prOpen, r.issueOpen, r.holdLabel, nil), r.logger), c.StartReviewRequestWatcher(ctx, r.review, nil), @@ -87,13 +114,17 @@ func defaultBootAgentsDeps() bootAgentsDeps { startSelfAuthoredSweep: func(ctx context.Context, c *github.Client, maxMerges int, acmmAllowed bool, acmmLevel *int, opts automerge.Options) <-chan struct{} { return automerge.StartSelfAuthoredAutoMergeSweep(ctx, c, maxMerges, acmmAllowed, acmmLevel, opts) }, - buildMinter: func(cfg *config.Config, logger *slog.Logger) (agent.AgentMintIssuer, error) { + buildMinter: func(cfg *config.Config, logger *slog.Logger) (crux.AgentMintIssuer, error) { m, err := buildAgentMinter(cfg, logger) if err != nil { return nil, err } return m, nil }, - startPermissionsWatcher: func(logger *slog.Logger) { go agent.StartPermissionsWatcher(logger) }, + startPermissionsWatcher: func(logger *slog.Logger) { go cruxruntime.StartPermissionsWatcher(logger) }, + newRuntime: newHostedRuntime, + startHiveCallback: startHiveCallback, + superviseRuntime: func(ctx context.Context, s *cruxRuntimeSupervision) { s.start(ctx) }, + exit: os.Exit, } } diff --git a/src/cmd/hive/boot_agents_deps_test.go b/src/cmd/hive/boot_agents_deps_test.go index 1b7f6bc7ef..6beb6d72c9 100644 --- a/src/cmd/hive/boot_agents_deps_test.go +++ b/src/cmd/hive/boot_agents_deps_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "log/slog" "net/http" "net/http/httptest" @@ -12,7 +13,10 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux" + cruxconfig "github.com/hivecommons/hive/crux/config" + cruxruntime "github.com/hivecommons/hive/crux/runtime" + "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/github/automerge" @@ -27,6 +31,9 @@ import ( type bootAgentsFake struct { deps bootAgentsDeps log bytes.Buffer + // supervised is the sidecar supervision bootAgentsWith handed to the + // superviseRuntime seam; nil in-process. + supervised *cruxRuntimeSupervision // relayMu guards the relay/sweep fields: since #9621 the relay // supervisor starts them on its own goroutine. sweepStarted receives once @@ -35,7 +42,7 @@ type bootAgentsFake struct { sweepStarted chan struct{} relayStarts int - loopsMgr *agent.Manager + loopsMgr crux.Runtime dirsPrepared bool auditStarted bool relays *requestRelays @@ -46,12 +53,13 @@ type bootAgentsFake struct { sweepLevel *int minterCalls int permWatcherRuns int + exitCodes []int } func newBootAgentsFake() *bootAgentsFake { f := &bootAgentsFake{sweepStarted: make(chan struct{}, fakeRelayStartBuffer)} f.deps = bootAgentsDeps{ - startAgentLoops: func(_ context.Context, mgr *agent.Manager) { f.loopsMgr = mgr }, + startAgentLoops: func(_ context.Context, mgr crux.Runtime) { f.loopsMgr = mgr }, prepareRequestDirs: func(*slog.Logger) { f.dirsPrepared = true }, startTokenAccessAudit: func(context.Context, *slog.Logger) { f.auditStarted = true }, startRequestRelays: func(ctx context.Context, c *github.Client, r requestRelays) <-chan struct{} { @@ -68,11 +76,16 @@ func newBootAgentsFake() *bootAgentsFake { f.sweepStarted <- struct{}{} return ctx.Done() }, - buildMinter: func(*config.Config, *slog.Logger) (agent.AgentMintIssuer, error) { + buildMinter: func(*config.Config, *slog.Logger) (crux.AgentMintIssuer, error) { f.minterCalls++ return nil, errors.New("no signing key in test") }, startPermissionsWatcher: func(*slog.Logger) { f.permWatcherRuns++ }, + // Construction is pure, so the real in-process runtime is used: the + // tests assert on the wiring installed on it. + newRuntime: newHostedRuntime, + superviseRuntime: func(_ context.Context, s *cruxRuntimeSupervision) { f.supervised = s }, + exit: func(code int) { f.exitCodes = append(f.exitCodes, code) }, } return f } @@ -249,7 +262,7 @@ func TestBootAgentsWith_MintFailureIsWarnOnly(t *testing.T) { func TestBootAgentsWith_MintSuccessAttachesIssuer(t *testing.T) { f := newBootAgentsFake() - f.deps.buildMinter = func(*config.Config, *slog.Logger) (agent.AgentMintIssuer, error) { + f.deps.buildMinter = func(*config.Config, *slog.Logger) (crux.AgentMintIssuer, error) { f.minterCalls++ return fakeMintIssuer{}, nil } @@ -274,7 +287,7 @@ func TestDefaultBootAgentsDeps_IsFullyWired(t *testing.T) { d := defaultBootAgentsDeps() if d.startAgentLoops == nil || d.prepareRequestDirs == nil || d.startTokenAccessAudit == nil || d.startRequestRelays == nil || d.startSelfAuthoredSweep == nil || d.buildMinter == nil || - d.startPermissionsWatcher == nil { + d.startPermissionsWatcher == nil || d.superviseRuntime == nil { t.Fatal("defaultBootAgentsDeps left a seam nil; production boot would panic") } // buildMinter's adapter must not smuggle a typed-nil pointer into the @@ -285,3 +298,30 @@ func TestDefaultBootAgentsDeps_IsFullyWired(t *testing.T) { t.Fatalf("buildMinter(no key) = (%v, %v), want (nil, error)", issuer, err) } } + +// A runtime mode that cannot be served is a boot error: the phase exits +// before installing any wiring rather than silently running in-process +// (#14138 stage 1; the sidecar client is stage 3). +func TestBootAgentsWith_UnavailableRuntimeModeExits(t *testing.T) { + f := newBootAgentsFake() + f.deps.newRuntime = func(cfg cruxconfig.RuntimeConfig, _ ...cruxruntime.Option) (hostedRuntime, error) { + return nil, fmt.Errorf("mode %q: %w", cfg.Mode, cruxruntime.ErrModeUnavailable) + } + cfg := bootAgentsConfig() + b := newBootAgentsBoot(t, f, cfg) + + b.bootAgentsWith(f.deps) + + if len(f.exitCodes) != 1 || f.exitCodes[0] != 1 { + t.Fatalf("exit codes = %v, want [1]", f.exitCodes) + } + if b.agentMgr != nil { + t.Fatal("agent manager constructed despite unavailable mode") + } + if f.loopsMgr != nil || f.dirsPrepared || f.permWatcherRuns != 0 { + t.Fatal("wiring installed after a failed runtime construction") + } + if !strings.Contains(f.log.String(), "crux runtime unavailable") { + t.Fatalf("log missing the unavailable-mode error:\n%s", f.log.String()) + } +} diff --git a/src/cmd/hive/boot_collectors_deps_test.go b/src/cmd/hive/boot_collectors_deps_test.go index a9750241eb..32220cb2e7 100644 --- a/src/cmd/hive/boot_collectors_deps_test.go +++ b/src/cmd/hive/boot_collectors_deps_test.go @@ -11,7 +11,9 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/beads" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" @@ -78,7 +80,7 @@ func newBootCollectorsBoot(t *testing.T, cfg *config.Config) *boot { b, _ := newDepsTestBoot(t, cfg) b.gov = governor.New(cfg.Governor, cfg.EnabledAgents(), b.logger) b.sched = scheduler.New(cfg, b.logger) - b.agentMgr = agent.NewManager(cfg.Agents, b.logger, agent.ProjectContext{}) + b.agentMgr = cruxtest.NewRuntime(cfg.Agents, b.logger, crux.ProjectContext{}) b.ghClient = fakeGitHubClient(t) b.dashSrv = dashboard.NewServer(0, b.logger) b.beadStores = map[string]*beads.Store{} diff --git a/src/cmd/hive/boot_dashboard_api_deps_test.go b/src/cmd/hive/boot_dashboard_api_deps_test.go index 2b6a47713b..076ef3cdc0 100644 --- a/src/cmd/hive/boot_dashboard_api_deps_test.go +++ b/src/cmd/hive/boot_dashboard_api_deps_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/github" @@ -72,7 +74,7 @@ func newBootDashboardAPIBoot(t *testing.T, f *bootDashboardAPIFake, cfg *config. logger: logger, gov: governor.New(cfg.Governor, cfg.Agents, logger), sched: scheduler.New(cfg, logger), - agentMgr: agent.NewManager(cfg.Agents, logger, agent.ProjectContext{}), + agentMgr: cruxtest.NewRuntime(cfg.Agents, logger, crux.ProjectContext{}), dashSrv: dashboard.NewServer(0, logger), advisoryIssues: map[string]int{}, appAuthFailure: "no App credentials in test", diff --git a/src/cmd/hive/boot_dashboard_deps_test.go b/src/cmd/hive/boot_dashboard_deps_test.go index dc8a48e704..8d9ba073bb 100644 --- a/src/cmd/hive/boot_dashboard_deps_test.go +++ b/src/cmd/hive/boot_dashboard_deps_test.go @@ -6,7 +6,9 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/beads" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" @@ -60,7 +62,7 @@ func newBootDashboardBoot(t *testing.T, cfg *config.Config) *boot { t.Helper() b, _ := newDepsTestBoot(t, cfg) b.sched = scheduler.New(cfg, b.logger) - b.agentMgr = agent.NewManager(cfg.Agents, b.logger, agent.ProjectContext{}) + b.agentMgr = cruxtest.NewRuntime(cfg.Agents, b.logger, crux.ProjectContext{}) b.ghClient = fakeGitHubClient(t) b.beadStores = map[string]*beads.Store{} return b @@ -203,11 +205,22 @@ func TestBootDashboardWithAdvisoryHookNoStoresIsNoop(t *testing.T) { } func TestBootDashboardWithClassifier(t *testing.T) { + // The fixture's agent runs on copilot, which boot would use as the + // classifier CLI when no endpoint is configured (#14499); these cases + // are about a hive with no usable backend at all, so they run on codex. + noToollessAgents := func(cfg *config.Config) { + t.Setenv(config.LiteLLMEndpointEnv, "") + for name, a := range cfg.Agents { + a.Backend = "codex" + cfg.Agents[name] = a + } + } t.Run("no endpoint logs and skips", func(t *testing.T) { cfg := bootDashboardConfig() on := true cfg.Ioscan.Enabled = &on cfg.Ioscan.Classifier.Enabled = &on + noToollessAgents(cfg) b := newBootDashboardBoot(t, cfg) // bootDashboardWith starts backfillRecentMergedPRs in a goroutine that // logs through b.logger, so the sink must be goroutine-safe (#13168). @@ -230,6 +243,7 @@ func TestBootDashboardWithClassifier(t *testing.T) { cfg.Ioscan.Enabled = &on level := 5 cfg.ACMMLevel = &level + noToollessAgents(cfg) b := newBootDashboardBoot(t, cfg) var log syncLogBuffer b.logger = slog.New(slog.NewTextHandler(&log, nil)) @@ -244,6 +258,27 @@ func TestBootDashboardWithClassifier(t *testing.T) { t.Fatalf("log:\n%s", log.String()) } }) + t.Run("L5, no endpoint, copilot agents installs the copilot CLI classifier", func(t *testing.T) { + cfg := bootDashboardConfig() + on := true + cfg.Ioscan.Enabled = &on + level := 5 + cfg.ACMMLevel = &level + t.Setenv(config.LiteLLMEndpointEnv, "") + b := newBootDashboardBoot(t, cfg) + var log syncLogBuffer + b.logger = slog.New(slog.NewTextHandler(&log, nil)) + b.sched = scheduler.New(cfg, b.logger) + + b.bootDashboardWith(newBootDashboardFake().deps) + + if _, _, installed := b.sched.IoscanHooksForTest(); !installed { + t.Fatal("copilot agents with no endpoint must install the copilot CLI classifier") + } + if !strings.Contains(log.String(), "backend=copilot") || !strings.Contains(log.String(), "backend_from_agents=true") || strings.Contains(log.String(), "not running") { + t.Fatalf("log:\n%s", log.String()) + } + }) t.Run("L5 with classifier disabled explicitly skips", func(t *testing.T) { cfg := bootDashboardConfig() on, off := true, false @@ -294,8 +329,8 @@ func TestBootDashboardWithSandboxAuditCreatesRejectionBead(t *testing.T) { b.bootDashboardWith(f.deps) - b.agentMgr.SandboxAuditForTest("scanner", "sandbox_broker_rejected", "push to main refused") - b.agentMgr.SandboxAuditForTest("scanner", "sandbox_failed", "oom") + b.agentMgr.(cruxtest.Runtime).SandboxAuditForTest("scanner", "sandbox_broker_rejected", "push to main refused") + b.agentMgr.(cruxtest.Runtime).SandboxAuditForTest("scanner", "sandbox_failed", "oom") if recent := f.srv.GetAudit().Recent(2); len(recent) != 2 { t.Fatalf("audit ring = %+v", recent) diff --git a/src/cmd/hive/boot_early_output.go b/src/cmd/hive/boot_early_output.go index e2d0150d14..0a8579ac0a 100644 --- a/src/cmd/hive/boot_early_output.go +++ b/src/cmd/hive/boot_early_output.go @@ -4,7 +4,7 @@ import ( "io" "log/slog" - "github.com/hivecommons/hive/pkg/logread" + "github.com/hivecommons/hive/crux/logread" ) // teeEarlyOutput keeps what hive prints before its configured log is set up diff --git a/src/cmd/hive/boot_early_output_test.go b/src/cmd/hive/boot_early_output_test.go index 9e4ec47ecc..3646d8bec0 100644 --- a/src/cmd/hive/boot_early_output_test.go +++ b/src/cmd/hive/boot_early_output_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" + "github.com/hivecommons/hive/crux/logread" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/logread" ) // AC1: what hive prints before its own log is set up still reaches stdout diff --git a/src/cmd/hive/boot_heartbeat_deps_test.go b/src/cmd/hive/boot_heartbeat_deps_test.go index e63b83e1ab..f71a15d6fb 100644 --- a/src/cmd/hive/boot_heartbeat_deps_test.go +++ b/src/cmd/hive/boot_heartbeat_deps_test.go @@ -10,7 +10,9 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/governor" @@ -97,7 +99,7 @@ func newBootHeartbeatBoot(t *testing.T, f *bootHeartbeatFake, cfg *config.Config repoTargetMisconfigured: func() bool { return false }, repoTargetIssueMessage: func() string { return "" }, gov: governor.New(cfg.Governor, cfg.Agents, logger), - agentMgr: agent.NewManager(cfg.Agents, logger, agent.ProjectContext{}), + agentMgr: cruxtest.NewRuntime(cfg.Agents, logger, crux.ProjectContext{}), dashSrv: dashboard.NewServer(0, logger), onDemandFromPack: map[string]bool{}, } diff --git a/src/cmd/hive/boot_knowledge_deps.go b/src/cmd/hive/boot_knowledge_deps.go index 7f2842f4f7..4ff16e6d46 100644 --- a/src/cmd/hive/boot_knowledge_deps.go +++ b/src/cmd/hive/boot_knowledge_deps.go @@ -5,7 +5,8 @@ import ( "log/slog" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/knowledge/connector" @@ -46,7 +47,7 @@ type bootKnowledgeDeps struct { // owner (#11696); nil leaves the state wired but unpolled. startNousGateWatch func(ctx context.Context, state *dashboard.NousState, interval time.Duration, logger *slog.Logger) newInceptionEngine func(api *knowledge.KnowledgeAPI, logger *slog.Logger) *knowledge.InceptionEngine - restartBrainstorm func(ctx context.Context, mgr *agent.Manager, msg string) error + restartBrainstorm func(ctx context.Context, mgr crux.Runtime, msg string) error // startConnectorSyncer runs the knowledge.connectors syncer (#11069); // nil leaves it built but not started. startConnectorSyncer func(ctx context.Context, s *connector.Syncer) @@ -76,7 +77,7 @@ func defaultBootKnowledgeDeps() bootKnowledgeDeps { newInceptionEngine: func(api *knowledge.KnowledgeAPI, logger *slog.Logger) *knowledge.InceptionEngine { return knowledge.NewInceptionEngine(inceptionDataDir, api, logger) }, - restartBrainstorm: func(ctx context.Context, mgr *agent.Manager, msg string) error { + restartBrainstorm: func(ctx context.Context, mgr crux.Runtime, msg string) error { return mgr.RestartWithBootstrap(ctx, "brainstorm", msg) }, startConnectorSyncer: func(ctx context.Context, s *connector.Syncer) { go s.Run(ctx) }, diff --git a/src/cmd/hive/boot_knowledge_deps_test.go b/src/cmd/hive/boot_knowledge_deps_test.go index 862e31f350..fae9813977 100644 --- a/src/cmd/hive/boot_knowledge_deps_test.go +++ b/src/cmd/hive/boot_knowledge_deps_test.go @@ -11,7 +11,9 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/beads" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" @@ -74,7 +76,7 @@ func newBootKnowledgeFake(t *testing.T) *bootKnowledgeFake { newInceptionEngine: func(api *knowledge.KnowledgeAPI, logger *slog.Logger) *knowledge.InceptionEngine { return knowledge.NewInceptionEngine(f.inceptionDir, api, logger) }, - restartBrainstorm: func(_ context.Context, _ *agent.Manager, msg string) error { + restartBrainstorm: func(_ context.Context, _ crux.Runtime, msg string) error { f.restarts = append(f.restarts, msg) return f.restartErr }, @@ -100,7 +102,7 @@ func newBootKnowledgeBoot(t *testing.T, cfg *config.Config) *boot { t.Helper() b, _ := newDepsTestBoot(t, cfg) b.sched = scheduler.New(cfg, b.logger) - b.agentMgr = agent.NewManager(cfg.Agents, b.logger, agent.ProjectContext{}) + b.agentMgr = cruxtest.NewRuntime(cfg.Agents, b.logger, crux.ProjectContext{}) b.dashSrv = dashboard.NewServer(0, b.logger) b.beadStores = map[string]*beads.Store{} return b diff --git a/src/cmd/hive/boot_launch_deps.go b/src/cmd/hive/boot_launch_deps.go index f2437c7ff9..77e6293c11 100644 --- a/src/cmd/hive/boot_launch_deps.go +++ b/src/cmd/hive/boot_launch_deps.go @@ -5,8 +5,9 @@ import ( "log/slog" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/chat" + "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/dashchat" @@ -35,7 +36,7 @@ type bootLaunchDeps struct { // waitStagger blocks for the launch stagger; it returns false when ctx // ended first so the loop aborts instead of launching into a shutdown. waitStagger func(ctx context.Context) bool - startAgent func(ctx context.Context, mgr *agent.Manager, name string) error + startAgent func(ctx context.Context, mgr crux.Runtime, name string) error serve func(srv *dashboard.Server) error } @@ -62,7 +63,7 @@ func defaultBootLaunchDeps() bootLaunchDeps { return false } }, - startAgent: func(ctx context.Context, mgr *agent.Manager, name string) error { return mgr.Start(ctx, name) }, + startAgent: func(ctx context.Context, mgr crux.Runtime, name string) error { return mgr.Start(ctx, name) }, serve: (*dashboard.Server).Start, } } diff --git a/src/cmd/hive/boot_launch_deps_test.go b/src/cmd/hive/boot_launch_deps_test.go index a51b03cfa3..bda4be61d3 100644 --- a/src/cmd/hive/boot_launch_deps_test.go +++ b/src/cmd/hive/boot_launch_deps_test.go @@ -12,8 +12,10 @@ import ( "sync" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" "github.com/hivecommons/hive/pkg/chat" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/dashchat" @@ -55,7 +57,7 @@ func (h *launchHarness) deps() bootLaunchDeps { h.waits++ return ctx.Err() == nil }, - startAgent: func(_ context.Context, _ *agent.Manager, name string) error { + startAgent: func(_ context.Context, _ crux.Runtime, name string) error { h.started = append(h.started, name) return h.startErr[name] }, @@ -68,7 +70,7 @@ func newLaunchBoot(t *testing.T, agents map[string]config.AgentConfig) *boot { cfg := &config.Config{AgentRuntimeConfig: config.AgentRuntimeConfig{Agents: agents}} cfg.Dashboard.Port = 8080 b, _ := newDepsTestBoot(t, cfg) - b.agentMgr = agent.NewManager(cfg.Agents, b.logger, agent.ProjectContext{}) + b.agentMgr = cruxtest.NewRuntime(cfg.Agents, b.logger, crux.ProjectContext{}) b.dashSrv = dashboard.NewServer(0, b.logger) return b } @@ -268,7 +270,7 @@ func TestBootLaunchWithAbortsLaunchOnShutdown(t *testing.T) { h := &launchHarness{} deps := h.deps() orig := deps.startAgent - deps.startAgent = func(ctx context.Context, mgr *agent.Manager, name string) error { + deps.startAgent = func(ctx context.Context, mgr crux.Runtime, name string) error { cancel() return orig(ctx, mgr, name) } diff --git a/src/cmd/hive/boot_phases_7571_test.go b/src/cmd/hive/boot_phases_7571_test.go index ec8e07c151..145d75effd 100644 --- a/src/cmd/hive/boot_phases_7571_test.go +++ b/src/cmd/hive/boot_phases_7571_test.go @@ -6,12 +6,14 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/watchdog" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/scheduler" - "github.com/hivecommons/hive/pkg/watchdog" ) // These tests drive individual boot phases in isolation (#7571, step 1). They @@ -155,7 +157,7 @@ func TestBootSupervision_WatchdogFollowsConfiguredMode(t *testing.T) { logger: logger, gov: governor.New(cfg.Governor, cfg.Agents, logger), sched: scheduler.New(cfg, logger), - agentMgr: agent.NewManager(cfg.Agents, logger, agent.ProjectContext{}), + agentMgr: cruxtest.NewRuntime(cfg.Agents, logger, crux.ProjectContext{}), dashSrv: dashboard.NewServer(0, logger), } } @@ -211,7 +213,7 @@ func TestBootLanes_BuildsOnlyTheRunnableLanes(t *testing.T) { cfg: cfg, logger: logger, gov: governor.New(cfg.Governor, cfg.Agents, logger), - agentMgr: agent.NewManager(cfg.Agents, logger, agent.ProjectContext{}), + agentMgr: cruxtest.NewRuntime(cfg.Agents, logger, crux.ProjectContext{}), dashSrv: dashboard.NewServer(0, logger), } } diff --git a/src/cmd/hive/boot_proxy_deps.go b/src/cmd/hive/boot_proxy_deps.go index b5ee9f11fb..e662b2238f 100644 --- a/src/cmd/hive/boot_proxy_deps.go +++ b/src/cmd/hive/boot_proxy_deps.go @@ -4,10 +4,11 @@ import ( "context" "log/slog" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/advisor" - "github.com/hivecommons/hive/pkg/agent" + + "github.com/hivecommons/hive/crux/proxy" "github.com/hivecommons/hive/pkg/jev" - "github.com/hivecommons/hive/pkg/proxy" ) // bootProxyDeps are the effects bootProxy performs that a test cannot let run @@ -19,7 +20,7 @@ import ( // setInferenceRoute / clearInferenceRoute recording what it decided. type bootProxyDeps struct { newGitHubProxy func(logger *slog.Logger, org string, repos []string) (*proxy.GitHubProxy, error) - installInferenceCallbacks func(mgr *agent.Manager, setRoute func(agentName, backend, model string), clearRoute func(agentName string)) + installInferenceCallbacks func(mgr crux.Host, setRoute func(agentName, backend, model string), clearRoute func(agentName string)) setInferenceRoute func(p *proxy.GitHubProxy, agentName string, route *proxy.InferenceRoute) clearInferenceRoute func(p *proxy.GitHubProxy, agentName string) // startProxy binds the GitHub MITM listener and the inference translator. @@ -36,7 +37,7 @@ type bootProxyDeps struct { func defaultBootProxyDeps() bootProxyDeps { return bootProxyDeps{ newGitHubProxy: proxy.NewGitHubProxy, - installInferenceCallbacks: func(mgr *agent.Manager, setRoute func(agentName, backend, model string), clearRoute func(agentName string)) { + installInferenceCallbacks: func(mgr crux.Host, setRoute func(agentName, backend, model string), clearRoute func(agentName string)) { mgr.SetInferenceCallbacks(setRoute, clearRoute) }, setInferenceRoute: func(p *proxy.GitHubProxy, agentName string, route *proxy.InferenceRoute) { diff --git a/src/cmd/hive/boot_proxy_deps_test.go b/src/cmd/hive/boot_proxy_deps_test.go index d1ca9f7388..1ebd5f4567 100644 --- a/src/cmd/hive/boot_proxy_deps_test.go +++ b/src/cmd/hive/boot_proxy_deps_test.go @@ -8,10 +8,12 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/proxy" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" - "github.com/hivecommons/hive/pkg/proxy" "github.com/hivecommons/hive/pkg/tokens" ) @@ -43,7 +45,7 @@ func newBootProxyFake() *bootProxyFake { f.proxy = p return p, err }, - installInferenceCallbacks: func(_ *agent.Manager, set func(string, string, string), clear func(string)) { + installInferenceCallbacks: func(_ crux.Host, set func(string, string, string), clear func(string)) { f.setRoute, f.clearRoute = set, clear }, setInferenceRoute: func(_ *proxy.GitHubProxy, agentName string, r *proxy.InferenceRoute) { @@ -77,10 +79,10 @@ func newBootProxyBoot(t *testing.T, f *bootProxyFake, cfg *config.Config) *boot ctx: ctx, cfg: cfg, logger: logger, - agentMgr: agent.NewManager(cfg.Agents, logger, agent.ProjectContext{}), + agentMgr: cruxtest.NewRuntime(cfg.Agents, logger, crux.ProjectContext{}), dashSrv: dashboard.NewServer(0, logger), tokenCollector: tokens.NewCollector(t.TempDir(), logger), - linearCredentialResolver: func() agent.LinearCredential { return agent.LinearCredential{} }, + linearCredentialResolver: func() crux.LinearCredential { return crux.LinearCredential{} }, } b.wireBootClosures() return b diff --git a/src/cmd/hive/boot_start_record.go b/src/cmd/hive/boot_start_record.go index 0b6fa2ea44..17298936a7 100644 --- a/src/cmd/hive/boot_start_record.go +++ b/src/cmd/hive/boot_start_record.go @@ -3,8 +3,8 @@ package main import ( "log/slog" + "github.com/hivecommons/hive/crux/logread" "github.com/hivecommons/hive/pkg/dashboard" - "github.com/hivecommons/hive/pkg/logread" "github.com/hivecommons/hive/pkg/startrec" ) diff --git a/src/cmd/hive/boot_start_record_test.go b/src/cmd/hive/boot_start_record_test.go index 72e55d951c..7c1982d46f 100644 --- a/src/cmd/hive/boot_start_record_test.go +++ b/src/cmd/hive/boot_start_record_test.go @@ -10,7 +10,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/logread" + "github.com/hivecommons/hive/crux/logread" "github.com/hivecommons/hive/pkg/startrec" ) diff --git a/src/cmd/hive/boot_state_deps_test.go b/src/cmd/hive/boot_state_deps_test.go index 0850997a94..fa7ba6d007 100644 --- a/src/cmd/hive/boot_state_deps_test.go +++ b/src/cmd/hive/boot_state_deps_test.go @@ -7,7 +7,9 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/snapshot" @@ -57,7 +59,7 @@ func newBootStateBoot(t *testing.T, cfg *config.Config) *boot { t.Helper() b, _ := newDepsTestBoot(t, cfg) b.gov = governor.New(cfg.Governor, cfg.EnabledAgents(), b.logger) - b.agentMgr = agent.NewManager(cfg.Agents, b.logger, agent.ProjectContext{}) + b.agentMgr = cruxtest.NewRuntime(cfg.Agents, b.logger, crux.ProjectContext{}) b.ghClient = fakeGitHubClient(t) return b } @@ -106,7 +108,7 @@ func TestBootStateWithRestoresGovernorAndAgents(t *testing.T) { lvl := 4 saved := &snapshot.PersistedState{ Agents: map[string]snapshot.AgentState{ - "scanner": {Paused: true, PausedReason: "fleet breaker", PausedTrigger: agent.BreakerTrigger}, + "scanner": {Paused: true, PausedReason: "fleet breaker", PausedTrigger: crux.BreakerTrigger}, "fixer": {Paused: true, PausedReason: "operator hold"}, "ghost": {Paused: true}, }, diff --git a/src/cmd/hive/boot_watchers_deps.go b/src/cmd/hive/boot_watchers_deps.go index 199ee8e1fc..d9dda603ba 100644 --- a/src/cmd/hive/boot_watchers_deps.go +++ b/src/cmd/hive/boot_watchers_deps.go @@ -4,9 +4,10 @@ import ( "context" "log/slog" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux" + + "github.com/hivecommons/hive/crux/proxy" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/proxy" ) // bootWatchersDeps are the long-lived effects bootWatchers performs (#7571, @@ -20,10 +21,10 @@ type bootWatchersDeps struct { // startConfigWatcher is `go w.Start(ctx)`. startConfigWatcher func(ctx context.Context, w *config.Watcher) // startAgent is the goroutine that launches an agent a reload added. - startAgent func(ctx context.Context, mgr *agent.Manager, name string, logger *slog.Logger) + startAgent func(ctx context.Context, mgr crux.Runtime, name string, logger *slog.Logger) // refreshAgentTokens is `go mgr.RefreshAgentTokens(ctx)` after an App // auth rebuild (#4072). - refreshAgentTokens func(ctx context.Context, mgr *agent.Manager) + refreshAgentTokens func(ctx context.Context, mgr crux.Runtime) // registerGitHubHost is proxy.RegisterGitHubHost (GHE allowlisting). registerGitHubHost func(host string) } @@ -32,7 +33,7 @@ func defaultBootWatchersDeps() bootWatchersDeps { return bootWatchersDeps{ newConfigWatcher: config.NewWatcher, startConfigWatcher: func(ctx context.Context, w *config.Watcher) { go w.Start(ctx) }, - startAgent: func(ctx context.Context, mgr *agent.Manager, name string, logger *slog.Logger) { + startAgent: func(ctx context.Context, mgr crux.Runtime, name string, logger *slog.Logger) { go func() { logger.Info("audit: starting reconciled agent", "name", name, "trigger", "config-reload") if err := mgr.Start(ctx, name); err != nil { @@ -40,7 +41,7 @@ func defaultBootWatchersDeps() bootWatchersDeps { } }() }, - refreshAgentTokens: func(ctx context.Context, mgr *agent.Manager) { go mgr.RefreshAgentTokens(ctx) }, + refreshAgentTokens: func(ctx context.Context, mgr crux.Runtime) { go mgr.RefreshAgentTokens(ctx) }, registerGitHubHost: proxy.RegisterGitHubHost, } } diff --git a/src/cmd/hive/boot_watchers_deps_test.go b/src/cmd/hive/boot_watchers_deps_test.go index 78dc785c19..82fa1595b1 100644 --- a/src/cmd/hive/boot_watchers_deps_test.go +++ b/src/cmd/hive/boot_watchers_deps_test.go @@ -10,7 +10,9 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/governor" @@ -40,10 +42,10 @@ func newBootWatchersFake() *bootWatchersFake { return f.watcher }, startConfigWatcher: func(_ context.Context, w *config.Watcher) { f.started = w }, - startAgent: func(_ context.Context, _ *agent.Manager, name string, _ *slog.Logger) { + startAgent: func(_ context.Context, _ crux.Runtime, name string, _ *slog.Logger) { f.startedAgents = append(f.startedAgents, name) }, - refreshAgentTokens: func(context.Context, *agent.Manager) { f.tokenRefreshes++ }, + refreshAgentTokens: func(context.Context, crux.Runtime) { f.tokenRefreshes++ }, registerGitHubHost: func(host string) { f.hosts = append(f.hosts, host) }, } return f @@ -75,7 +77,7 @@ func newBootWatchersBoot(t *testing.T, f *bootWatchersFake, cfg *config.Config) logger: logger, configPath: filepath.Join(t.TempDir(), "hive.yaml"), gov: governor.New(cfg.Governor, cfg.EnabledAgents(), logger), - agentMgr: agent.NewManager(cfg.EnabledAgents(), logger, agent.ProjectContext{}), + agentMgr: cruxtest.NewRuntime(cfg.EnabledAgents(), logger, crux.ProjectContext{}), dashSrv: dashboard.NewServer(0, logger), refreshDashboard: func() { refreshes++ }, } diff --git a/src/cmd/hive/claimswire.go b/src/cmd/hive/claimswire.go index ad1b54b446..15582d0f4b 100644 --- a/src/cmd/hive/claimswire.go +++ b/src/cmd/hive/claimswire.go @@ -5,9 +5,11 @@ import ( "fmt" "log/slog" "strings" + "sync" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/claims" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" @@ -36,6 +38,58 @@ var claimsLedgerPath = "/data/issue-claims.json" // claimsGitHubTimeout bounds each best-effort GitHub side effect. const claimsGitHubTimeout = 15 * time.Second +// claimEffectQueue runs the GitHub side effects of one issue's ledger +// transitions in the order the ledger fired them, one transition at a time +// per issue; different issues still run concurrently. The hooks used to +// spawn a free goroutine per transition, so a claim's side effects and the +// release of that same claim raced: on hivecommons/hive#14065 the release +// (26s after the claim) re-read the issue, found no `hive/claimed-by-human` +// yet, posted its release marker — and the claim's goroutine, slowed by the +// stage write ahead of it, then applied the label at 12:23:55Z, four +// seconds after the release had finished. The orphaned label kept the +// issue out of every lane until a person noticed (hivecommons/hive#14201). +type claimEffectQueue struct { + mu sync.Mutex + pending map[string][]func() +} + +// run schedules fn for key after every fn previously scheduled for key has +// returned. It never waits for GitHub (the ledger fires hooks on the +// claimant's path). One worker drains each active key; a slow GitHub call +// must not turn claim/release churn into a chain of waiting goroutines. +// Pending effects are retained, not dropped: losing a release could leave +// the preceding claim's labels behind. This bounds goroutines per key, not +// pending work or the number of distinct active keys. +func (q *claimEffectQueue) run(key string, fn func()) { + q.mu.Lock() + if q.pending == nil { + q.pending = map[string][]func(){} + } + _, running := q.pending[key] + q.pending[key] = append(q.pending[key], fn) + q.mu.Unlock() + if !running { + go q.drain(key) + } +} + +func (q *claimEffectQueue) drain(key string) { + for { + q.mu.Lock() + pending := q.pending[key] + if len(pending) == 0 { + delete(q.pending, key) + q.mu.Unlock() + return + } + fn := pending[0] + pending[0] = nil // Do not retain completed claims in the backing array. + q.pending[key] = pending[1:] + q.mu.Unlock() + fn() + } +} + // buildClaimsLedger constructs the ledger from config, or returns nil when // `claims.enabled: false`. A corrupt ledger file is logged and the ledger // starts empty rather than failing boot — losing claims is recoverable (they @@ -80,6 +134,8 @@ func installClaimAdmissionCheck(ctx context.Context, ledger *claims.Ledger, clie // githubClaimHooks returns the hooks that mirror ledger transitions onto the // GitHub issue. Every call is best-effort: a failed comment or label never // affects the ledger, which is authoritative for the hub's own decisions. +// Per issue, the side effects run in ledger order (claimEffectQueue): a +// release only reads the issue once the claim it ends has finished writing. // // client is resolved per call because the GitHub client is (re)built after // boot when App auth completes (see the ghClient swap in main); capturing the @@ -90,6 +146,7 @@ func githubClaimHooks(ctx context.Context, cfg *config.Config, client func() *gi } comment := cfg.Governor.Claims.CommentEnabled() label := cfg.Governor.Claims.LabelEnabled() + queue := &claimEffectQueue{} run := func(what string, c claims.Claim, fn func(ctx context.Context, gh *github.Client) error) { gh := client() if gh == nil { @@ -103,7 +160,7 @@ func githubClaimHooks(ctx context.Context, cfg *config.Config, client func() *gi } return claims.Hooks{ OnClaimed: func(c claims.Claim, outcome claims.Outcome) { - go func() { + queue.run(c.Key(), func() { if comment && outcome == claims.OutcomeClaimed { run("claim comment", c, func(ctx context.Context, gh *github.Client) error { return gh.CreateIssueCommentBudgeted(ctx, c.Repo, c.Issue, claims.ClaimComment(c)) @@ -127,10 +184,10 @@ func githubClaimHooks(ctx context.Context, cfg *config.Config, client func() *gi return err }) } - }() + }) }, OnTakenOver: func(now, previous claims.Claim) { - go func() { + queue.run(now.Key(), func() { if comment { run("takeover comment", now, func(ctx context.Context, gh *github.Client) error { return gh.CreateIssueComment(ctx, now.Repo, now.Issue, claims.TakeoverComment(now, previous)) @@ -151,10 +208,10 @@ func githubClaimHooks(ctx context.Context, cfg *config.Config, client func() *gi return err }) } - }() + }) }, OnReleased: func(c claims.Claim, reason string) { - go func() { + queue.run(c.Key(), func() { if label { // One fresh read, then only the "who" labels still present // come off, journaled against that read: the plain remove @@ -196,7 +253,7 @@ func githubClaimHooks(ctx context.Context, cfg *config.Config, client func() *gi return gh.CreateIssueCommentBudgeted(ctx, c.Repo, c.Issue, claims.ReleaseComment(c, reason)) }) } - }() + }) }, } } @@ -372,7 +429,7 @@ func ledgerClaimLookup(ledger func() *claims.Ledger) func(repo string, number in // (hivecommons/hive, 2026-10-20): the scanner claimed #13083 at 14:21Z, two // image rollouts relaunched every agent by 15:35Z, and the issue sat // claimed-by-scanner with nothing working it until the 16:21Z until=. -func releaseClaimsOnAgentRelaunch(mgr *agent.Manager, ledger func() *claims.Ledger, logger *slog.Logger) { +func releaseClaimsOnAgentRelaunch(mgr crux.Host, ledger func() *claims.Ledger, logger *slog.Logger) { if mgr == nil || ledger == nil { return } diff --git a/src/cmd/hive/claimswire_relaunch_test.go b/src/cmd/hive/claimswire_relaunch_test.go index 6d1bbb3d7f..253ac6e3fb 100644 --- a/src/cmd/hive/claimswire_relaunch_test.go +++ b/src/cmd/hive/claimswire_relaunch_test.go @@ -11,8 +11,10 @@ import ( "testing" "time" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/internal/testutil" - "github.com/hivecommons/hive/pkg/agent" "github.com/hivecommons/hive/pkg/claims" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" @@ -23,7 +25,7 @@ import ( // stage/020-claimed by the scanner for two hours across two rollouts). func TestReleaseClaimsOnAgentRelaunch_ReleasesOnlyThatAgentsClaims(t *testing.T) { cfg := activityTestConfig() - mgr := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) var mu sync.Mutex var released []string l, err := claims.New("", claims.DefaultPolicy(), claims.Hooks{OnReleased: func(c claims.Claim, reason string) { diff --git a/src/cmd/hive/claimswire_test.go b/src/cmd/hive/claimswire_test.go index 7b1c9250d3..ac5850d3fe 100644 --- a/src/cmd/hive/claimswire_test.go +++ b/src/cmd/hive/claimswire_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "net/url" "path/filepath" + "runtime" "strings" "sync" "testing" @@ -385,3 +386,199 @@ func TestClaimHooks_ClaimedByLabelIsGenericHumanNeverLogin(t *testing.T) { return has(removed, claims.ClaimedByHumanLabel) && has(removed, "hive/claimed-by-clubanderson") }, "release never removed the holder's generic and legacy claimed-by labels") } + +// A claim's GitHub side effects and the release of that same claim used to +// run in independent goroutines. On hivecommons/hive#14065 the lease was +// revoked 26s after the claim while the claim's goroutine was still working +// through its stage write: the release re-read the issue, saw no +// `hive/claimed-by-human` yet, finished, and THEN the claim applied the +// label — orphaned (hivecommons/hive#14201). The hooks now run one issue's +// transitions in ledger order, so the release's fresh read happens after +// the claim's label write and strips it. The mock delays the claimed-by add +// to reproduce the ordering. +func TestClaimHooks_ReleaseWaitsForSlowClaimLabel(t *testing.T) { + var mu sync.Mutex + var removed []string + truth := []string{"stage/010-actionable"} + has := func(list []string, want string) bool { + for _, l := range list { + if l == want { + return true + } + } + return false + } + mux := http.NewServeMux() + mux.HandleFunc("/repos/acme/widgets/issues/11", func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + labels := make([]map[string]string, 0, len(truth)) + for _, l := range truth { + labels = append(labels, map[string]string{"name": l}) + } + mu.Unlock() + _ = json.NewEncoder(w).Encode(map[string]any{"number": 11, "labels": labels}) + }) + mux.HandleFunc("/repos/acme/widgets/issues/11/labels", func(w http.ResponseWriter, r *http.Request) { + var body []string + _ = json.NewDecoder(r.Body).Decode(&body) + if has(body, claims.ClaimedByHumanLabel) { + // The slow write: without ordering the release's read lands + // well inside this window. + select { + case <-time.After(400 * time.Millisecond): + case <-r.Context().Done(): + return + } + } + mu.Lock() + truth = append(truth, body...) + mu.Unlock() + _, _ = w.Write([]byte(`[]`)) + }) + mux.HandleFunc("/repos/acme/widgets/issues/11/labels/", func(w http.ResponseWriter, r *http.Request) { + name := strings.TrimPrefix(r.URL.Path, "/repos/acme/widgets/issues/11/labels/") + if u, err := url.PathUnescape(name); err == nil { + name = u + } + mu.Lock() + removed = append(removed, name) + kept := truth[:0] + for _, l := range truth { + if l != name { + kept = append(kept, l) + } + } + truth = kept + mu.Unlock() + w.WriteHeader(http.StatusOK) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + gh := github.NewClientForTest(srv.URL, "acme", []string{"widgets"}, slog.New(slog.NewTextHandler(io.Discard, nil))) + cfg := &config.Config{} + cfg.Governor.Claims.Label = boolPtr(true) + cfg.Governor.Claims.Comment = boolPtr(false) + hooks := githubClaimHooks(context.Background(), cfg, func() *github.Client { return gh }, nil) + claim := claims.Claim{Repo: "acme/widgets", Issue: 11, Holder: "arteeh", HolderID: "c-1", Kind: claims.KindContributor} + hooks.OnClaimed(claim, claims.OutcomeClaimed) + hooks.OnReleased(claim, "lease revoked") + + testutil.Eventually(t, 10*time.Second, func() bool { + mu.Lock() + defer mu.Unlock() + return has(removed, claims.ClaimedByHumanLabel) + }, "release never removed the claimed-by label its own claim wrote") + mu.Lock() + defer mu.Unlock() + if has(truth, claims.ClaimedByHumanLabel) { + t.Fatalf("labels after release = %v, want %s gone", truth, claims.ClaimedByHumanLabel) + } +} + +// claimEffectQueue runs one key's work in submission order and different +// keys concurrently. +func TestClaimEffectQueue_OrdersPerKey(t *testing.T) { + q := &claimEffectQueue{} + var mu sync.Mutex + var order []string + firstDone := make(chan struct{}) + otherRan := make(chan struct{}) + q.run("a", func() { + // Block until the other key's work has run: proves keys are + // independent and that "a"'s second step waits for this one. + <-otherRan + mu.Lock() + order = append(order, "a1") + mu.Unlock() + close(firstDone) + }) + q.run("a", func() { + mu.Lock() + order = append(order, "a2") + mu.Unlock() + }) + q.run("b", func() { + mu.Lock() + order = append(order, "b1") + mu.Unlock() + close(otherRan) + }) + <-firstDone + // Wait for both the work and the worker's idle-key cleanup. + testutil.Eventually(t, 5*time.Second, func() bool { + mu.Lock() + defer mu.Unlock() + if len(order) != 3 { + return false + } + q.mu.Lock() + defer q.mu.Unlock() + return len(q.pending) == 0 + }, "queued work never finished or idle keys were retained") + mu.Lock() + defer mu.Unlock() + if order[0] != "b1" || order[1] != "a1" || order[2] != "a2" { + t.Fatalf("order = %v, want [b1 a1 a2]", order) + } +} + +func TestClaimEffectQueue_ChurnDoesNotSpawnWaitingGoroutines(t *testing.T) { + q := &claimEffectQueue{} + blocked := make(chan struct{}) + started := make(chan struct{}) + var unblock sync.Once + release := func() { unblock.Do(func() { close(blocked) }) } + defer release() + q.run("busy", func() { + close(started) + <-blocked + }) + <-started + + before := runtime.NumGoroutine() + const transitions = 2048 + var mu sync.Mutex + var order []int + for i := 0; i < transitions; i++ { + q.run("busy", func() { + mu.Lock() + order = append(order, i) + mu.Unlock() + }) + } + // Allow unrelated runtime/HTTP goroutines some headroom. The old + // goroutine chain adds one waiting goroutine for every transition. + if growth := runtime.NumGoroutine() - before; growth > 32 { + t.Errorf("one blocked issue added %d goroutines for %d transitions", growth, transitions) + } + release() + testutil.Eventually(t, 5*time.Second, func() bool { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.pending) == 0 + }, "churn worker did not drain and remove its idle key") + mu.Lock() + defer mu.Unlock() + if len(order) != transitions { + t.Fatalf("executed %d transitions, want %d (including the final release)", len(order), transitions) + } + for i, got := range order { + if got != i { + t.Fatalf("transition %d = %d, want FIFO order", i, got) + } + } +} + +func TestClaimEffectQueue_RestartsAfterIdle(t *testing.T) { + q := &claimEffectQueue{} + for i := 0; i < 20; i++ { + done := make(chan struct{}) + q.run("same", func() { close(done) }) + <-done + testutil.Eventually(t, 5*time.Second, func() bool { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.pending) == 0 + }, "idle key was retained") + } +} diff --git a/src/cmd/hive/close_on_merge_sweep.go b/src/cmd/hive/close_on_merge_sweep.go index 7906fc127a..56039ed47d 100644 --- a/src/cmd/hive/close_on_merge_sweep.go +++ b/src/cmd/hive/close_on_merge_sweep.go @@ -125,6 +125,7 @@ func runCloseOnMergeSweepWith(ctx context.Context, cfg *config.Config, ghClient MergedAfter: plan.MergedAfter, Now: sched.clock(), LabelDriven: cfg.RepoLabelDriven, + RepoOnGitHub: cfg.RepoOnGitHub, } sweep := ghClient.CloseOnMergeLedgerSweep(ctx, ledger, opts) msg := "close-on-merge ledger sweep" diff --git a/src/cmd/hive/delegatedecider.go b/src/cmd/hive/delegatedecider.go new file mode 100644 index 0000000000..d2a5e45541 --- /dev/null +++ b/src/cmd/hive/delegatedecider.go @@ -0,0 +1,130 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/delegatedecision" +) + +// delegateDeciderRunner runs the owner-delegate decider turn behind "Let hive +// decide" (decisions.delegate.llm) as one chat completion on the governor +// reviewer endpoint — the same endpoint/key resolution the intent-alignment +// and trajectory reviewers use. Config is read per call so a reload takes +// effect without a restart. +type delegateDeciderRunner struct { + cfg func() *config.Config + http *http.Client + // model returns the per-feature model override ("" keeps the reviewer + // model); nil means the "Let hive decide" override. + model func(*config.Config) string +} + +const ( + delegateDeciderTimeout = 60 * time.Second + delegateDeciderMaxTokens = 800 + delegateDeciderMaxReply = 1 << 20 +) + +var errDelegateDeciderUnconfigured = errors.New("owner-delegate decider: no reviewer endpoint or model resolved") + +func newDelegateDeciderRunner(cfg func() *config.Config) delegatedecision.TurnRunner { + return &delegateDeciderRunner{cfg: cfg, http: &http.Client{Timeout: delegateDeciderTimeout}} +} + +// newChatGroundingRunner is the same reviewer-endpoint turn for the +// dashboard's grounded chat answers (chat.grounded_answers), with that +// block's model override instead of the decider's. +func newChatGroundingRunner(cfg func() *config.Config) delegatedecision.TurnRunner { + return &delegateDeciderRunner{ + cfg: cfg, + http: &http.Client{Timeout: delegateDeciderTimeout}, + model: func(c *config.Config) string { return c.Chat.GroundedAnswersModel() }, + } +} + +type delegateDeciderMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type delegateDeciderRequest struct { + Model string `json:"model"` + Messages []delegateDeciderMessage `json:"messages"` + Temperature float64 `json:"temperature"` + MaxTokens int `json:"max_tokens"` +} + +type delegateDeciderResponse struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` +} + +func (r *delegateDeciderRunner) RunDeciderTurn(ctx context.Context, profile, system, user string) (string, error) { + cfg := r.cfg() + if cfg == nil { + return "", errDelegateDeciderUnconfigured + } + endpoint, apiKey, model := cfg.Governor.ResolveReviewer() + override := cfg.Decisions.Delegate.LLMModel + if r.model != nil { + override = func() string { return r.model(cfg) } + } + if m := override(); m != "" { + model = m + } + endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/") + if endpoint == "" || strings.TrimSpace(model) == "" { + return "", errDelegateDeciderUnconfigured + } + body, err := json.Marshal(delegateDeciderRequest{ + Model: model, + Messages: []delegateDeciderMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, + Temperature: 0, + MaxTokens: delegateDeciderMaxTokens, + }) + if err != nil { + return "", fmt.Errorf("%s: marshal request: %w", profile, err) + } + reqCtx, cancel := context.WithTimeout(ctx, delegateDeciderTimeout) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, endpoint+"/v1/chat/completions", bytes.NewReader(body)) + if err != nil { + return "", fmt.Errorf("%s: build request: %w", profile, err) + } + req.Header.Set("Content-Type", "application/json") + if apiKey != "" { + req.Header.Set("Authorization", "Bearer "+apiKey) + } + resp, err := r.http.Do(req) + if err != nil { + return "", fmt.Errorf("%s: request failed: %w", profile, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("%s: endpoint returned %d", profile, resp.StatusCode) + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, delegateDeciderMaxReply)) + if err != nil { + return "", fmt.Errorf("%s: read response: %w", profile, err) + } + var cr delegateDeciderResponse + if err := json.Unmarshal(raw, &cr); err != nil { + return "", fmt.Errorf("%s: decode response: %w", profile, err) + } + if len(cr.Choices) == 0 { + return "", fmt.Errorf("%s: no choices", profile) + } + return cr.Choices[0].Message.Content, nil +} diff --git a/src/cmd/hive/delegatedecider_test.go b/src/cmd/hive/delegatedecider_test.go new file mode 100644 index 0000000000..bc30cd1ebd --- /dev/null +++ b/src/cmd/hive/delegatedecider_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/delegatedecision" +) + +func TestDelegateDeciderRunner(t *testing.T) { + t.Setenv(config.LiteLLMEndpointEnv, "") + var got delegateDeciderRequest + status := http.StatusOK + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/chat/completions" { + http.NotFound(w, r) + return + } + _ = json.NewDecoder(r.Body).Decode(&got) + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"verdict\":\"approve\"}"}}]}`)) + })) + t.Cleanup(srv.Close) + + cfg := &config.Config{} + cfg.Governor.Trajectory.Endpoint = srv.URL + cfg.Governor.Trajectory.Model = "reviewer-model" + runner := newDelegateDeciderRunner(func() *config.Config { return cfg }) + + reply, err := runner.RunDeciderTurn(context.Background(), delegatedecision.DeciderProfile, "sys", "usr") + if err != nil || reply != `{"verdict":"approve"}` { + t.Fatalf("reply=%q err=%v", reply, err) + } + if got.Model != "reviewer-model" || len(got.Messages) != 2 || got.Messages[0].Content != "sys" || got.Messages[1].Content != "usr" || got.Temperature != 0 { + t.Fatalf("request = %+v", got) + } + + on := true + cfg.Decisions.Delegate.LLM = &config.DelegateDecisionLLMConfig{Enabled: &on, Model: "decider-model"} + if _, err := runner.RunDeciderTurn(context.Background(), delegatedecision.DeciderProfile, "sys", "usr"); err != nil || got.Model != "decider-model" { + t.Fatalf("model override: model=%q err=%v", got.Model, err) + } + + status = http.StatusBadGateway + if _, err := runner.RunDeciderTurn(context.Background(), delegatedecision.DeciderProfile, "sys", "usr"); err == nil || !strings.Contains(err.Error(), "502") { + t.Fatalf("non-200 must error, got %v", err) + } + + empty := newDelegateDeciderRunner(func() *config.Config { return &config.Config{} }) + if _, err := empty.RunDeciderTurn(context.Background(), delegatedecision.DeciderProfile, "s", "u"); !errors.Is(err, errDelegateDeciderUnconfigured) { + t.Fatalf("unconfigured: %v", err) + } + none := newDelegateDeciderRunner(func() *config.Config { return nil }) + if _, err := none.RunDeciderTurn(context.Background(), delegatedecision.DeciderProfile, "s", "u"); !errors.Is(err, errDelegateDeciderUnconfigured) { + t.Fatalf("nil config: %v", err) + } +} diff --git a/src/cmd/hive/escalation_runtime.go b/src/cmd/hive/escalation_runtime.go index 0e91f3a0e9..786eaa53fc 100644 --- a/src/cmd/hive/escalation_runtime.go +++ b/src/cmd/hive/escalation_runtime.go @@ -8,10 +8,10 @@ import ( "strings" "sync" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/logscrub" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/escalate" - "github.com/hivecommons/hive/pkg/logscrub" "github.com/hivecommons/hive/pkg/notify" "github.com/hivecommons/hive/pkg/review" ) @@ -44,7 +44,7 @@ func newEscalationRuntimeKey(cfg *config.Config, notifier *notify.Notifier) *esc } } -func configureEscalationDispatcher(ctx context.Context, cfg *config.Config, notifier *notify.Notifier, audit agent.AuditSink, logger *slog.Logger) { +func configureEscalationDispatcher(ctx context.Context, cfg *config.Config, notifier *notify.Notifier, audit crux.AuditSink, logger *slog.Logger) { escalationRuntime.Lock() defer escalationRuntime.Unlock() var key *escalationRuntimeKey @@ -70,7 +70,7 @@ func configureEscalationDispatcher(ctx context.Context, cfg *config.Config, noti // buildEscalationDispatcher returns nil when no sink is configured. A new // email sink inherits prev's pending digest. -func buildEscalationDispatcher(ctx context.Context, cfg *config.Config, notifier *notify.Notifier, audit agent.AuditSink, logger *slog.Logger, prev *escalate.EmailSink) (*escalate.Dispatcher, *escalate.EmailSink) { +func buildEscalationDispatcher(ctx context.Context, cfg *config.Config, notifier *notify.Notifier, audit crux.AuditSink, logger *slog.Logger, prev *escalate.EmailSink) (*escalate.Dispatcher, *escalate.EmailSink) { auditFn := func(action, detail, sink string) { if audit != nil { audit.Record("system", action, sink, map[string]any{"detail": detail}) @@ -158,7 +158,7 @@ func emitReviewHumanEscalations(plan review.DispatchPlan) { } } -func emitAgentPauseEscalation(event agent.PauseTransitionEvent) { +func emitAgentPauseEscalation(event crux.PauseTransitionEvent) { if !event.Paused { return } diff --git a/src/cmd/hive/escalation_runtime_test.go b/src/cmd/hive/escalation_runtime_test.go index 9f25d0cf2e..0d3a3a2f80 100644 --- a/src/cmd/hive/escalation_runtime_test.go +++ b/src/cmd/hive/escalation_runtime_test.go @@ -5,7 +5,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/cruxtest" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/escalate" ) @@ -42,7 +43,7 @@ func installTestEscalationDispatcher(t *testing.T) *recordingEscalationSink { func TestAgentPauseEmitterPagesThroughRealPausePath(t *testing.T) { sink := installTestEscalationDispatcher(t) - mgr := agent.NewManager(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, hookTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(map[string]config.AgentConfig{"scanner": {Backend: "claude"}}, hookTestLogger(), crux.ProjectContext{}) installAgentPauseEmitter(mgr) if err := mgr.PauseBy("scanner", "dashboard-api", "operator maintenance", "owner"); err != nil { t.Fatal(err) diff --git a/src/cmd/hive/eval_cycle_run_7232_test.go b/src/cmd/hive/eval_cycle_run_7232_test.go index d96fdb093a..b3f6dd5371 100644 --- a/src/cmd/hive/eval_cycle_run_7232_test.go +++ b/src/cmd/hive/eval_cycle_run_7232_test.go @@ -10,7 +10,9 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/beads" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" @@ -57,7 +59,7 @@ type evalCycleFixture struct { gh *github.Client gov *governor.Governor sched *scheduler.Scheduler - agentMgr *agent.Manager + agentMgr cruxtest.Runtime dashSrv *dashboard.Server last *atomic.Pointer[github.ActionableResult] hits []string @@ -83,7 +85,7 @@ func newEvalCycleFixture(t *testing.T) *evalCycleFixture { f.gh = github.NewClientForTest(api.URL, "testorg", []string{"widget"}, logger) f.gov = governor.New(f.cfg.Governor, f.cfg.Agents, logger) f.sched = scheduler.New(f.cfg, logger) - f.agentMgr = agent.NewManager(f.cfg.Agents, logger, agent.ProjectContext{}) + f.agentMgr = cruxtest.NewRuntime(f.cfg.Agents, logger, crux.ProjectContext{}) f.dashSrv = dashboard.NewServer(0, logger) return f } diff --git a/src/cmd/hive/eval_cycle_seams.go b/src/cmd/hive/eval_cycle_seams.go index 40330f9e4b..159190b43b 100644 --- a/src/cmd/hive/eval_cycle_seams.go +++ b/src/cmd/hive/eval_cycle_seams.go @@ -10,9 +10,9 @@ import ( "strings" "time" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/advisory" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/governor" @@ -165,7 +165,7 @@ func mergeResumeKicks(agentsDue, restartedAgents []string, allow func(string) (b return agentsDue, held } -// resumeKickStatusSource is the subset of *agent.Manager the resume-kick-held +// resumeKickStatusSource is the subset of crux.Runtime the resume-kick-held // reconcile reads. type resumeKickStatusSource interface { GetStatusFast(name string) (*crux.Process, error) diff --git a/src/cmd/hive/eval_cycle_seams_test.go b/src/cmd/hive/eval_cycle_seams_test.go index 6958fbb8df..72be1b032a 100644 --- a/src/cmd/hive/eval_cycle_seams_test.go +++ b/src/cmd/hive/eval_cycle_seams_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/scheduler" diff --git a/src/cmd/hive/extwork_claude_managed_agents.go b/src/cmd/hive/extwork_claude_managed_agents.go new file mode 100644 index 0000000000..8e357094e1 --- /dev/null +++ b/src/cmd/hive/extwork_claude_managed_agents.go @@ -0,0 +1,15 @@ +//go:build extwork_claude_managed_agents + +package main + +import ( + "github.com/hivecommons/hive/pkg/extwork" + "github.com/hivecommons/hive/pkg/extwork/managedagents" +) + +// Claude Managed Agents is linked only when the private hosted spoke explicitly +// opts into the build tag. Configuration alone cannot add a vendor client or a +// credential-bearing backend to an ordinary hive binary. +func init() { + extwork.DefaultRegistry.Register(managedagents.Engine, managedagents.Factory) +} diff --git a/src/cmd/hive/extworkwire.go b/src/cmd/hive/extworkwire.go index 0f9b2883f2..1a14545faf 100644 --- a/src/cmd/hive/extworkwire.go +++ b/src/cmd/hive/extworkwire.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "path/filepath" + "strconv" "strings" "sync" "time" @@ -19,17 +20,20 @@ import ( // External-execution wiring (#8361). pkg/dashboard owns the extwork-free // seams (lease authority, hub accessor, ExternalExecution status); this file // binds the concrete pkg/extwork pieces to them. Each adapter is registered -// only under its build tag (extwork_flue.go, extwork_omp.go), so without the +// only under its build tag (extwork_flue.go, extwork_omp.go, +// extwork_vibekanban.go, extwork_claude_managed_agents.go), so without the // tag the registry lacks that engine and its constructor here fails closed // with extwork.ErrEngineNotLinked. -// extExecEngineFlue and extExecEngineOMP mirror dashboard's private engine -// names; the adapter packages cannot be imported here without linking them -// into every build. +// extExecEngineFlue, extExecEngineOMP, extExecEngineVibeKanban, and +// extExecEngineClaudeManagedAgents mirror dashboard's private engine names; +// the adapter packages cannot be imported here without linking them into every +// build. const ( - extExecEngineFlue = "flue" - extExecEngineOMP = "omp" - extExecEngineVibeKanban = "vibe-kanban" + extExecEngineFlue = "flue" + extExecEngineOMP = "omp" + extExecEngineVibeKanban = "vibe-kanban" + extExecEngineClaudeManagedAgents = "claude-managed-agents" ) var ( @@ -129,6 +133,8 @@ func (s *extworkStatus) bindingFor(engine string) (*externalExecutionBinding, er b, err = newExternalOMPExecutionBinding(s.srv, s.cfg, s.registry, s.now) case extExecEngineVibeKanban: b, err = newExternalVibeKanbanExecutionBinding(s.srv, s.cfg, s.registry, s.now) + case extExecEngineClaudeManagedAgents: + b, err = newExternalClaudeManagedAgentsExecutionBinding(s.srv, s.cfg, s.registry, s.now) default: err = fmt.Errorf("%w: %s", extwork.ErrEngineNotLinked, engine) } @@ -156,6 +162,26 @@ func configForEngine(cfg *config.Config, engine string) string { return strings.Join([]string{cfg.VibeKanbanBindingMode(), vk.MCPCommand, vk.ProjectID, vk.StateDir, vk.WorkflowVersion}, "\x00") case extExecEngineOMP: return strings.Join([]string{cfg.OMPBindingMode(), cfg.Runs.External.OMP.WorkflowVersion}, "\x00") + case extExecEngineClaudeManagedAgents: + m := cfg.Runs.External.ClaudeManagedAgents + return strings.Join([]string{ + cfg.ClaudeManagedAgentsBindingMode(), + m.Endpoint, + m.WorkflowVersion, + m.AgentID, + strconv.Itoa(m.AgentVersion), + m.ToolsetVersion, + m.EnvironmentID, + m.EnvironmentType, + m.NetworkPolicy, + m.APIKeyEnv, + m.APIKeyFile, + m.Model, + m.StateDir, + strconv.FormatInt(m.DailyBudgetCents, 10), + m.GitHubWritePath, + strconv.FormatBool(m.DirectGitHubWrite), + }, "\x00") default: return "" } @@ -334,3 +360,48 @@ func newExternalOMPExecutionBinding(srv *dashboard.Server, cfg *config.Config, r store := extwork.NewLeaseStore(hubLeaseAuthority{hub: hub, now: now}, extworkRecordDir()) return &externalExecutionBinding{engine: extExecEngineOMP, adapter: adapter, binding: extwork.New(adapter, store, store, extwork.NewAuditProgressSink(srv.AgentAuditSink()), mode)}, nil } + +// newExternalClaudeManagedAgentsExecutionBinding builds the private-spoke +// hosted binding. It is intentionally yaml-only: the API key is resolved by +// the tagged adapter from a secret reference at request time. +func newExternalClaudeManagedAgentsExecutionBinding(srv *dashboard.Server, cfg *config.Config, registry *extwork.Registry, now func() time.Time) (*externalExecutionBinding, error) { + mode := cfg.ClaudeManagedAgentsBindingMode() + if mode == config.FlueBindingModeOff { + return nil, errExternalBindingOff + } + if now == nil { + now = time.Now + } + m := cfg.Runs.External.ClaudeManagedAgents + stateDir := strings.TrimSpace(m.StateDir) + if stateDir == "" { + stateDir = filepath.Join(extworkRecordDir(), extExecEngineClaudeManagedAgents) + } + adapter, err := registry.Open(extExecEngineClaudeManagedAgents, map[string]string{ + extwork.SettingEndpoint: m.Endpoint, + extwork.SettingWorkflowVersion: m.WorkflowVersion, + "agent_id": m.AgentID, + "agent_version": strconv.Itoa(m.AgentVersion), + "toolset_version": m.ToolsetVersion, + "environment_id": m.EnvironmentID, + "environment_type": m.EnvironmentType, + "network_policy": m.NetworkPolicy, + "api_key_env": m.APIKeyEnv, + "api_key_file": m.APIKeyFile, + "model": m.Model, + "state_dir": stateDir, + "metrics_dir": cfg.Data.MetricsDir, + "daily_budget_cents": strconv.FormatInt(m.DailyBudgetCents, 10), + "direct_github_write": strconv.FormatBool(m.DirectGitHubWrite), + "github_write_path": m.GitHubWritePath, + }) + if err != nil { + return nil, err + } + hub := srv.ContributeHub() + if hub == nil { + return nil, errExternalHubNotRunning + } + store := extwork.NewLeaseStore(hubLeaseAuthority{hub: hub, now: now}, extworkRecordDir()) + return &externalExecutionBinding{engine: extExecEngineClaudeManagedAgents, adapter: adapter, binding: extwork.New(adapter, store, store, extwork.NewAuditProgressSink(srv.AgentAuditSink()), mode)}, nil +} diff --git a/src/cmd/hive/forge_merge_test.go b/src/cmd/hive/forge_merge_test.go new file mode 100644 index 0000000000..60d111c630 --- /dev/null +++ b/src/cmd/hive/forge_merge_test.go @@ -0,0 +1,170 @@ +package main + +import ( + "context" + "errors" + "path/filepath" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/forge" + "github.com/hivecommons/hive/pkg/github" +) + +// fakeMerger records Merge calls in place of a real GitLab/Gitea adapter. +type fakeMerger struct { + kind forge.Kind + calls []fakeMergeCall + err error +} + +type fakeMergeCall struct { + repo string + number int + opts forge.MergeOptions +} + +func (f *fakeMerger) Merge(_ context.Context, repo string, number int, opts forge.MergeOptions) error { + f.calls = append(f.calls, fakeMergeCall{repo: repo, number: number, opts: opts}) + return f.err +} + +// withFakeForgeMergers swaps newForgeMerger for one handing out a fake per +// forge kind, and returns the fakes. +func withFakeForgeMergers(t *testing.T) map[forge.Kind]*fakeMerger { + t.Helper() + fakes := map[forge.Kind]*fakeMerger{ + forge.KindGitLab: {kind: forge.KindGitLab}, + forge.KindGitea: {kind: forge.KindGitea}, + } + orig := newForgeMerger + newForgeMerger = func(kind forge.Kind, _ string, _ forge.Options) (forge.Merger, error) { + f, ok := fakes[kind] + if !ok { + t.Fatalf("newForgeMerger called for unexpected kind %q", kind) + } + return f, nil + } + t.Cleanup(func() { newForgeMerger = orig }) + return fakes +} + +func mixedForgeConfig() *config.Config { + cfg := &config.Config{} + cfg.Project.Org = "acme" + cfg.Project.Repos = []string{"app", "lab", "tea"} + cfg.Project.RepoPolicies = []config.RepoPolicy{ + {Repo: "lab", Forge: config.ForgeGitLab, ForgeURL: "https://gitlab.example.com"}, + {Repo: "tea", Forge: config.ForgeGitea, ForgeURL: "https://codeberg.org"}, + } + return cfg +} + +// A GitHub repo (explicit, defaulted, or an unknown kind that falls back to +// GitHub) gets no forge merger, so the relay keeps its GitHub path unchanged. +func TestForgeMergeSelectorKeepsGitHubOnGitHubPath(t *testing.T) { + withFakeForgeMergers(t) + cfg := mixedForgeConfig() + sel := forgeMergeSelector(func() *config.Config { return cfg }, discardLogger()) + if sel("acme/app") != nil { + t.Fatal("GitHub repo must stay on the relay's GitHub path (nil forge merge)") + } + if sel("lab") == nil || sel("acme/tea") == nil { + t.Fatal("GitLab/Gitea repos must route through a forge merger") + } + if forgeMergeSelector(func() *config.Config { return nil }, discardLogger())("acme/tea") != nil { + t.Fatal("nil config must keep the GitHub path") + } + if m, err := forgeMergerForRepo(cfg, "acme/app"); m != nil || err != nil { + t.Fatalf("forgeMergerForRepo(GitHub) = (%v, %v), want (nil, nil)", m, err) + } +} + +// End-to-end through merge-eligibility on a non-GitHub repo: the governor +// writes the Gitea PR to merge-eligible.json, the relay's target binding +// authorizes it only at the reviewed head, and the merge lands through the +// Gitea forge.Merger (never the GitHub client) pinned to that head. +func TestForgeMergeThroughMergeEligibilityOnGitea(t *testing.T) { + dir := t.TempDir() + origMerge, origFail := mergeEligiblePath, ciFailingPath + mergeEligiblePath = filepath.Join(dir, "merge-eligible.json") + ciFailingPath = filepath.Join(dir, "ci-failing.json") + t.Cleanup(func() { + mergeEligiblePath = origMerge + ciFailingPath = origFail + }) + fakes := withFakeForgeMergers(t) + cfg := mixedForgeConfig() + + const head = "c0ffee1234" + actionable := &github.ActionableResult{PRs: github.PRResult{Items: []github.PullRequest{ + {Repo: "acme/tea", Number: 7, Title: "green", Author: "agent", CIStatus: "success", Mergeable: github.MergeableYes, HeadSHA: head}, + {Repo: "acme/tea", Number: 8, Title: "red", Author: "agent", CIStatus: "failure", FailingChecks: []string{"test"}, HeadSHA: "bad"}, + }}} + writeMergeEligible(actionable, github.HoldResult{}, "acme", nil, false, nil, false, nil, nil, nil, discardLogger()) + + authz := bindMergeAuthz(func(string, int, string) error { return nil }) + sel := forgeMergeSelector(func() *config.Config { return cfg }, discardLogger()) + ctx := context.Background() + + // relay mirrors the watcher's order: authorize (merge-eligible binding), + // then route to the repo's forge. + relay := func(repo string, number int, sha string) error { + if err := authz("scanner", 0, repo, number, sha); err != nil { + return err + } + merge := sel(repo) + if merge == nil { + t.Fatalf("%s routed to the GitHub path", repo) + } + return merge(ctx, repo, number, "squash", sha) + } + + if err := relay("acme/tea", 8, "bad"); err == nil { + t.Fatal("a ci-failing PR must not be authorized") + } + if err := relay("acme/tea", 7, "moved"); err == nil { + t.Fatal("an eligible PR at a moved head must not be authorized") + } + if len(fakes[forge.KindGitea].calls) != 0 { + t.Fatalf("refused targets reached the forge: %+v", fakes[forge.KindGitea].calls) + } + + if err := relay("acme/tea", 7, head); err != nil { + t.Fatalf("eligible Gitea PR at the reviewed head: %v", err) + } + calls := fakes[forge.KindGitea].calls + if len(calls) != 1 { + t.Fatalf("Gitea merges = %d, want 1", len(calls)) + } + if c := calls[0]; c.repo != "acme/tea" || c.number != 7 || c.opts.HeadSHA != head || c.opts.Strategy != string(forge.MergeStrategySquash) { + t.Fatalf("merge call = %+v, want acme/tea#7 squash pinned at %s", c, head) + } + if len(fakes[forge.KindGitLab].calls) != 0 { + t.Fatal("a Gitea repo must not merge through the GitLab adapter") + } +} + +// A retried request whose first attempt already landed is success; any other +// forge refusal is surfaced to the relay's retry path. +func TestForgeMergeErrors(t *testing.T) { + fakes := withFakeForgeMergers(t) + cfg := mixedForgeConfig() + merge := forgeMergeSelector(func() *config.Config { return cfg }, discardLogger())("acme/lab") + + fakes[forge.KindGitLab].err = forge.ErrAlreadyMerged + if err := merge(context.Background(), "acme/lab", 3, "squash", "abc"); err != nil { + t.Fatalf("already merged should be success, got %v", err) + } + fakes[forge.KindGitLab].err = forge.ErrConflict + if err := merge(context.Background(), "acme/lab", 3, "squash", "abc"); !errors.Is(err, forge.ErrConflict) { + t.Fatalf("err = %v, want ErrConflict", err) + } + + newForgeMerger = func(forge.Kind, string, forge.Options) (forge.Merger, error) { + return nil, errors.New("no url") + } + if err := merge(context.Background(), "acme/lab", 3, "squash", "abc"); err == nil { + t.Fatal("an unbuildable adapter must fail closed, not fall back to GitHub") + } +} diff --git a/src/cmd/hive/forgewire.go b/src/cmd/hive/forgewire.go index 7bd5f9ad3c..eb22c77b44 100644 --- a/src/cmd/hive/forgewire.go +++ b/src/cmd/hive/forgewire.go @@ -1,6 +1,9 @@ package main import ( + "context" + "errors" + "fmt" "log/slog" "os" @@ -12,7 +15,8 @@ import ( // governorForge returns the forge-neutral write seam the governor path should // use for this config, or nil when no forge can be reached at all. // -// Selection follows project.forge: +// Selection follows project.forge, or per repo when any +// project.repo_policies[] entry sets forge/forge_url (see repoForgeRouter): // // - "github" (and unset, the default): ghClient itself. No adapter is // interposed, so the GitHub path is byte-for-byte what it was before. @@ -31,6 +35,19 @@ import ( // which is why this is called per governor cycle instead of being memoized: a // hot config reload takes effect on the next tick with no invalidation logic. func governorForge(cfg *config.Config, ghClient *github.Client, logger *slog.Logger) forge.IssueWriter { + // Per-repo forge overrides (project.repo_policies[].forge / forge_url, + // hivecommons/hive#14112) need a writer that picks the forge per call. A + // hive with none keeps the single project-wide writer below, unchanged. + if cfg.HasRepoForgeOverrides() { + return &repoForgeRouter{cfg: cfg, ghClient: ghClient, logger: logger} + } + return forgeForRepo(cfg, "", ghClient, logger) +} + +// forgeForRepo is governorForge for one repo: the writer for the forge repo +// lives on (its repo_policies override, else project.forge). An empty repo +// resolves the project-wide forge. Fallback rules are governorForge's. +func forgeForRepo(cfg *config.Config, repo string, ghClient *github.Client, logger *slog.Logger) forge.IssueWriter { // A typed-nil *github.Client stored in an interface is NOT == nil, and the // callers' "no client, do nothing" guards test the interface. Normalize it // to an untyped nil here so those guards keep working. @@ -49,16 +66,16 @@ func governorForge(cfg *config.Config, ghClient *github.Client, logger *slog.Log baseURL string tokenID string ) - switch k := cfg.Project.ForgeKind(); k { + switch k := cfg.ForgeKindForRepo(repo); k { case config.ForgeGitHub: return fallback() case config.ForgeGitLab: - kind, baseURL, tokenID = forge.KindGitLab, cfg.GitLab.InstanceURL(), cfg.GitLab.TokenEnvName() + kind, baseURL, tokenID = forge.KindGitLab, cfg.ForgeURLForRepo(repo), cfg.GitLab.TokenEnvName() case config.ForgeGitea: - kind, baseURL, tokenID = forge.KindGitea, cfg.Gitea.InstanceURL(), cfg.Gitea.TokenEnvName() + kind, baseURL, tokenID = forge.KindGitea, cfg.ForgeURLForRepo(repo), cfg.Gitea.TokenEnvName() default: logger.Warn("unknown project.forge; falling back to the GitHub client", - "forge", k, "known", []string{config.ForgeGitHub, config.ForgeGitLab, config.ForgeGitea}) + "forge", k, "repo", repo, "known", config.KnownForgeKinds) return fallback() } @@ -73,8 +90,114 @@ func governorForge(cfg *config.Config, ghClient *github.Client, logger *slog.Log }) if err != nil { logger.Warn("forge adapter unavailable; falling back to the GitHub client", - "forge", kind, "token_env", tokenID, "error", err) + "forge", kind, "repo", repo, "token_env", tokenID, "error", err) return fallback() } return f } + +// repoForgeRouter is the governor's writer on a hive whose repos do not all +// live on one forge: each write goes to the forge its repo argument names. +type repoForgeRouter struct { + cfg *config.Config + ghClient *github.Client + logger *slog.Logger +} + +func (r *repoForgeRouter) writerFor(repo string) (forge.IssueWriter, error) { + w := forgeForRepo(r.cfg, repo, r.ghClient, r.logger) + if w == nil { + return nil, fmt.Errorf("forge: no writer reachable for repo %q (%s)", repo, r.cfg.ForgeKindForRepo(repo)) + } + return w, nil +} + +func (r *repoForgeRouter) CreateIssueComment(ctx context.Context, repo string, number int, body string) error { + w, err := r.writerFor(repo) + if err != nil { + return err + } + return w.CreateIssueComment(ctx, repo, number, body) +} + +func (r *repoForgeRouter) AddLabels(ctx context.Context, repo string, number int, labels []string) error { + w, err := r.writerFor(repo) + if err != nil { + return err + } + return w.AddLabels(ctx, repo, number, labels) +} + +// newForgeMerger builds the merge primitive for a non-GitHub forge. It is a +// variable so tests can substitute a fake forge. +var newForgeMerger = func(kind forge.Kind, token string, opts forge.Options) (forge.Merger, error) { + f, err := forge.NewForge(kind, token, opts) + if err != nil { + return nil, err + } + m, ok := f.(forge.Merger) + if !ok { + return nil, fmt.Errorf("forge %q adapter does not implement merge", kind) + } + return m, nil +} + +// forgeMergerForRepo returns the forge.Merger for a repo that lives on GitLab +// or Gitea, or (nil, nil) for a GitHub repo — and for an unknown kind, which +// falls back to GitHub exactly as forgeForRepo does — so the GitHub merge path +// (MergePR on the concrete client) stays byte-for-byte unchanged. Unlike the +// comment/label writer, a GitLab/Gitea repo whose adapter cannot be built +// returns an error rather than falling back: merging it through the GitHub +// client could never succeed and must not be attempted. +func forgeMergerForRepo(cfg *config.Config, repo string) (forge.Merger, error) { + var kind forge.Kind + var tokenID string + switch cfg.ForgeKindForRepo(repo) { + case config.ForgeGitLab: + kind, tokenID = forge.KindGitLab, cfg.GitLab.TokenEnvName() + case config.ForgeGitea: + kind, tokenID = forge.KindGitea, cfg.Gitea.TokenEnvName() + default: + return nil, nil + } + return newForgeMerger(kind, os.Getenv(tokenID), forge.Options{ + BaseURL: cfg.ForgeURLForRepo(repo), + Org: cfg.Project.Org, + }) +} + +// forgeMergeSelector is the merge relay's per-repo forge router: a repo whose +// forge.Kind is GitLab or Gitea merges through that forge's Merger at the +// pinned head; a GitHub repo gets nil and keeps the relay's GitHub path. cfg +// is read per request so a reload takes effect without a restart. The +// merge-eligible binding (bindMergeAuthz) has already run by the time the +// returned function is called — it is the same gate on every forge. +func forgeMergeSelector(cfgFn func() *config.Config, logger *slog.Logger) github.ForgeMergeSelector { + return func(repo string) github.ForgeMergeFunc { + cfg := cfgFn() + if cfg == nil { + return nil + } + kind := cfg.ForgeKindForRepo(repo) + if kind != config.ForgeGitLab && kind != config.ForgeGitea { + return nil + } + return func(ctx context.Context, repo string, number int, method, expectSHA string) error { + m, err := forgeMergerForRepo(cfg, repo) + if err != nil { + return fmt.Errorf("forge merge unavailable for %s (%s): %w", repo, kind, err) + } + if m == nil { + return fmt.Errorf("forge merge unavailable for %s (%s)", repo, kind) + } + err = m.Merge(ctx, repo, number, forge.MergeOptions{Strategy: method, HeadSHA: expectSHA}) + if errors.Is(err, forge.ErrAlreadyMerged) { + // A retried request whose first attempt landed: the desired + // end state is reached. + logger.Info("forge merge: change request already merged", "repo", repo, "number", number, "forge", kind) + return nil + } + return err + } + } +} diff --git a/src/cmd/hive/forgewire_test.go b/src/cmd/hive/forgewire_test.go index fd9ecc2899..a9694927ca 100644 --- a/src/cmd/hive/forgewire_test.go +++ b/src/cmd/hive/forgewire_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "testing" "github.com/hivecommons/hive/pkg/config" @@ -123,3 +124,60 @@ func TestGovernorForgeFallsBackOnUnusableConfig(t *testing.T) { }) } } + +// TestConfigForgeKindsMatchForgeKinds pins the by-value contract between +// config's forge names and pkg/forge.Kind: config cannot import pkg/forge, so +// a rename on either side must fail here rather than silently fall back. +func TestConfigForgeKindsMatchForgeKinds(t *testing.T) { + want := []forge.Kind{forge.KindGitHub, forge.KindGitLab, forge.KindGitea} + if len(config.KnownForgeKinds) != len(want) { + t.Fatalf("config.KnownForgeKinds = %v, want %v", config.KnownForgeKinds, want) + } + for i, k := range want { + if config.KnownForgeKinds[i] != string(k) { + t.Fatalf("config.KnownForgeKinds[%d] = %q, want %q", i, config.KnownForgeKinds[i], k) + } + } +} + +// TestGovernorForgeRoutesPerRepo covers project.repo_policies[].forge +// (hivecommons/hive#14112): one hive watching a GitHub, a GitLab and a Gitea +// repo hands each write to the forge its repo lives on. +func TestGovernorForgeRoutesPerRepo(t *testing.T) { + gh := github.NewClient("t", "acme", nil, discardLogger(), "") + cfg := &config.Config{} + cfg.Project.Org = "acme" + cfg.Project.Repos = []string{"app", "lab", "tea"} + cfg.Project.RepoPolicies = []config.RepoPolicy{ + {Repo: "lab", Forge: config.ForgeGitLab, ForgeURL: "https://gitlab.example.com"}, + {Repo: "tea", Forge: config.ForgeGitea, ForgeURL: "https://codeberg.org"}, + } + + w := governorForge(cfg, gh, discardLogger()) + router, ok := w.(*repoForgeRouter) + if !ok { + t.Fatalf("want a per-repo router when repo_policies set a forge, got %T", w) + } + cases := map[string]forge.Kind{"acme/app": "", "acme/lab": forge.KindGitLab, "acme/tea": forge.KindGitea} + for repo, want := range cases { + got, err := router.writerFor(repo) + if err != nil { + t.Fatalf("writerFor(%q): %v", repo, err) + } + if want == "" { + if got != forge.IssueWriter(gh) { + t.Fatalf("writerFor(%q) = %T, want the *github.Client itself", repo, got) + } + continue + } + if k := forgeKindOf(t, got); k != want { + t.Fatalf("writerFor(%q) kind = %q, want %q", repo, k, want) + } + } + + // No GitHub client and a GitHub repo: a clean error, never a nil deref. + noGH := &repoForgeRouter{cfg: cfg, logger: discardLogger()} + if err := noGH.AddLabels(context.Background(), "acme/app", 1, []string{"x"}); err == nil { + t.Fatal("want an error for a GitHub repo with no GitHub client") + } +} diff --git a/src/cmd/hive/github_client_config.go b/src/cmd/hive/github_client_config.go index 9f5c0205ad..19f78f5c51 100644 --- a/src/cmd/hive/github_client_config.go +++ b/src/cmd/hive/github_client_config.go @@ -6,7 +6,9 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/agent" + cruxconfig "github.com/hivecommons/hive/crux/config" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/beads" "github.com/hivecommons/hive/pkg/claims" "github.com/hivecommons/hive/pkg/dashboard" @@ -124,6 +126,7 @@ func (b *boot) applyGitHubClientConfigHooks(client *github.Client) { // gates which issues become actionable at all, so it must be installed // even when no exempt labels are configured. client.SetIssueFilter(b.cfg.Project.IssueFilter) + b.applyPriorArtFilingGate(client) client.SetAgentIssueDecisionPolicy(b.cfg.Project.EffectiveAgentIssueDecisionPolicy()) client.SetReporterConfirmationEnabledFunc(func() bool { return b.cfg.Issues.ReporterConfirmationEnabled() @@ -200,7 +203,7 @@ func (b *boot) prRepoPolicyGate(agentName, repo string) error { if level <= 0 { level = b.cfg.ACMMLevelOrZero() } - if !agent.DefaultAgentMode(agentName, level).CanPush() { + if !crux.DefaultAgentMode(agentName, level).CanPush() { return fmt.Errorf("repo %s effective ACMM L%d does not allow %s to open PRs", repo, level, agentName) } return nil @@ -315,7 +318,7 @@ func (b *boot) applyGitHubClientAgentHooks(client *github.Client) { // Same resolver the Manager uses, not a second copy of the // rule: a hardcoded default here would drift silently the // moment agy's default effort changed. - effort = agent.ResolveReasoningEffort(backend, model, ac.ReasoningEffort) + effort = cruxconfig.ResolveReasoningEffort(backend, model, ac.ReasoningEffort) } } tool, toolVersion := github.ResolveToolVersion(backend) @@ -398,9 +401,14 @@ func (b *boot) applyGitHubClientDashboardHooks(client *github.Client) { client.SetAttributionAuditRecord(func(rec github.AuditRecord) { b.dashSrv.AuditLogRecord("system", rec.Action, rec.Detail, rec.Agent, rec.Repo, rec.Target) recordLifecycleFromAudit(b.dashSrv, b.cfg.Project.Org, rec) + b.dashSrv.ApplyPRTerminalAudit(rec) // #10927/#10933: the agent card's "what is it on" and stale warning // read each agent's last hive-mediated write. dashboard.RecordAgentLastAction(rec, time.Now()) + // A PR the hive merged or closed itself leaves the tracker and the + // governor's cached snapshot now, instead of waiting for GitHub's + // `closed` delivery to come back round through the hub. + evictHiveClosedPR(b.dashSrv, rec, time.Now()) }) client.SetPRTerminalObservedHook(func(obs github.PRTerminalObservation) { ts := obs.ObservedAt.UTC().Format(time.RFC3339) @@ -471,6 +479,10 @@ func (b *boot) applyGitHubClientDashboardHooks(client *github.Client) { // The issues the PR fixes get their own pr_opened stage: the // start→PR SLA boundary, and the link a later merge folds back on. recordPROpenedForIssues(b.dashSrv, b.cfg.Project.Org, d.Agent, d.Repo, d.Number, d.URL, d.Issues) + // The duplicate-PR claim ledger learns about the PR now, not at the + // next scan, so a contributor cannot be admitted to an issue this + // PR already covers in the gap between (hivecommons/hive#14472). + client.RecordOpenedPRClaims(getClaimLedger(b.logger), d, hiveIdentity(b.cfg), time.Now()) }) client.SetCanaryScanner(b.cfg.Ioscan.IsEnabled() && b.cfg.Ioscan.CanariesEnabled(), b.cfg.Ioscan.FailClosedAtLevel(b.cfg.ACMMLevelOrZero()), ioscan.DefaultCanaries, b.handleCanaryLeak) } @@ -505,3 +517,20 @@ func holdMigrationFailedAlertMessage(hiveID, reportPath string) string { } return msg } + +// evictHiveClosedPR removes a pull request from the tracker and the cached +// actionable snapshot the moment the hive records having merged or closed it +// (AuditActionPRMerged / AuditActionPRClosed). The audit record is the one +// point every merge path (sweep, queue, relay, forge) passes through, so it +// is where the hive's own write stops depending on the webhook round-trip. +func evictHiveClosedPR(srv *dashboard.Server, rec github.AuditRecord, now time.Time) { + if srv == nil || rec.Target <= 0 || rec.Repo == "" { + return + } + switch rec.Action { + case github.AuditActionPRMerged: + srv.EvictHiveClosedItem(rec.Repo, rec.Target, true, true, rec.Action, now) + case github.AuditActionPRClosed: + srv.EvictHiveClosedItem(rec.Repo, rec.Target, true, false, rec.Action, now) + } +} diff --git a/src/cmd/hive/github_client_config_evict_test.go b/src/cmd/hive/github_client_config_evict_test.go new file mode 100644 index 0000000000..bfa315e150 --- /dev/null +++ b/src/cmd/hive/github_client_config_evict_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/github" +) + +// The attribution audit sink is the one point every hive merge path passes +// through; a pr_merged or pr_closed record there must take the PR off the +// dashboard's listed items, and anything else must leave them alone. +func TestEvictHiveClosedPRFromAuditRecord(t *testing.T) { + now := time.Date(2026, 10, 26, 18, 25, 46, 0, time.UTC) + newSrv := func() *dashboard.Server { + srv := dashboard.NewServer(0, restoreTestLogger()) + srv.UpdateStatus(&dashboard.StatusPayload{Repos: []dashboard.FrontendRepo{{ + Name: "docs", Full: "hivecommons/docs", PRs: 2, + OpenPrs: []any{ + github.PullRequest{Repo: "hivecommons/docs", Number: 469, State: "open", UpdatedAt: now.Add(-time.Hour)}, + github.PullRequest{Repo: "hivecommons/docs", Number: 470, State: "open", UpdatedAt: now.Add(-time.Hour)}, + }, + }}}) + return srv + } + listed := func(srv *dashboard.Server) []int { + req := httptest.NewRequest(http.MethodGet, "/api/status", nil) + rec := httptest.NewRecorder() + srv.Handler().ServeHTTP(rec, req) + var out struct { + Repos []struct { + OpenPrs []struct{ Number int } `json:"openPrs"` + } `json:"repos"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("status body is not JSON: %v\n%s", err, rec.Body.String()) + } + var nums []int + for _, r := range out.Repos { + for _, pr := range r.OpenPrs { + nums = append(nums, pr.Number) + } + } + return nums + } + + cases := []struct { + name string + rec github.AuditRecord + want []int + }{ + {"pr_merged evicts", github.AuditRecord{Action: github.AuditActionPRMerged, Repo: "hivecommons/docs", Target: 469}, []int{470}}, + {"pr_closed evicts", github.AuditRecord{Action: github.AuditActionPRClosed, Repo: "hivecommons/docs", Target: 470}, []int{469}}, + {"other action ignored", github.AuditRecord{Action: github.AuditActionAgentPRCreated, Repo: "hivecommons/docs", Target: 469}, []int{469, 470}}, + {"no target ignored", github.AuditRecord{Action: github.AuditActionPRMerged, Repo: "hivecommons/docs"}, []int{469, 470}}, + {"no repo ignored", github.AuditRecord{Action: github.AuditActionPRMerged, Target: 469}, []int{469, 470}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + srv := newSrv() + evictHiveClosedPR(srv, tc.rec, now) + got := listed(srv) + if len(got) != len(tc.want) { + t.Fatalf("listed PRs = %v, want %v", got, tc.want) + } + for i := range got { + if got[i] != tc.want[i] { + t.Fatalf("listed PRs = %v, want %v", got, tc.want) + } + } + }) + } + evictHiveClosedPR(nil, github.AuditRecord{Action: github.AuditActionPRMerged, Repo: "hivecommons/docs", Target: 469}, now) +} diff --git a/src/cmd/hive/github_client_config_test.go b/src/cmd/hive/github_client_config_test.go index f1ee87e503..381e2179dc 100644 --- a/src/cmd/hive/github_client_config_test.go +++ b/src/cmd/hive/github_client_config_test.go @@ -34,7 +34,9 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/github" @@ -61,6 +63,10 @@ var clientSetterAllowlist = map[string]string{ "it on whichever client the relays run on, and every rebuilt client re-arms the relays " + "(armRequestRelays in github_client_provider.go), so the gate is reinstalled alongside the " + "merge authorizer it extends", + "SetForgeMergeSelector": "owned by the request-relay supervisor, like SetMergeRequestPolicyGate: " + + "startRequestRelays installs the per-repo forge router on whichever client the merge relay " + + "runs on, and every rebuilt client re-arms the relays (armRequestRelays), so the selector is " + + "reinstalled alongside the merge authorizer it routes (#14113)", "SetStageFactSink": "test seam: production uses the client's own default stage FactSink " + "(enforce-mode reconciler over stagelabels.Shared()), which every rebuilt client " + "constructs lazily on first use (ADR-0025, #13009)", @@ -85,7 +91,7 @@ const ( rebuildTestOpenRepo = "acme/open" rebuildTestLockedRep = "acme/locked" // rebuildTestHiveLevel lets "quality" open PRs; rebuildTestLockedLevel - // (advisory) does not. See agent.DefaultAgentMode. + // (advisory) does not. See crux.DefaultAgentMode. rebuildTestHiveLevel = 5 rebuildTestLockedLevel = 2 rebuildTestAgent = "quality" @@ -225,7 +231,7 @@ func TestConfigureGitHubClientTiersWaitForDependencies(t *testing.T) { t.Fatal("App policy tier (no dependencies) was not applied") } - b.agentMgr = agent.NewManager(map[string]config.AgentConfig{}, b.logger, agent.ProjectContext{}) + b.agentMgr = cruxtest.NewRuntime(map[string]config.AgentConfig{}, b.logger, crux.ProjectContext{}) b.dashSrv = dashboard.NewServer(0, b.logger) b.configureGitHubClient(client) b.configureGitHubClient(client) diff --git a/src/cmd/hive/github_client_provider.go b/src/cmd/hive/github_client_provider.go index efaabb1f7e..3c36263c2d 100644 --- a/src/cmd/hive/github_client_provider.go +++ b/src/cmd/hive/github_client_provider.go @@ -3,8 +3,8 @@ package main import ( "context" + "github.com/hivecommons/hive/crux/pushbroker" "github.com/hivecommons/hive/pkg/github" - "github.com/hivecommons/hive/pkg/pushbroker" ) // githubClientRef boxes the live client so the provider can tell "never @@ -55,6 +55,18 @@ func (b *boot) currentGitHubClient() *github.Client { // // newClient must already be configured (newConfiguredGitHubAppClient did // that); this only swaps references. +// appMinter builds a push-token minter over the GitHub App credential. The +// broker takes a pushbroker.TokenScoper interface, so a nil *AppAuth must be +// left as a nil interface here rather than stored as a typed nil the minter's +// nil check would miss. +func appMinter(auth *github.AppAuth, tier string) pushbroker.GitHubAppMinter { + m := pushbroker.GitHubAppMinter{Tier: tier} + if auth != nil { + m.Auth = auth + } + return m +} + func (b *boot) adoptGitHubClient(newClient *github.Client, newAppAuth *github.AppAuth) { if b == nil || newClient == nil { return @@ -69,7 +81,7 @@ func (b *boot) adoptGitHubClient(newClient *github.Client, newAppAuth *github.Ap // The sandbox kick executor opens PRs and mints push tokens itself; // leaving it on the old client/AppAuth is the stale pointer the // dashboard and heartbeat rebuilds used to miss. - b.agentMgr.SetSandboxPushMinter(pushbroker.GitHubAppMinter{Auth: newAppAuth}) + b.agentMgr.SetSandboxPushMinter(appMinter(newAppAuth, "")) b.agentMgr.SetSandboxPRClient(newClient) } if b.dashSrv != nil { diff --git a/src/cmd/hive/github_client_provider_test.go b/src/cmd/hive/github_client_provider_test.go index 3910deee89..77a595d62a 100644 --- a/src/cmd/hive/github_client_provider_test.go +++ b/src/cmd/hive/github_client_provider_test.go @@ -32,10 +32,12 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux" + + "github.com/hivecommons/hive/crux/cruxtest" + "github.com/hivecommons/hive/crux/pushbroker" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/github/automerge" - "github.com/hivecommons/hive/pkg/pushbroker" ) const ( @@ -262,8 +264,8 @@ func TestAdoptGitHubClientRepointsEveryConsumer(t *testing.T) { if b.currentGitHubClient() != rebuilt { t.Fatal("provider still serves the old client; collectors and the scheduler would stay stale") } - pr, minter := b.agentMgr.SandboxGitHubWiring() - if pr != agent.PRCreator(rebuilt) { + pr, minter := b.agentMgr.(cruxtest.Runtime).SandboxGitHubWiring() + if pr != crux.PRCreator(rebuilt) { t.Fatalf("sandbox PR client = %v, want the rebuilt client", pr) } if m, ok := minter.(pushbroker.GitHubAppMinter); !ok || m.Auth != rebuiltAuth { diff --git a/src/cmd/hive/heartbeat_helpers_test.go b/src/cmd/hive/heartbeat_helpers_test.go index d9d768cd4a..4a859381bc 100644 --- a/src/cmd/hive/heartbeat_helpers_test.go +++ b/src/cmd/hive/heartbeat_helpers_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux" spoke "github.com/hivecommons/hive/pkg/hub/spoke" ) diff --git a/src/cmd/hive/heartbeat_idle_cadence_test.go b/src/cmd/hive/heartbeat_idle_cadence_test.go index 2c6783bb7a..56aa4bebd1 100644 --- a/src/cmd/hive/heartbeat_idle_cadence_test.go +++ b/src/cmd/hive/heartbeat_idle_cadence_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux" "github.com/hivecommons/hive/pkg/governor" spoke "github.com/hivecommons/hive/pkg/hub/spoke" ) diff --git a/src/cmd/hive/hold_guard_mergeability_test.go b/src/cmd/hive/hold_guard_mergeability_test.go index d8ed3fb981..2daf51b550 100644 --- a/src/cmd/hive/hold_guard_mergeability_test.go +++ b/src/cmd/hive/hold_guard_mergeability_test.go @@ -42,6 +42,12 @@ func TestMaintainHeldPRMergeabilityUpdatesBehindBranchAndAdvancesBaseline(t *tes if calls != 1 { t.Fatalf("update-branch calls for PR 7 = %d, want 1", calls) } + fake.mu.Lock() + expected := fake.updateBranchExpected[7] + fake.mu.Unlock() + if len(expected) != 1 || expected[0] != "c1" { + t.Fatalf("update-branch expected_head_sha = %v, want [c1] (the enumerated head)", expected) + } rec, ok := store.Recorded("acme/widgets", 7) if !ok || rec.HeadSHA != "c2" { t.Fatalf("baseline = %+v ok=%v, want it advanced to the merged head c2", rec, ok) @@ -211,6 +217,45 @@ func TestMaintainHeldPRMergeabilityNotesUnresolvableConflictOnce(t *testing.T) { } } +// #14537: the update is pinned to the enumerated head, so a contributor push +// landing between enumeration and the update makes GitHub refuse it instead of +// stacking the hive's merge on unevaluated commits. That refusal is a moved +// head, not a merge conflict: no conflict notice, baseline untouched. +func TestMaintainHeldPRMergeabilityRefusedWhenHeadMovedSinceEnumeration(t *testing.T) { + for _, state := range []string{"behind", "dirty"} { + t.Run(state, func(t *testing.T) { + store := newTestHoldGuardStore(t) + fake := &holdGuardServer{ + commits: map[int][]map[string]any{7: {ghCommit("c1", "strategist-bot", "docs: plan"), ghCommit("c9", "contributor", "feat: late push")}}, + liveHead: map[int]string{7: "c9"}, + } + client := newHoldGuardClient(t, fake) + cfg := escalationTestConfig() + store.Snapshot("acme/widgets", 7, "c1", []holdguard.Commit{{SHA: "c1", Author: "strategist-bot"}}) + + held := heldPRsActionable(github.PullRequest{Repo: "widgets", Number: 7, HeadSHA: "c1", BaseSHA: "b1", MergeableState: state}) + _ = enforceHoldGuard(context.Background(), cfg, client, client, held, discardLogger()) + + fake.mu.Lock() + calls := fake.updateBranchCalls[7] + comments := append([]string(nil), fake.comments...) + fake.mu.Unlock() + if calls != 1 { + t.Fatalf("update-branch calls for PR 7 = %d, want 1", calls) + } + for _, c := range comments { + if strings.Contains(c, "conflict") { + t.Fatalf("a moved head must not be reported as a merge conflict, saw %q", c) + } + } + rec, ok := store.Recorded("acme/widgets", 7) + if !ok || rec.HeadSHA != "c1" || rec.ConflictNoted { + t.Fatalf("snapshot = %+v ok=%v, want baseline pinned at c1 and no conflict noted", rec, ok) + } + }) + } +} + func TestMaintainHeldPRMergeabilitySkipsUntrackedPR(t *testing.T) { newTestHoldGuardStore(t) fake := &holdGuardServer{commits: map[int][]map[string]any{42: {ghCommit("z1", "strategist-bot", "docs: plan")}}} diff --git a/src/cmd/hive/hold_guard_test.go b/src/cmd/hive/hold_guard_test.go index 9d5b62b138..ddaca7de95 100644 --- a/src/cmd/hive/hold_guard_test.go +++ b/src/cmd/hive/hold_guard_test.go @@ -95,6 +95,13 @@ type holdGuardServer struct { refuseStatus int refuseMessage string + // updateBranchExpected records the expected_head_sha each update-branch + // request carried; liveHead, when set for a PR, makes the endpoint refuse + // a request whose expected_head_sha differs, as GitHub does when the head + // moved after enumeration (#14537). + updateBranchExpected map[int][]string + liveHead map[int]string + // baseContains lists the SHAs the fake base branch contains; the compare // endpoint reports them "behind" the base and anything else "diverged" // (#14534: proving the merged parent of a hygiene merge is a base commit). @@ -177,6 +184,20 @@ func (s *holdGuardServer) handler(t *testing.T) http.Handler { http.Error(w, string(body), s.refuseStatus) return } + var opts struct { + ExpectedHeadSHA string `json:"expected_head_sha"` + } + if body, _ := io.ReadAll(r.Body); len(body) > 0 { + _ = json.Unmarshal(body, &opts) + } + if s.updateBranchExpected == nil { + s.updateBranchExpected = map[int][]string{} + } + s.updateBranchExpected[number] = append(s.updateBranchExpected[number], opts.ExpectedHeadSHA) + if live, ok := s.liveHead[number]; ok && opts.ExpectedHeadSHA != "" && opts.ExpectedHeadSHA != live { + http.Error(w, `{"message":"expected head sha didn't match current head ref."}`, http.StatusUnprocessableEntity) + return + } if s.failUpdateBranch { http.Error(w, `{"message":"Merge conflict"}`, http.StatusUnprocessableEntity) return diff --git a/src/cmd/hive/hookwire.go b/src/cmd/hive/hookwire.go index d0131c12dc..20dd378d3f 100644 --- a/src/cmd/hive/hookwire.go +++ b/src/cmd/hive/hookwire.go @@ -8,7 +8,7 @@ import ( "strings" "sync" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/hooks" @@ -57,7 +57,7 @@ func hookDispatcher() *hooks.Dispatcher { // than a silent no-op. type hookSinks struct { Notifier *notify.Notifier - AgentMgr *agent.Manager + AgentMgr crux.Runtime Timeline *timeline.Store Audit hooks.AuditSink Approvals hooks.ApprovalQueue // #4000; nil until that RFC lands @@ -185,11 +185,11 @@ func installGovernorModeChangeEmitter(gov *governor.Governor) { }) } -func installAgentPauseEmitter(mgr *agent.Manager) { +func installAgentPauseEmitter(mgr crux.Host) { if mgr == nil { return } - mgr.SetPauseTransitionObserver(func(event agent.PauseTransitionEvent) { + mgr.SetPauseTransitionObserver(func(event crux.PauseTransitionEvent) { emitAgentPauseEscalation(event) transition := hooks.TransitionAgentResumed if event.Paused { @@ -266,7 +266,7 @@ func hookPauseTriggerFor(cause hooks.Causation) string { // pauserAdapter routes the pause action through the AUDITED agent-manager // pause — the same entry point the dashboard's pause button uses. It never // writes the paused flag directly. -type pauserAdapter struct{ mgr *agent.Manager } +type pauserAdapter struct{ mgr crux.Runtime } func (a *pauserAdapter) PauseAgent(ctx context.Context, agentName, reason string, cause hooks.Causation) error { if a == nil || a.mgr == nil { @@ -291,14 +291,14 @@ func (a *pauserAdapter) PauseAgent(ctx context.Context, agentName, reason string // identity is the HOOK, not a person — PauseBy's contract is "never // fabricate" a human actor, so hookPauseActor is an explicitly // non-human identity that cannot be mistaken for a dashboard user. - return a.mgr.PauseByCause(agentName, hookPauseTriggerFor(cause), reason, hookPauseActor(cause), agent.PauseCausation{ + return a.mgr.PauseByCause(agentName, hookPauseTriggerFor(cause), reason, hookPauseActor(cause), crux.PauseCausation{ Depth: cause.Depth, HookName: cause.HookName, OriginTransition: string(cause.OriginTransition), }) } -type kickerAdapter struct{ mgr *agent.Manager } +type kickerAdapter struct{ mgr crux.Runtime } func (a *kickerAdapter) Kick(agentName, reason string) error { if a == nil || a.mgr == nil { diff --git a/src/cmd/hive/hookwire_test.go b/src/cmd/hive/hookwire_test.go index caa1473380..902db18b82 100644 --- a/src/cmd/hive/hookwire_test.go +++ b/src/cmd/hive/hookwire_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/hooks" "github.com/hivecommons/hive/pkg/timeline" @@ -311,9 +313,9 @@ func TestAgentPausedEmitterCarriesHookCausationAndStopsPauseLoop(t *testing.T) { resetHookDispatcher(t) t.Cleanup(func() { resetHookDispatcher(t) }) - mgr := agent.NewManager(map[string]config.AgentConfig{ + mgr := cruxtest.NewRuntime(map[string]config.AgentConfig{ "scanner": {Backend: "claude"}, - }, hookTestLogger(), agent.ProjectContext{}) + }, hookTestLogger(), crux.ProjectContext{}) audit := &hookWireAudit{ch: make(chan string, 8)} cfg := &config.Config{Hooks: []config.HookRule{{ Name: "pause-again", On: "agent_paused", Action: "pause", diff --git a/src/cmd/hive/inference_wrapper_seam_7238_test.go b/src/cmd/hive/inference_wrapper_seam_7238_test.go index 730a2ded84..cf33ca4c3f 100644 --- a/src/cmd/hive/inference_wrapper_seam_7238_test.go +++ b/src/cmd/hive/inference_wrapper_seam_7238_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" + "github.com/hivecommons/hive/crux/watsonx" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/watsonx" ) // The inference route and gateway resolvers moved to pkg/inference (#7238 diff --git a/src/cmd/hive/ioscan_classifier_boot.go b/src/cmd/hive/ioscan_classifier_boot.go index 20f2bf7039..2625e71c64 100644 --- a/src/cmd/hive/ioscan_classifier_boot.go +++ b/src/cmd/hive/ioscan_classifier_boot.go @@ -1,7 +1,10 @@ package main import ( + "fmt" "log/slog" + "sort" + "strings" "time" "github.com/hivecommons/hive/pkg/config" @@ -13,45 +16,84 @@ import ( type ioscanClassifierSink interface { SetClassifier(ioscan.Classifier, ioscan.Thresholds) SetClassifierBudgetPerEval(int) + SetClassifierUnavailableFunc(func(error)) } +// ioscanClassifierAlerter is the part of the dashboard boot raises the +// blocked-kick alert on (dashboard.Server satisfies it). nil means no +// dashboard. +type ioscanClassifierAlerter interface { + AddSystemAlert(id, severity, message string) + ClearSystemAlert(id string) +} + +// ioscanClassifierBlockedAlertID identifies the persistent dashboard alert +// raised when the classifier is required at the current ACMM level but +// cannot run, so every kick carrying untrusted text is being withheld. +const ioscanClassifierBlockedAlertID = "ioscan-classifier-blocked" + // installIoscanClassifier builds the semantic injection classifier when the // ACMM level (or an explicit setting) enables it. An empty -// ioscan.classifier.backend uses the OpenAI-compatible HTTP endpoint; a named -// backend runs the classifier through that subscription CLI, tool-less, with -// the low per-pass budget and the daily quota. A backend that cannot run -// tool-less is refused with one log line and handled like any classifier -// that cannot be built: withheld text at fail-closed, no classifier at -// fail-open. -func installIoscanClassifier(sched ioscanClassifierSink, cfg *config.Config, logger *slog.Logger, now func() time.Time) { +// ioscan.classifier.backend uses the OpenAI-compatible HTTP endpoint when +// one is resolved; with no endpoint it falls back to the subscription CLI the +// hive's own agents run on, when that CLI can run tool-less (#14499). A named +// backend runs the classifier through that CLI, tool-less, with the low +// per-pass budget and the daily quota. A backend that cannot run tool-less +// is refused with one log line and handled like any classifier that cannot +// be built: withheld text at fail-closed, no classifier at fail-open. When +// the classifier is fail-closed and cannot run, a persistent dashboard alert +// says that kicks are being blocked and how to fix it; the alert is cleared +// on a successful score or a boot where the classifier runs or is not required. +func installIoscanClassifier(sched ioscanClassifierSink, alerts ioscanClassifierAlerter, cfg *config.Config, logger *slog.Logger, now func() time.Time) { acmmLevel := cfg.ACMMLevelOrZero() + sched.SetClassifierUnavailableFunc(func(err error) { + if err == nil { + clearIoscanClassifierAlert(alerts) + return + } + raiseIoscanClassifierAlert(alerts, acmmLevel, err) + }) if !cfg.Ioscan.ClassifierEnabledAtLevel(acmmLevel) { + clearIoscanClassifierAlert(alerts) return } classifierFailClosed := cfg.Ioscan.ClassifierFailClosedAtLevel(acmmLevel) cl := cfg.Ioscan.Classifier thresholds := ioscan.Thresholds{Warn: cl.WarnThreshold, Block: cl.BlockThreshold} + accepted := strings.Join(ioscan.ToollessBackends(), ", ") - if backend := cl.CLIBackend(); backend != "" { + backend := cl.CLIBackend() + endpoint, apiKey, model := cfg.Governor.ResolveReviewer() + fallback := false + if backend == "" && endpoint == "" { + if backend = ioscanClassifierFallbackBackend(cfg); backend != "" { + fallback = true + } + } + + if backend != "" { // NewCLIClassifier refuses exactly the backends SupportsToolless rejects. classifier, cerr := ioscan.NewCLIClassifier(backend, cl.Model, cl.CLIDailyQuotaOrDefault(), now) if cerr != nil { if classifierFailClosed { sched.SetClassifier(ioscan.UnavailableClassifier{Err: cerr}, ioscan.Thresholds{}) + raiseIoscanClassifierAlert(alerts, acmmLevel, cerr) + } else { + clearIoscanClassifierAlert(alerts) } logger.Error("ioscan classifier backend refused: it cannot run with no tools, no MCP, no GitHub token and an empty working directory", - "backend", backend, "accepted", "claude", "fail_closed", classifierFailClosed, "acmm_level", acmmLevel, - "fix", "set ioscan.classifier.backend to claude, or leave it empty to use the model endpoint (Settings → Security → Input Defense)") + "backend", backend, "accepted", accepted, "fail_closed", classifierFailClosed, "acmm_level", acmmLevel, + "fix", "set ioscan.classifier.backend to one of "+accepted+", or leave it empty to use the model endpoint (Settings → Security → Input Defense)") return } sched.SetClassifier(ioscan.NewCachedClassifier(classifier, ioscan.DefaultClassifierCacheEntries), thresholds) sched.SetClassifierBudgetPerEval(cl.CLIBudgetPerEvalOrDefault()) - logger.Info("ioscan semantic classifier enabled", "backend", backend, "model", cl.Model, + clearIoscanClassifierAlert(alerts) + logger.Info("ioscan semantic classifier enabled", "backend", backend, "backend_from_agents", fallback, "model", cl.Model, "budget_per_eval", cl.CLIBudgetPerEvalOrDefault(), "daily_quota", cl.CLIDailyQuotaOrDefault(), "fail_closed", classifierFailClosed) return } - endpoint, apiKey, model := cfg.Governor.ResolveReviewer() if cl.Model != "" { model = cl.Model } @@ -68,13 +110,69 @@ func installIoscanClassifier(sched ioscanClassifierSink, cfg *config.Config, log // The level requires the classifier and it cannot run: withhold // untrusted text rather than silently scanning without it. sched.SetClassifier(ioscan.UnavailableClassifier{Err: cerr}, ioscan.Thresholds{}) - logger.Error("ioscan classifier required at this ACMM level but not running; untrusted text is withheld", + raiseIoscanClassifierAlert(alerts, acmmLevel, cerr) + logger.Error("ioscan classifier required at this ACMM level but not running; untrusted text is withheld and kicks carrying it are blocked", "reason", cerr.Error(), "acmm_level", acmmLevel, - "fix", "configure governor.trajectory/litellm, set ioscan.classifier.backend: claude, or set ioscan.classifier.enabled: false or ioscan.classifier.fail_mode: open (Settings → Security → Input Defense)") + "fix", "configure governor.trajectory/litellm, set ioscan.classifier.backend to one of "+accepted+", or set ioscan.classifier.enabled: false or ioscan.classifier.fail_mode: open") case cerr != nil: + clearIoscanClassifierAlert(alerts) logger.Warn("ioscan classifier enabled but not running", "reason", cerr.Error()) default: sched.SetClassifier(ioscan.NewCachedClassifier(classifier, ioscan.DefaultClassifierCacheEntries), thresholds) + clearIoscanClassifierAlert(alerts) logger.Info("ioscan semantic classifier enabled", "model", model, "fail_closed", classifierFailClosed) } } + +// ioscanClassifierFallbackBackend picks the subscription CLI the classifier +// runs through when no backend is configured and no model endpoint is +// resolved: the tool-less-capable backend most of the hive's agents run on, +// ties broken by name so the choice is stable across boots. Empty when no +// agent runs on a backend ioscan.SupportsToolless accepts. +func ioscanClassifierFallbackBackend(cfg *config.Config) string { + counts := map[string]int{} + for _, agent := range cfg.Agents { + b := strings.ToLower(strings.TrimSpace(agent.Backend)) + if ioscan.SupportsToolless(b) { + counts[b]++ + } + } + if len(counts) == 0 { + return "" + } + backends := make([]string, 0, len(counts)) + for b := range counts { + backends = append(backends, b) + } + sort.Slice(backends, func(i, j int) bool { + if counts[backends[i]] != counts[backends[j]] { + return counts[backends[i]] > counts[backends[j]] + } + return backends[i] < backends[j] + }) + return backends[0] +} + +// ioscanClassifierBlockedAlertMessage is the dashboard text for a required +// classifier that cannot run. +func ioscanClassifierBlockedAlertMessage(acmmLevel int, cause error) string { + return fmt.Sprintf("ioscan semantic classifier is required at ACMM L%d but failed or is unavailable (%v). "+ + "Kicks carrying untrusted text (issues, comments, PR bodies, reviews) can be withheld until this is fixed. "+ + "Fix: configure a model endpoint (governor.trajectory or governor.litellm), set ioscan.classifier.backend to one of %s "+ + "(Settings → Security → Input Defense), or accept scanning without the classifier with ioscan.classifier.fail_mode: open. "+ + "Changes take effect after a restart.", acmmLevel, cause, strings.Join(ioscan.ToollessBackends(), ", ")) +} + +func raiseIoscanClassifierAlert(alerts ioscanClassifierAlerter, acmmLevel int, cause error) { + if alerts == nil { + return + } + alerts.AddSystemAlert(ioscanClassifierBlockedAlertID, "error", ioscanClassifierBlockedAlertMessage(acmmLevel, cause)) +} + +func clearIoscanClassifierAlert(alerts ioscanClassifierAlerter) { + if alerts == nil { + return + } + alerts.ClearSystemAlert(ioscanClassifierBlockedAlertID) +} diff --git a/src/cmd/hive/ioscan_classifier_boot_test.go b/src/cmd/hive/ioscan_classifier_boot_test.go index c642be224a..2833d935a1 100644 --- a/src/cmd/hive/ioscan_classifier_boot_test.go +++ b/src/cmd/hive/ioscan_classifier_boot_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "fmt" "log/slog" "strings" "testing" @@ -13,9 +14,10 @@ import ( ) type fakeClassifierSink struct { - classifier ioscan.Classifier - sets int - budget int + classifier ioscan.Classifier + sets int + budget int + unavailable func(error) } func (f *fakeClassifierSink) SetClassifier(c ioscan.Classifier, _ ioscan.Thresholds) { @@ -25,6 +27,32 @@ func (f *fakeClassifierSink) SetClassifier(c ioscan.Classifier, _ ioscan.Thresho func (f *fakeClassifierSink) SetClassifierBudgetPerEval(n int) { f.budget = n } +func (f *fakeClassifierSink) SetClassifierUnavailableFunc(fn func(error)) { f.unavailable = fn } + +type fakeClassifierAlerts struct { + alerts map[string]string + adds int + clears int +} + +func (f *fakeClassifierAlerts) AddSystemAlert(id, severity, message string) { + if f.alerts == nil { + f.alerts = map[string]string{} + } + f.alerts[id] = severity + ": " + message + f.adds++ +} + +func (f *fakeClassifierAlerts) ClearSystemAlert(id string) { + delete(f.alerts, id) + f.clears++ +} + +func (f *fakeClassifierAlerts) blocked() (string, bool) { + msg, ok := f.alerts[ioscanClassifierBlockedAlertID] + return msg, ok +} + func ioscanBootConfig(level int, backend, failMode string) *config.Config { return &config.Config{ ACMMLevel: &level, @@ -42,15 +70,15 @@ func ioscanBootLogger() (*slog.Logger, *bytes.Buffer) { var ioscanBootNow = func() time.Time { return time.Date(2026, 10, 24, 12, 0, 0, 0, time.UTC) } -// A backend that cannot run tool-less (copilot, codex, gemini) is refused -// with one log line. At a fail-closed level the unavailable classifier is -// installed so untrusted text is withheld; at fail-open nothing is installed. +// A backend that cannot run tool-less (codex, gemini) is refused with one +// log line. At a fail-closed level the unavailable classifier is installed +// so untrusted text is withheld; at fail-open nothing is installed. func TestInstallIoscanClassifier_RefusesNonToollessBackend(t *testing.T) { - for _, backend := range []string{"copilot", "codex", "gemini"} { + for _, backend := range []string{"codex", "gemini"} { t.Run(backend+" fail closed", func(t *testing.T) { sink := &fakeClassifierSink{} logger, logs := ioscanBootLogger() - installIoscanClassifier(sink, ioscanBootConfig(6, backend, ""), logger, ioscanBootNow) + installIoscanClassifier(sink, nil, ioscanBootConfig(6, backend, ""), logger, ioscanBootNow) if _, ok := sink.classifier.(ioscan.UnavailableClassifier); !ok || sink.sets != 1 { t.Fatalf("installed %T (%d sets), want UnavailableClassifier once", sink.classifier, sink.sets) } @@ -67,7 +95,7 @@ func TestInstallIoscanClassifier_RefusesNonToollessBackend(t *testing.T) { t.Run(backend+" fail open", func(t *testing.T) { sink := &fakeClassifierSink{} logger, logs := ioscanBootLogger() - installIoscanClassifier(sink, ioscanBootConfig(6, backend, "open"), logger, ioscanBootNow) + installIoscanClassifier(sink, nil, ioscanBootConfig(6, backend, "open"), logger, ioscanBootNow) if sink.sets != 0 { t.Fatalf("fail-open refusal installed %T", sink.classifier) } @@ -78,27 +106,30 @@ func TestInstallIoscanClassifier_RefusesNonToollessBackend(t *testing.T) { } } -func TestInstallIoscanClassifier_ClaudeBackendUsesCLIBudget(t *testing.T) { - sink := &fakeClassifierSink{} - logger, logs := ioscanBootLogger() - installIoscanClassifier(sink, ioscanBootConfig(6, "claude", ""), logger, ioscanBootNow) - if sink.sets != 1 { - t.Fatalf("SetClassifier calls = %d, want 1", sink.sets) - } - if _, ok := sink.classifier.(*ioscan.CachedClassifier); !ok { - t.Fatalf("installed %T, want *ioscan.CachedClassifier", sink.classifier) - } - if sink.budget != ioscan.DefaultCLIClassifierBudgetPerEval { - t.Fatalf("budget = %d, want %d", sink.budget, ioscan.DefaultCLIClassifierBudgetPerEval) - } - if !strings.Contains(logs.String(), "backend=claude") || strings.Contains(logs.String(), "refused") { - t.Fatalf("unexpected boot log:\n%s", logs.String()) +func TestInstallIoscanClassifier_CLIBackendUsesCLIBudget(t *testing.T) { + for _, backend := range []string{"claude", "copilot"} { + sink := &fakeClassifierSink{} + logger, logs := ioscanBootLogger() + installIoscanClassifier(sink, nil, ioscanBootConfig(6, backend, ""), logger, ioscanBootNow) + if sink.sets != 1 { + t.Fatalf("%s: SetClassifier calls = %d, want 1", backend, sink.sets) + } + if _, ok := sink.classifier.(*ioscan.CachedClassifier); !ok { + t.Fatalf("%s: installed %T, want *ioscan.CachedClassifier", backend, sink.classifier) + } + if sink.budget != ioscan.DefaultCLIClassifierBudgetPerEval { + t.Fatalf("%s: budget = %d, want %d", backend, sink.budget, ioscan.DefaultCLIClassifierBudgetPerEval) + } + if !strings.Contains(logs.String(), "backend="+backend) || !strings.Contains(logs.String(), "backend_from_agents=false") || strings.Contains(logs.String(), "refused") { + t.Fatalf("%s: unexpected boot log:\n%s", backend, logs.String()) + } } + logger, _ := ioscanBootLogger() cfg := ioscanBootConfig(6, "claude", "") cfg.Ioscan.Classifier.CLIBudgetPerEval = 3 - sink = &fakeClassifierSink{} - installIoscanClassifier(sink, cfg, logger, ioscanBootNow) + sink := &fakeClassifierSink{} + installIoscanClassifier(sink, nil, cfg, logger, ioscanBootNow) if sink.budget != 3 { t.Fatalf("explicit cli_budget_per_eval not used: %d", sink.budget) } @@ -108,7 +139,7 @@ func TestInstallIoscanClassifier_ClaudeBackendUsesCLIBudget(t *testing.T) { func TestInstallIoscanClassifier_EmptyBackendKeepsHTTPPath(t *testing.T) { sink := &fakeClassifierSink{} logger, logs := ioscanBootLogger() - installIoscanClassifier(sink, ioscanBootConfig(6, "", ""), logger, ioscanBootNow) + installIoscanClassifier(sink, nil, ioscanBootConfig(6, "", ""), logger, ioscanBootNow) if sink.budget != 0 { t.Fatalf("HTTP path set a CLI budget: %d", sink.budget) } @@ -119,9 +150,155 @@ func TestInstallIoscanClassifier_EmptyBackendKeepsHTTPPath(t *testing.T) { func TestInstallIoscanClassifier_DisabledInstallsNothing(t *testing.T) { sink := &fakeClassifierSink{} + alerts := &fakeClassifierAlerts{} + alerts.AddSystemAlert(ioscanClassifierBlockedAlertID, "error", "stale from an earlier boot") logger, logs := ioscanBootLogger() - installIoscanClassifier(sink, ioscanBootConfig(1, "copilot", ""), logger, ioscanBootNow) + installIoscanClassifier(sink, alerts, ioscanBootConfig(1, "codex", ""), logger, ioscanBootNow) if sink.sets != 0 || logs.Len() != 0 { t.Fatalf("classifier off at L1 still acted: sets=%d logs=%q", sink.sets, logs.String()) } + if _, ok := alerts.blocked(); ok { + t.Fatal("classifier off at L1 left a stale blocked-kick alert") + } +} + +// ioscanBootConfigWithAgents is a config with no backend set, no endpoint +// (HIVE_LITELLM_ENDPOINT cleared so the host cannot supply one) and one +// agent per listed backend. +func ioscanBootConfigWithAgents(t *testing.T, level int, backends ...string) *config.Config { + t.Helper() + t.Setenv(config.LiteLLMEndpointEnv, "") + cfg := ioscanBootConfig(level, "", "") + cfg.Agents = map[string]config.AgentConfig{} + for i, b := range backends { + cfg.Agents[fmt.Sprintf("agent-%d", i)] = config.AgentConfig{Backend: b} + } + return cfg +} + +// With no backend set and no model endpoint, a hive whose agents run on a +// tool-less-capable CLI gets the classifier through that CLI (#14499): the +// most common such backend, ties by name. +func TestIoscanClassifierFallbackBackend(t *testing.T) { + cases := map[string]struct { + agents []string + want string + }{ + "no agents": {nil, ""}, + "only refused": {[]string{"codex", "gemini", "litellm"}, ""}, + "copilot only": {[]string{"copilot", "copilot"}, "copilot"}, + "majority wins": {[]string{"copilot", "claude", "copilot", "codex"}, "copilot"}, + "tie breaks by name": {[]string{"copilot", "claude"}, "claude"}, + "case and space trim": {[]string{" Copilot "}, "copilot"}, + "refused do not count": {[]string{"codex", "codex", "codex", "copilot"}, "copilot"}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + if got := ioscanClassifierFallbackBackend(ioscanBootConfigWithAgents(t, 6, tc.agents...)); got != tc.want { + t.Fatalf("fallback = %q, want %q", got, tc.want) + } + }) + } +} + +func TestInstallIoscanClassifier_FallsBackToAgentCLIBackend(t *testing.T) { + sink := &fakeClassifierSink{} + alerts := &fakeClassifierAlerts{} + alerts.AddSystemAlert(ioscanClassifierBlockedAlertID, "error", "stale from an earlier boot") + logger, logs := ioscanBootLogger() + installIoscanClassifier(sink, alerts, ioscanBootConfigWithAgents(t, 6, "copilot", "copilot", "codex"), logger, ioscanBootNow) + if _, ok := sink.classifier.(*ioscan.CachedClassifier); !ok || sink.sets != 1 { + t.Fatalf("installed %T (%d sets), want *ioscan.CachedClassifier once", sink.classifier, sink.sets) + } + if sink.budget != ioscan.DefaultCLIClassifierBudgetPerEval { + t.Fatalf("fallback CLI backend did not set the CLI budget: %d", sink.budget) + } + if !strings.Contains(logs.String(), "backend=copilot") || !strings.Contains(logs.String(), "backend_from_agents=true") { + t.Fatalf("boot log does not say the backend came from the agents:\n%s", logs.String()) + } + if _, ok := alerts.blocked(); ok { + t.Fatal("running classifier left the blocked-kick alert up") + } +} + +// An explicit backend or a resolved endpoint wins over the agent fallback. +func TestInstallIoscanClassifier_FallbackYieldsToExplicitConfig(t *testing.T) { + cfg := ioscanBootConfigWithAgents(t, 6, "copilot") + cfg.Ioscan.Classifier.Backend = "claude" + sink := &fakeClassifierSink{} + logger, logs := ioscanBootLogger() + installIoscanClassifier(sink, nil, cfg, logger, ioscanBootNow) + if !strings.Contains(logs.String(), "backend=claude") { + t.Fatalf("explicit backend overridden:\n%s", logs.String()) + } + + cfg = ioscanBootConfigWithAgents(t, 6, "copilot") + cfg.Governor.LiteLLM.Endpoint = "http://litellm.example:4000" + sink = &fakeClassifierSink{} + logger, logs = ioscanBootLogger() + installIoscanClassifier(sink, nil, cfg, logger, ioscanBootNow) + if sink.budget != 0 || strings.Contains(logs.String(), "backend=") { + t.Fatalf("resolved endpoint should keep the HTTP path: budget=%d\n%s", sink.budget, logs.String()) + } +} + +// A required classifier that cannot run raises one persistent dashboard +// alert naming the fixes; a fail-open hive gets no alert. +func TestInstallIoscanClassifier_RequiredButUnavailableRaisesAlert(t *testing.T) { + sink := &fakeClassifierSink{} + alerts := &fakeClassifierAlerts{} + logger, logs := ioscanBootLogger() + installIoscanClassifier(sink, alerts, ioscanBootConfigWithAgents(t, 6, "codex"), logger, ioscanBootNow) + if _, ok := sink.classifier.(ioscan.UnavailableClassifier); !ok { + t.Fatalf("installed %T, want UnavailableClassifier", sink.classifier) + } + msg, ok := alerts.blocked() + if !ok || alerts.adds != 1 { + t.Fatalf("blocked-kick alert not raised exactly once: adds=%d alerts=%v", alerts.adds, alerts.alerts) + } + for _, want := range []string{"error: ", "ACMM L6", "withheld", "ioscan.classifier.backend", "claude, copilot", "fail_mode: open", "governor.litellm"} { + if !strings.Contains(msg, want) { + t.Errorf("alert %q lacks %q", msg, want) + } + } + if !strings.Contains(logs.String(), "kicks carrying it are blocked") { + t.Errorf("boot log does not say kicks are blocked:\n%s", logs.String()) + } + + // A refused explicit backend at fail-closed raises the same alert. + sink, alerts = &fakeClassifierSink{}, &fakeClassifierAlerts{} + installIoscanClassifier(sink, alerts, ioscanBootConfig(6, "codex", ""), logger, ioscanBootNow) + if msg, ok := alerts.blocked(); !ok || !strings.Contains(msg, "codex") { + t.Fatalf("refused backend did not raise the blocked-kick alert: %v", alerts.alerts) + } + + // Fail-open: no alert, and a stale one is cleared. + sink, alerts = &fakeClassifierSink{}, &fakeClassifierAlerts{} + alerts.AddSystemAlert(ioscanClassifierBlockedAlertID, "error", "stale") + installIoscanClassifier(sink, alerts, ioscanBootConfig(6, "", "open"), logger, ioscanBootNow) + if _, ok := alerts.blocked(); ok || sink.sets != 0 { + t.Fatalf("fail-open raised or kept the blocked-kick alert: %v sets=%d", alerts.alerts, sink.sets) + } + + // No dashboard: nothing panics. + installIoscanClassifier(&fakeClassifierSink{}, nil, ioscanBootConfigWithAgents(t, 6, "codex"), logger, ioscanBootNow) +} + +func TestInstallIoscanClassifier_RuntimeUnavailableAlert(t *testing.T) { + sink := &fakeClassifierSink{} + alerts := &fakeClassifierAlerts{} + logger, _ := ioscanBootLogger() + installIoscanClassifier(sink, alerts, ioscanBootConfig(6, "copilot", ""), logger, ioscanBootNow) + if _, ok := alerts.blocked(); ok { + t.Fatal("unexpected alert before runtime failure") + } + sink.unavailable(fmt.Errorf("authentication failed")) + msg, ok := alerts.blocked() + if !ok || !strings.Contains(msg, "authentication failed") { + t.Fatalf("runtime alert = %q, present = %v", msg, ok) + } + sink.unavailable(nil) + if _, ok := alerts.blocked(); ok { + t.Fatal("runtime alert remains after successful score") + } } diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 3d9b2ff154..7272ea16a2 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -21,10 +21,22 @@ import ( "sync/atomic" "time" + cruxconfig "github.com/hivecommons/hive/crux/config" + cruxruntime "github.com/hivecommons/hive/crux/runtime" + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/inference" + "github.com/hivecommons/hive/crux/logscrub" + "github.com/hivecommons/hive/crux/proxy" + "github.com/hivecommons/hive/crux/pushbroker" + "github.com/hivecommons/hive/crux/taskmcp" + "github.com/hivecommons/hive/crux/toolapprove" + "github.com/hivecommons/hive/crux/trajectory" + "github.com/hivecommons/hive/crux/watchdog" + "github.com/hivecommons/hive/crux/watsonx" "github.com/hivecommons/hive/pkg/advisor" "github.com/hivecommons/hive/pkg/advisory" - "github.com/hivecommons/hive/pkg/agent" "github.com/hivecommons/hive/pkg/apphealth" "github.com/hivecommons/hive/pkg/beads" "github.com/hivecommons/hive/pkg/celtrigger" @@ -34,7 +46,6 @@ import ( convergenceaudit "github.com/hivecommons/hive/pkg/convergence/audit" "github.com/hivecommons/hive/pkg/convergence/mutation" "github.com/hivecommons/hive/pkg/convergence/outcome" - "github.com/hivecommons/hive/pkg/crux" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/dashboard/collect" "github.com/hivecommons/hive/pkg/dashchat" @@ -54,13 +65,11 @@ import ( "github.com/hivecommons/hive/pkg/hooks" "github.com/hivecommons/hive/pkg/hub" spoke "github.com/hivecommons/hive/pkg/hub/spoke" - "github.com/hivecommons/hive/pkg/inference" "github.com/hivecommons/hive/pkg/intent" "github.com/hivecommons/hive/pkg/ioscan" "github.com/hivecommons/hive/pkg/jev" "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/loginscan" - "github.com/hivecommons/hive/pkg/logscrub" "github.com/hivecommons/hive/pkg/matrix" "github.com/hivecommons/hive/pkg/mint" "github.com/hivecommons/hive/pkg/msteams" @@ -72,8 +81,6 @@ import ( "github.com/hivecommons/hive/pkg/planning" "github.com/hivecommons/hive/pkg/policies" "github.com/hivecommons/hive/pkg/promptsrc" - "github.com/hivecommons/hive/pkg/proxy" - "github.com/hivecommons/hive/pkg/pushbroker" "github.com/hivecommons/hive/pkg/retro" "github.com/hivecommons/hive/pkg/review" "github.com/hivecommons/hive/pkg/rfccoverage" @@ -85,15 +92,10 @@ import ( "github.com/hivecommons/hive/pkg/spokealerts" "github.com/hivecommons/hive/pkg/stageengine" "github.com/hivecommons/hive/pkg/startrec" - "github.com/hivecommons/hive/pkg/taskmcp" "github.com/hivecommons/hive/pkg/telegram" "github.com/hivecommons/hive/pkg/timeline" "github.com/hivecommons/hive/pkg/tokens" - "github.com/hivecommons/hive/pkg/toolapprove" "github.com/hivecommons/hive/pkg/tracing" - "github.com/hivecommons/hive/pkg/trajectory" - "github.com/hivecommons/hive/pkg/watchdog" - "github.com/hivecommons/hive/pkg/watsonx" "github.com/hivecommons/hive/pkg/worksource" "github.com/hivecommons/hive/pkg/worksource/wavefront" // registers the additive Wavefront source (#8362) "go.opentelemetry.io/otel/attribute" @@ -219,7 +221,7 @@ func heartbeatProviders(cfg *config.Config, agents []spoke.AgentSummary) []crux. return spoke.ProviderStatusForHeartbeat(agents) } -func agentActivityFor(mgr *agent.Manager, cfg *config.Config, govState governor.State, currentMode, name string, proc *crux.Process, onDemandFromPack map[string]bool) hub.AgentActivity { +func agentActivityFor(mgr crux.Runtime, cfg *config.Config, govState governor.State, currentMode, name string, proc *crux.Process, onDemandFromPack map[string]bool) hub.AgentActivity { act := hub.AgentActivity{ Paused: proc.Paused, PausedTrigger: proc.PausedTrigger, @@ -925,7 +927,6 @@ func (b *boot) bootConfigWith(deps bootConfigDeps) bool { // hub is always the same length it stores (no short-vs-full mismatch). gitShort = canonicalGitShort(gitShort) dashboard.SetGitVersion(gitHash, gitShort) - agent.SetBuildCommit(gitShort) dashboard.SetFleetReportBuildInfo(reportedVersion(), gitShort) dashboard.SetGitBranch(gitBranch) // Resolve channel and tracking together from the cached Deployment image. @@ -1451,8 +1452,9 @@ func (b *boot) wireBootClosures() { auditJournal = boundary.Executor.Journal } return &dashboard.Dependencies{ - Config: b.cfg, - AgentMgr: b.agentMgr, + Config: b.cfg, + AgentMgr: b.agentMgr, + CruxRuntimeHealth: cruxRuntimeHealthFunc(b.cruxSupervision), // Provider gateways (#5565 slice 3): concrete openrouter/watsonx/ // linearagent adapters behind the dashboard's consumer-defined // interfaces — this composition root is the only non-test place that @@ -1463,6 +1465,8 @@ func (b *boot) wireBootClosures() { LinearStoredViewerID: linearStoredViewerID, MentionWebhook: b.mentionWebhook, MentionStore: b.mentionStore, + DelegateDecider: newDelegateDeciderRunner(func() *config.Config { return b.cfg }), + ChatGrounder: newChatGroundingRunner(func() *config.Config { return b.cfg }), DashboardChatSubmit: func(user, role, text string) (uint64, error) { if b.dashChat == nil { return 0, fmt.Errorf("dashboard chat is not configured") @@ -2139,7 +2143,7 @@ func (b *boot) bootAdvisoryWith(deps bootAdvisoryDeps) bool { b.logger.Warn("failed to seed default policies", "path", b.policyDirPath, "error", err) } - b.projectCtx = agent.ProjectContext{ + b.projectCtx = crux.ProjectContext{ Org: b.cfg.Project.Org, // Repos stays the full watched set; the functions below narrow it at // read time — RepoPaused to the repos open for work (#6203), AgentRepos @@ -2311,11 +2315,46 @@ func (b *boot) bootAgentsWith(deps bootAgentsDeps) { for _, name := range b.cfg.Governor.ClearInvalidInferenceLaunchCmds(b.cfg.Agents) { b.logger.Warn("cleared invalid inference launch_cmd; inference and gateway backends launch through claude", "agent", name) } - b.agentMgr = agent.NewManager(b.cfg.EnabledAgents(), b.logger, b.projectCtx) + rtCfg := cruxconfig.RuntimeConfigFromEnv() + b.agentMgr, err = deps.newRuntime(rtCfg, + cruxruntime.WithDefinitions(b.cfg.EnabledAgents()), + cruxruntime.WithLogger(b.logger), + cruxruntime.WithProject(b.projectCtx)) + if err != nil { + b.logger.Error("crux runtime unavailable", "mode", rtCfg.Mode, "error", err) + deps.exit(1) + return + } + if sh, ok := b.agentMgr.(*sidecarHost); ok { + sh.setLogger(b.logger) + } + // A sidecar runtime reaches hive's capabilities (token minting, PR + // creation, Linear credentials, scanners, conditions) over the callback + // endpoint instead of the Set* calls below; serve it before the runtime + // starts dialing. Failing to bind is as fatal as no runtime: the runtime + // would otherwise run with every capability reporting unavailable. + if rtCfg.Mode == cruxconfig.ModeSidecar { + if err := deps.startHiveCallback(b.ctx, b.hiveCallbackCapabilities(), b.logger); err != nil { + b.logger.Error("hive callback endpoint unavailable", "error", err) + deps.exit(1) + return + } + // Stage 4: spawn/supervise `crux serve` (CRUX_SUPERVISE=child) and + // keep the crux_runtime_unavailable condition in step with the + // socket. The callback endpoint is up first so a child's first + // capability call lands. + if b.cruxSupervision = newCruxRuntimeSupervision(rtCfg, b.agentMgr, b.logger); b.cruxSupervision != nil { + deps.superviseRuntime(b.ctx, b.cruxSupervision) + } + } // The runtime's spans go through pkg/tracing so they carry the hive // component attribute and count toward reach (#3973); the runtime itself // only knows the OpenTelemetry API (#13536). b.agentMgr.SetSpanStarter(tracing.StartSpan) + // The runtime's self-reported error conditions (stalls, restart storms, + // token-mint failures…) feed the fleet-report tracker through the + // types.ConditionReporter seam; the runtime does not import pkg/fleetreport. + b.agentMgr.SetConditionReporter(fleetreport.DefaultReporter{}) // SIGTERM (pod roll, hive upgrade) destroys every tmux server and with it // the in-flight kick's scrollback; archive it to /data first (#4296). archiveOnShutdown := func() { b.agentMgr.ArchiveAllKickLogs("shutdown") } @@ -2420,7 +2459,7 @@ func (b *boot) bootAgentsWith(deps bootAgentsDeps) { } if b.appAuth != nil { b.agentMgr.SetAppAuth(b.appAuth) - b.agentMgr.SetSandboxPushMinter(pushbroker.GitHubAppMinter{Auth: b.appAuth}) + b.agentMgr.SetSandboxPushMinter(appMinter(b.appAuth, "")) } // Start the per-agent token refresh loop UNCONDITIONALLY. It no-ops until // App auth is wired, and on hosted spokes that wiring happens AFTER boot @@ -2628,6 +2667,7 @@ func (b *boot) bootAgentsWith(deps bootAgentsDeps) { autoMergeOpts.SelfAuthorizationHoldEnabled = selfAuthorizationHoldEnabled autoMergeOpts.MinHeadAge = b.cfg.AutoMerge.EffectiveMinHeadAge() autoMergeOpts.RepoAutoMergeEnabled = func(repo string) bool { return b.cfg.RepoAutoMergeEnabled(repo) } + autoMergeOpts.RepoOnGitHub = func(repo string) bool { return b.cfg.RepoOnGitHub(repo) } // Read through b.cfg on every sweep tick so a config reload of // auto_merge.trusted_bot_authors takes effect without a restart. autoMergeOpts.TrustedBotAuthors = func() map[string]bool { return b.cfg.AutoMerge.TrustedBotAuthorSet() } @@ -2670,6 +2710,10 @@ func (b *boot) bootAgentsWith(deps bootAgentsDeps) { // (tier, soak, min_head_age, review verdict) the sweep applies; // hive-authored PRs keep the relay's existing behaviour. mergePolicy: relayLanePolicyGate(autoMergeOpts), + // Per-repo forge (#14113): a repo on GitLab/Gitea merges through + // its forge.Merger after the same merge-eligible binding; GitHub + // repos keep the path above unchanged. + forgeMerge: forgeMergeSelector(func() *config.Config { return b.cfg }, b.logger), // Push relay (#9771): the same forge-resistance + CanPush ACMM // gate a direct `git push` is governed by, so the relay grants // no privilege — it only changes which process performs the push. @@ -2896,28 +2940,21 @@ func (b *boot) bootDashboardWith(deps bootDashboardDeps) { b.sched.SetRunTriageDeps(b.dashSrv, liveTriageCommenter{client: b.currentGitHubClient}) // Prior-art gate search (hivecommons/hive#13575): knowledge, the // Settings index and the forge, through the client provider. - b.sched.SetPriorArtSearcher(newPriorArtSearcher(b.cfg, func() *knowledge.KnowledgeAPI { return b.knowledgeAPI }, func() []rfccoverage.Setting { return b.rfcCoverageState().index(b.logger) }, b.currentGitHubClient, b.logger)) + priorArtSearcher := newPriorArtSearcher(b.cfg, func() *knowledge.KnowledgeAPI { return b.knowledgeAPI }, func() []rfccoverage.Setting { return b.rfcCoverageState().index(b.logger) }, b.currentGitHubClient, b.logger) + b.priorArtSearcher.Store(priorArtSearcher) + b.sched.SetPriorArtSearcher(priorArtSearcher) b.sched.SetAdvisoryFunc(func(title, detail, agentName string) { - store := b.beadStores[agentName] - if store == nil { - store = b.beadStores["scanner"] - } - if store == nil { - store = b.beadStores["supervisor"] - } - if store == nil { - for _, candidate := range b.beadStores { - store = candidate - break - } - } - if store != nil { + if store := b.beadStoreFor(agentName); store != nil { if b, err := store.Create(title, beads.TypeAdvisory, beads.PriorityHigh, agentName, ""); err == nil { _ = store.SetMetadata(b.ID, "ioscan_classifier", detail) } } }) - installIoscanClassifier(b.sched, b.cfg, b.logger, time.Now) + var ioscanAlerts ioscanClassifierAlerter + if b.dashSrv != nil { + ioscanAlerts = b.dashSrv + } + installIoscanClassifier(b.sched, ioscanAlerts, b.cfg, b.logger, time.Now) b.agentMgr.SetSandboxAuditCallback(func(agentName, action, detail string) { b.dashSrv.AuditLog(agentName, action, detail, agentName) if action == "sandbox_broker_rejected" { @@ -3167,6 +3204,9 @@ func (b *boot) bootCollectorsWith(deps bootCollectorsDeps) { b.metricsCollector.SetProjectScope(b.cfg.Project.Org, primaryRepo, b.cfg.Project.Repos, b.cfg.EffectiveAIAuthor()) b.metricsCollector.SetGitHubClientProvider(b.currentGitHubClient) b.metricsCollector.SetCoverageTargetProvider(func() int { return b.cfg.Governor.EffectiveCoverageTarget() }) + b.metricsCollector.SetReviewEvidenceVerifierProvider(func() (github.ReviewEvidenceVerifier, error) { + return github.NewReviewEvidenceVerifier(b.cfg.Evidence.SigningKeyPath()) + }) b.metricsCollector.SetPRIssueWindowStartProvider(b.dashSrv.CostWindowStart) // Merged / closed counts move on webhook events and the Search API is // asked only hourly; with forgestate.enabled the Overview's open totals @@ -3698,41 +3738,49 @@ func (b *boot) bootSupervision() { for _, e := range wdCfgErrs { b.logger.Warn("watchdog config problem", "error", e) } - if wdSettings.Enabled() { - wdFleet := agent.WatchdogFleet{ - M: b.agentMgr, - // Queue depth for the readiness gate: an agent producing nothing - // while nothing is queued is correct, not unhealthy. Read live - // from the governor so it reflects the current sweep. - Queued: func() (int, bool) { - st := b.gov.GetState() - return st.QueueIssues + st.QueuePRs, true - }, - // The items behind that count, from the same enumeration the - // governor counted, so the not-producing alert can link them. - QueuedRefs: func() []watchdog.QueuedRef { - return watchdogQueuedRefs(b.lastActionable.Load(), b.cfg.Project.Org, watchdog.MaxQueuedRefs) - }, - // Agents the governor deliberately does not kick are idle by - // design, not "alive but not producing" (#11910). - KickSuppressed: func(name string) string { - mode := strings.ToLower(string(b.gov.GetState().Mode)) - ac, ok := b.cfg.Agents[name] - onDemand := (ok && ac.OnDemand) || b.onDemandFromPack[name] - if b.cfg.OffByCadence(name, mode, onDemand) { - return fmt.Sprintf("cadence paused in %s mode", mode) - } - return b.sched.ReviewerLaneIdleReason(name) - }, - // The no-production window is never shorter than the agent's own - // cadence: a daily lane is on schedule for a day (#13817). - CadencePeriod: func(name string) (time.Duration, bool) { - mode := strings.ToLower(string(b.gov.GetState().Mode)) - return b.cfg.CadenceValueForMode(name, mode).Period(time.Now()) - }, - } + // The fleet view reads the manager's agent table, so it exists only for + // the in-process runtime; hostedRuntime guarantees that today, and a + // sidecar replaces it with an API family (#14138 stage 3). + wdFleet, wdErr := cruxruntime.NewWatchdogFleet(b.agentMgr, cruxruntime.WatchdogHooks{ + // Queue depth for the readiness gate: an agent producing nothing + // while nothing is queued is correct, not unhealthy. Read live + // from the governor so it reflects the current sweep. + Queued: func() (int, bool) { + st := b.gov.GetState() + return st.QueueIssues + st.QueuePRs, true + }, + // The items behind that count, from the same enumeration the + // governor counted, so the not-producing alert can link them. + QueuedRefs: func() []watchdog.QueuedRef { + return watchdogQueuedRefs(b.lastActionable.Load(), b.cfg.Project.Org, watchdog.MaxQueuedRefs) + }, + // Agents the governor deliberately does not kick are idle by + // design, not "alive but not producing" (#11910). + KickSuppressed: func(name string) string { + mode := strings.ToLower(string(b.gov.GetState().Mode)) + ac, ok := b.cfg.Agents[name] + onDemand := (ok && ac.OnDemand) || b.onDemandFromPack[name] + if b.cfg.OffByCadence(name, mode, onDemand) { + return fmt.Sprintf("cadence paused in %s mode", mode) + } + return b.sched.ReviewerLaneIdleReason(name) + }, + // The no-production window is never shorter than the agent's own + // cadence: a daily lane is on schedule for a day (#13817). + CadencePeriod: func(name string) (time.Duration, bool) { + mode := strings.ToLower(string(b.gov.GetState().Mode)) + return b.cfg.CadenceValueForMode(name, mode).Period(time.Now()) + }, + }) + switch { + case !wdSettings.Enabled(): + b.logger.Info("agent watchdog disabled by config", "mode", string(wdSettings.Mode)) + case wdErr != nil: + b.logger.Error("agent watchdog disabled: fleet view unavailable", "error", wdErr) + default: b.wd = watchdog.New(wdSettings, wdFleet, b.dashSrv, b.logger, - watchdog.WithAuthProbes(watchdogAuthProbes(b.cfg))) + watchdog.WithAuthProbes(watchdogAuthProbes(b.cfg)), + watchdog.WithConditionReporter(fleetreport.DefaultReporter{})) if b.saved != nil && len(b.saved.Watchdog) > 0 { b.wd.Restore(b.saved.Watchdog) } @@ -3750,8 +3798,6 @@ func (b *boot) bootSupervision() { if wdSettings.Mode == watchdog.ModeObserve { b.logger.Info("agent watchdog is in OBSERVE mode: it will classify agents, publish conditions and record what it WOULD have done, but will not restart or pause anything. Set governor.watchdog.mode: heal to enable healing.") } - } else { - b.logger.Info("agent watchdog disabled by config", "mode", string(wdSettings.Mode)) } // Linear write credential for ISSUES_ONLY+ agents (GitHub-issue parity): @@ -3771,14 +3817,14 @@ func (b *boot) bootSupervision() { // through the session environment (a process keeps the environment it was // forked with), so the proxy, not the environment, is what keeps agent // Linear writes authenticated across rotations. - b.linearCredentialResolver = func() agent.LinearCredential { + b.linearCredentialResolver = func() crux.LinearCredential { if tok := b.dashSrv.LinearAgentAccessToken(); tok != "" { - return agent.LinearCredential{AccessToken: tok} + return crux.LinearCredential{AccessToken: tok} } if b.cfg.Governor.WorkSource.Type == "linear" { - return agent.LinearCredential{APIKey: strings.TrimSpace(b.cfg.Governor.WorkSource.Linear.APIKey)} + return crux.LinearCredential{APIKey: strings.TrimSpace(b.cfg.Governor.WorkSource.Linear.APIKey)} } - return agent.LinearCredential{} + return crux.LinearCredential{} } b.agentMgr.SetLinearCredentialResolver(b.linearCredentialResolver) @@ -4417,6 +4463,10 @@ func (b *boot) bootProxyWith(deps bootProxyDeps) { b.logger.Error("failed to create github proxy", "error", err) } else { b.githubProxy.SetCanaryScanner(b.cfg.Ioscan.IsEnabled() && b.cfg.Ioscan.CanariesEnabled(), b.cfg.Ioscan.FailClosedAtLevel(b.cfg.ACMMLevelOrZero()), ioscan.DefaultCanaries, canaryLeakHandler) + // REST accounting and the shared core-bucket snapshot the agent + // GitHub budget reads live in pkg/github; the proxy takes them through + // a seam (#13536). + b.githubProxy.SetGitHubRESTAccounting(githubRESTAccounting{}) // Per-repo pause (#6203). This is the deterministic refusal the feature // rests on: whatever an agent believes, a write to a paused repo is // answered with a 403 here. The predicate reads live config, so pausing @@ -4483,6 +4533,7 @@ func (b *boot) bootProxyWith(deps bootProxyDeps) { // instance on the same dir would clobber the first's numbers. inferenceSink := tokens.NewInferenceSink(b.cfg.Data.MetricsDir, b.logger) b.githubProxy.SetTokenSink(inferenceSink) + b.setInferenceSink(inferenceSink) // Live Linear credential for agent requests — see // spokeWire.linearCredentialResolver and proxy.injectLinearCredential. b.githubProxy.SetLinearCredentialResolver(b.linearCredentialResolver) @@ -5340,6 +5391,7 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { PendingGitHubAppInstall: b.dashSrv.IsPendingGitHubAppInstall(), AutoUpgrade: b.cfg.Hub.AutoUpgrade, UpgradeDrainUntil: upgradeDrainUntilForHeartbeat(selfUpgradeDrainNow()), + UpgradeDrain: selfUpgradeDrain.Report(selfUpgradeDrainNow()), ClusterHealth: spoke.TimedSection("cluster_health", func() *spoke.HeartbeatClusterHealthReport { if !shouldCollectHeartbeatClusterHealth(b.cfg.Hub.ClusterID) { return nil @@ -5520,7 +5572,21 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { // heartbeat) and withhold new kicks; proceed once the turns end or // hub.upgrade_drain_timeout_s elapses. if b.agentMgr != nil { - inFlight := agentKicksInFlight(b.agentMgr.AllStatuses(), b.startTime) + inFlight, wedged := agentKicksInFlight(b.agentMgr.AllStatuses(), b.startTime, selfUpgradeDrainNow(), func(name string) time.Duration { + if ac, ok := b.cfg.Agents[name]; ok { + return ac.EffectiveContinuousTurnCeiling() + } + return 0 + }) + for _, w := range selfUpgradeWedgedKicks.fresh(wedged) { + b.logger.Warn("self-upgrade drain: ignoring wedged agent kick that never reported a turn end", + "agent", w.Agent, + "kick_at", w.KickAt.UTC().Format(time.RFC3339), + "age", w.Age.Round(time.Second), + "ceiling", w.Ceiling, + ) + } + ignoredWedged := wedgedKickAgents(wedged) drainTimeout := b.cfg.Hub.EffectiveUpgradeDrainTimeout() dec := selfUpgradeDrain.decide(targetSHA, inFlight, drainTimeout, selfUpgradeDrainNow()) switch { @@ -5532,6 +5598,7 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { "waited", dec.Waited.Round(time.Second), "max_wait_remaining", dec.Remaining.Round(time.Second), "drain_timeout", drainTimeout, + "ignored_wedged", ignoredWedged, ) return case dec.TimedOut: @@ -5541,12 +5608,14 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { "cut_kicks", dec.InFlight, "waited", dec.Waited.Round(time.Second), "drain_timeout", drainTimeout, + "ignored_wedged", ignoredWedged, ) - case dec.Waited > 0: + case dec.Waited > 0 || len(ignoredWedged) > 0: b.logger.Info("self-upgrade drain complete: no agent kicks in flight", "target", targetSHA, "current", gitShort, "waited", dec.Waited.Round(time.Second), + "ignored_wedged", ignoredWedged, ) } } @@ -5642,9 +5711,30 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { // UpgradeSelfToSHA patches the image instead when we are pinned. needsRestart, err := spoke.UpgradeSelfToSHA(b.logger, targetSHA) if err != nil { + // The patch did not go through. When the Deployment tracks a + // floating tag that the kubelet re-pulls on container start, + // a plain process exit still lands the upgrade — so skip the + // RolloutRestartSelf PATCH (same API, same admission webhook, + // same failure, another timeout wait) and exit NOW, reporting + // pending-restart rather than failed. On vllm-d a Kyverno + // webhook timed out every Deployment patch; the upgrade + // landed on each restart yet the hub showed "failed 3 + // attempts" with a floating-tag hint (#14385). + fb := spoke.RestartFallbackForPatchFailure(b.logger, err) + recordUpgradeError(upgradeMarkerPath, fmt.Errorf("%s: %w", fb.Hint, err), b.logger) + if fb.RestartLands { + b.logger.Warn("self-upgrade: Deployment patch failed, exiting so the kubelet re-pulls the floating tag", + "target", targetSHA, + "current", gitShort, + "kind", string(fb.Kind), + "hint", fb.Hint, + "error", err, + ) + spoke.ReportUpgradePendingRestart(b.hubURL, b.cfg.HiveID, targetSHA, gitShort, fb.Hint, b.logger) + os.Exit(selfUpgradePendingRestartExitCode) + } b.logger.Warn("pinned-image upgrade failed, falling back to rolling restart", - "target", targetSHA, "error", err) - recordUpgradeError(upgradeMarkerPath, err, b.logger) + "target", targetSHA, "kind", string(fb.Kind), "hint", fb.Hint, "error", err) needsRestart = true } if needsRestart { @@ -5655,14 +5745,15 @@ func (b *boot) bootHeartbeatWith(deps bootHeartbeatDeps) { // the spoke lacks patch on its own Deployment) must be both // persisted for the next attempt and reported to the hub so // the UI stops showing a permanent "Upgrading". + hint := spoke.PatchFailureHint(spoke.ClassifyPatchFailure(err)) b.logger.Error("self-upgrade FAILED: could not patch own Deployment, restarting onto the same image", "target", targetSHA, "current", gitShort, "error", err, - "hint", "grant get/patch on deployments/hive in this namespace (hive-self-upgrade Role/RoleBinding)", + "hint", hint, ) recordUpgradeError(upgradeMarkerPath, err, b.logger) - spoke.ReportUpgradeFailure(b.hubURL, b.cfg.HiveID, targetSHA, gitShort, err.Error(), b.logger) + spoke.ReportUpgradeFailure(b.hubURL, b.cfg.HiveID, targetSHA, gitShort, hint+": "+err.Error(), b.logger) // os.Exit skips the cleanup stack: record this end // now so the next start reads self-upgrade-failed // (#11945). @@ -6897,12 +6988,12 @@ func activeIssueRefsFromLastActionable(actionable *github.ActionableResult) (map return refs, coveredRepos } -// agentKicker adapts *agent.Manager to planning.Kicker for the Phase 3 +// agentKicker adapts crux.Runtime to planning.Kicker for the Phase 3 // stall-replan lane. Kick delegates to SendKick, which takes the manager lock // ITSELF and is only ever called here from the governor tick (never from the // agent-launch path), so it cannot re-enter a held manager lock. This is the // same out-of-band kick path the eval loop already uses for governor kicks. -type agentKicker struct{ mgr *agent.Manager } +type agentKicker struct{ mgr crux.Runtime } func (k agentKicker) Kick(agent, message string) error { if k.mgr == nil { @@ -6926,7 +7017,7 @@ func (k agentKicker) Kick(agent, message string) error { func planFromLabeledIssues( actionable *github.ActionableResult, beadStores map[string]*beads.Store, - agentMgr *agent.Manager, + agentMgr crux.Runtime, gov *governor.Governor, dashSrv *dashboard.Server, logger *slog.Logger, @@ -7410,7 +7501,7 @@ func runEvalCycle( ghClient *github.Client, gov *governor.Governor, sched *scheduler.Scheduler, - agentMgr *agent.Manager, + agentMgr crux.Runtime, dashSrv *dashboard.Server, notifier *notify.Notifier, beadStores map[string]*beads.Store, @@ -7895,6 +7986,8 @@ func runEvalCycle( applyHumanDecisionLabels(ctx, cfg, ghClient, actionable, reviewPlan, logger) if ghClient != nil { applyReviewPriorityLabels(ctx, cfg, ghClient, reviewQueue, logger) + applyReviewVerdictLabels(ctx, cfg, ghClient, actionable, reviewPlan, logger) + sweepClosedPRVerdictLabels(ctx, cfg, ghClient, time.Now(), logger) } messages, messageSkips := sched.BuildKickMessagesWithSkips(kickActionable, agentsDue) agentsSkipped = append(agentsSkipped, messageSkips...) @@ -8373,7 +8466,7 @@ func runEvalCycle( dispatchAgentKicks(messages, releaseProviderBudgetProbe, kickDispatchDeps{ backoffRemaining: agentMgr.ProviderErrorBackoffRemaining, sendKick: func(agentName, message string) error { - return agentMgr.SendKickWithSource(agentName, message, agent.GovernorCadenceKickSource) + return agentMgr.SendKickWithSource(agentName, message, crux.GovernorCadenceKickSource) }, startStoppedAgent: func(agentName string) error { if !agentMgr.GovernorMayStart(agentName) { @@ -8592,11 +8685,20 @@ func watchdogAuthProbes(cfg *config.Config) map[string]watchdog.AuthProbe { } out := make(map[string]watchdog.AuthProbe, len(probers)) for _, p := range probers { - out[p.Provider()] = watchdog.RotationAuthProbe{Prober: p} + out[p.Provider()] = watchdog.RotationAuthProbe{Prober: usageProber{p}} } return out } +// usageProber adapts a rotation prober to the watchdog's UsageProber slice +// (provider key + sanitised probe error); the watchdog does not import +// pkg/rotation. +type usageProber struct{ rotation.Prober } + +func (u usageProber) ProbeError(ctx context.Context) string { + return u.Prober.Probe(ctx).ProbeError() +} + // turnLossToSnapshot converts the manager's in-memory turn-loss accumulation // into its persisted form, or nil when nothing has been recorded. // @@ -8670,11 +8772,11 @@ func restartEventsFromSnapshot(events []snapshot.AgentRestartEvent) []crux.Resta // this var cannot race a goroutine still reading it. var persistPathsForRuntime = defaultPersistPaths -func persistState(agentMgr *agent.Manager, gov *governor.Governor, cfg *config.Config, path string, logger *slog.Logger, dashSrv *dashboard.Server, wd *watchdog.Reconciler) { +func persistState(agentMgr crux.Runtime, gov *governor.Governor, cfg *config.Config, path string, logger *slog.Logger, dashSrv *dashboard.Server, wd *watchdog.Reconciler) { persistStateWithPaths(agentMgr, gov, cfg, path, logger, dashSrv, wd, persistPathsForRuntime()) } -func persistStateWithPaths(agentMgr *agent.Manager, gov *governor.Governor, cfg *config.Config, path string, logger *slog.Logger, dashSrv *dashboard.Server, wd *watchdog.Reconciler, paths persistPaths) { +func persistStateWithPaths(agentMgr crux.Runtime, gov *governor.Governor, cfg *config.Config, path string, logger *slog.Logger, dashSrv *dashboard.Server, wd *watchdog.Reconciler, paths persistPaths) { statuses := agentMgr.AllStatuses() agents := make(map[string]snapshot.AgentState, len(statuses)) for name, proc := range statuses { @@ -9066,7 +9168,7 @@ func recordRedStaleness(cfg *config.Config, actionable *github.ActionableResult) } // fixKicker is the out-of-band kick path the re-engagement lanes deliver -// through. *agent.Manager satisfies it via agentKicker, which is the same +// through. crux.Runtime satisfies it via agentKicker, which is the same // adapter the governor tick's replan lane uses. type fixKicker interface { Kick(agent, message string) error @@ -9078,7 +9180,7 @@ type fixKicker interface { // every mode (the v5 L6 pack pauses `strategist` everywhere) can never answer a // FIX-BEFORE-NEW kick, so charging its red PRs' re-engagement budget only // escalates them to a human having asked nobody (hivecommons/hive#9472). -func agentAvailability(cfg *config.Config, mgr *agent.Manager) func(string) bool { +func agentAvailability(cfg *config.Config, mgr crux.Runtime) func(string) bool { return func(name string) bool { if name == "" { return false @@ -9610,7 +9712,7 @@ const rotationTrigger = "provider-rotation" // nothing has headroom, pause it loudly (strand). Stranded agents are // auto-resumed when their provider recovers headroom. Never runs mid-task: // only idle agents are candidates. -func runRotationCheck(ctx context.Context, cfg *config.Config, rotMgr *rotation.Manager, gov *governor.Governor, agentMgr *agent.Manager, logger *slog.Logger) { +func runRotationCheck(ctx context.Context, cfg *config.Config, rotMgr *rotation.Manager, gov *governor.Governor, agentMgr crux.Runtime, logger *slog.Logger) { if rotMgr == nil || !cfg.Governor.Rotation.Enabled { return } @@ -9724,6 +9826,7 @@ func runAutoMergeSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *c opts.QueueOnApproveAndUnhold = func(repo string) bool { return cfg.AutoMerge.QueueOnApproveAndUnholdAllowed(repo, cfg.ACMMLevel) } + opts.RepoOnGitHub = func(repo string) bool { return cfg.RepoOnGitHub(repo) } opts.HumanMergePaths = func(repo string) []string { return humanMergePathsFor(cfg, repo) } opts.RequireCurrentBase = func(repo string) bool { return cfg.AutoMerge.RequireCurrentBaseFor(repo) } opts.MergeForwardTrustedHuman = func(repo string) bool { return cfg.AutoMerge.MergeForwardTrustedHumanFor(repo) } @@ -10014,6 +10117,35 @@ func runSharedCIRerunSweepIfDue(ctx context.Context, ghClient *github.Client, cf } } +// blockedIssuesSweepInterval is how often the periodic sweep reconciles open +// issues carrying the `blocked` label. +const blockedIssuesSweepInterval = 5 * time.Minute + +func runBlockedIssuesSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *config.Config, dashSrv *dashboard.Server, lastRun *time.Time, logger *slog.Logger) { + if ghClient == nil { + return + } + now := time.Now() + if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < blockedIssuesSweepInterval { + return + } + if lastRun != nil { + *lastRun = now + } + var repos []string + if cfg != nil { + repos = cfg.Project.Repos + } + res, err := ghClient.SweepBlockedIssues(ctx, github.BlockedSweepOptions{Repos: repos}) + if err != nil { + logger.Warn("blocked issues reconciliation sweep failed", "error", err) + return + } + if res.Unblocked+res.Retargeted+res.Stripped+res.Errors > 0 { + logger.Info("blocked issues reconciliation sweep complete", "items", res.Items, "unblocked", res.Unblocked, "retargeted", res.Retargeted, "help_wanted_stripped", res.Stripped, "errors", res.Errors) + } +} + // runDuplicateSweepIfDue clusters open PRs by changed-file set and suggests // which one to keep, at most once per duplicateSweepInterval // (hivecommons/hive#7469 capability B). @@ -10337,10 +10469,49 @@ func runClosedIssueStageSweepIfDue(ctx context.Context, ghClient *github.Client, }) if err != nil { logger.Warn("closed-issue stage sweep failed", "error", err) + } else if result != nil && (len(result.Cleaned) > 0 || result.Skipped > 0) { + logger.Info("closed-issue stage sweep complete", "seen", result.Seen, "cleaned", len(result.Cleaned), "skipped", result.Skipped) + } + runReviewBacklogSplitSweep(ctx, ghClient, dashSrv, since, logger) +} + +// runReviewBacklogSplitSweep re-files still-unchecked review findings folded +// into an issue that has since closed (hivecommons/hive#13914). It shares the +// closed-issue sweep's cadence and window; every rule lives in +// SweepReviewBacklogSplits. It then runs the one-off +// BackfillReviewBacklogSplits for hosts closed before that window +// (hivecommons/hive#13947), a no-op once the ledger records it complete. +func runReviewBacklogSplitSweep(ctx context.Context, ghClient *github.Client, dashSrv *dashboard.Server, since time.Time, logger *slog.Logger) { + audit := func(event github.ReviewBacklogSplitEvent) { + if dashSrv == nil { + return + } + detail := fmt.Sprintf("repo=%s, host=%d, issue=%d, reused=%t, key=%s", + event.Repo, event.Host, event.Number, event.Reused, event.Key) + dashSrv.AuditLogRecord("system", "review-backlog-split", detail, "", event.Repo, event.Host) + } + result, err := ghClient.SweepReviewBacklogSplits(ctx, github.ReviewBacklogSplitSweepOptions{ + Since: since, + MaxActions: github.DefaultReviewBacklogSplitMaxActions, + Audit: audit, + }) + if err != nil { + logger.Warn("review backlog split sweep failed", "error", err) return } - if result != nil && (len(result.Cleaned) > 0 || result.Skipped > 0) { - logger.Info("closed-issue stage sweep complete", "seen", result.Seen, "cleaned", len(result.Cleaned), "skipped", result.Skipped) + if result != nil && (len(result.Split) > 0 || result.Skipped > 0 || result.Deferred > 0) { + logger.Info("review backlog split sweep complete", "hosts", result.Hosts, "split", len(result.Split), "skipped", result.Skipped, "deferred", result.Deferred) + } + backfill, err := ghClient.BackfillReviewBacklogSplits(ctx, github.ReviewBacklogSplitSweepOptions{ + MaxActions: github.DefaultReviewBacklogSplitMaxActions, + Audit: audit, + }) + if err != nil { + logger.Warn("review backlog split backfill failed", "error", err) + return + } + if backfill != nil && (len(backfill.Split) > 0 || backfill.Skipped > 0 || backfill.Deferred > 0) { + logger.Info("review backlog split backfill pass complete", "hosts", backfill.Hosts, "split", len(backfill.Split), "skipped", backfill.Skipped, "deferred", backfill.Deferred) } } @@ -10464,8 +10635,16 @@ func applyDuplicatePRGuard( // hivecommons/hive#13093: issues parked on an unmerged PR come back once // that PR settles, and never advertise `help wanted` while parked. blockedCtx, blockedCancel := context.WithTimeout(ctx, issueStageSyncTimeout) - if br := github.ReconcileBlockedIssues(blockedCtx, ghClient, actionable, logger); br.Unblocked+br.Stripped+br.Errors > 0 { - logger.Info("blocked issues reconciled", "items", br.Items, "unblocked", br.Unblocked, "help_wanted_stripped", br.Stripped, "errors", br.Errors) + // hivecommons/hive#13833: review-backlog items wait on the PR they were + // filed from in every destination; lift them when it merges, and when it + // closes unmerged unblock them with a re-triage note (never auto-close). + // Runs first so the items get the sweep's own note rather than the + // generic one from the sweep below. + if rb := github.ReconcileReviewBacklogBlocks(blockedCtx, ghClient, logger); rb.Lifted+rb.Retriaged+rb.Errors > 0 { + logger.Info("review backlog blocks reconciled", "checked", rb.Checked, "lifted", rb.Lifted, "retriaged", rb.Retriaged, "errors", rb.Errors) + } + if br := github.ReconcileBlockedIssues(blockedCtx, ghClient, actionable, logger); br.Unblocked+br.Retargeted+br.Stripped+br.SoakTriaged+br.Errors > 0 { + logger.Info("blocked issues reconciled", "items", br.Items, "unblocked", br.Unblocked, "retargeted", br.Retargeted, "help_wanted_stripped", br.Stripped, "soak_triaged", br.SoakTriaged, "errors", br.Errors) } blockedCancel() // Closing an issue is irreversible: refuse while the snapshot is stale and @@ -10670,6 +10849,9 @@ func installReviewRelaySettings(client *github.Client, cfg *config.Config, logge client.SetReviewEvidence(func(repo string) github.ReviewEvidenceSettings { return reviewEvidenceSettings(cfg, repo) }) + // hivecommons/hive#14581: issue-author credit on every merge path, read + // live so a reload of auto_merge.credit_issue_authors applies. + client.SetCreditIssueAuthors(func(string) bool { return cfg.AutoMerge.CreditIssueAuthorsEnabled() }) // hivecommons/hive#13241: an issue a person reopens after its closing PR // merged is a change failure on that PR's evidence record, from both the // issues.reopened webhook and the enumeration reopen check. @@ -10954,7 +11136,7 @@ func parseReviseCutoff(raw string, logger *slog.Logger) time.Time { return cutoff } -func planReviewDispatch(cfg *config.Config, actionable *github.ActionableResult, agentMgr *agent.Manager, beadStores map[string]*beads.Store, logger *slog.Logger) review.DispatchPlan { +func planReviewDispatch(cfg *config.Config, actionable *github.ActionableResult, agentMgr crux.Runtime, beadStores map[string]*beads.Store, logger *slog.Logger) review.DispatchPlan { if cfg == nil || actionable == nil || !cfg.Review.RequireApproval || !cfg.Review.FanOut { return review.DispatchPlan{} } @@ -11643,24 +11825,24 @@ func initAgentConfigDrivenSystems(cfg *config.Config) { discordIdentities := make(map[string]discord.AgentIdentity) discordAliases := make(map[string]string) - for name, agent := range cfg.Agents { - if len(agent.LaneKeywords) > 0 { + for name, ac := range cfg.Agents { + if len(ac.LaneKeywords) > 0 { lanes = append(lanes, classify.LaneConfig{ Name: name, - Keywords: agent.LaneKeywords, + Keywords: ac.LaneKeywords, }) } - if len(agent.DetectKeywords) > 0 { - detectKeywords[name] = agent.DetectKeywords + if len(ac.DetectKeywords) > 0 { + detectKeywords[name] = ac.DetectKeywords } - if agent.Emoji != "" || agent.Color != "" { + if ac.Emoji != "" || ac.Color != "" { discordIdentities[name] = discord.AgentIdentity{ - Emoji: agent.Emoji, - Color: parseColorInt(agent.Color), + Emoji: ac.Emoji, + Color: parseColorInt(ac.Color), } } sort.Slice(lanes, func(i, j int) bool { return lanes[i].Name < lanes[j].Name }) - for _, alias := range agent.Aliases { + for _, alias := range ac.Aliases { discordAliases[alias] = name } } @@ -11805,7 +11987,7 @@ func dispatchSubcommand(args []string, stdout, stderr io.Writer) (bool, int) { case "agy-turn": // Run from inside a headless agy agent's pane, one per kick; see // pkg/agent/agy_turn.go. - return true, agent.RunAgyTurn(args[1:], stdout, stderr) + return true, cruxruntime.RunAgyTurn(args[1:], stdout, stderr) case jev.Subcommand: // Run from inside an agent's pane by the jev-decide skill; see // pkg/jev/cli.go. Talks only to the hive's loopback decision endpoint. diff --git a/src/cmd/hive/main_helpers.go b/src/cmd/hive/main_helpers.go index 853e7db321..9375f7ad37 100644 --- a/src/cmd/hive/main_helpers.go +++ b/src/cmd/hive/main_helpers.go @@ -333,7 +333,20 @@ func maintainHeldPRMergeability( stopHeldPRUpdate(ctx, writer, store, repo, pr, holdguard.UpdateStopForkNotEditable, nil, logger) continue } + // The update is pinned to the enumerated head: GitHub then refuses it + // if a contributor push landed after enumeration, so the hive never + // stacks its merge on commits the guard has not evaluated. The + // post-hoc IsHygieneMerge check below still covers a push landing + // after the 202 (hivecommons/hive#14537). if err := ghClient.UpdateBranchExpectedHead(ctx, pr.Repo, pr.Number, head, base); err != nil { + if github.IsUpdateBranchStaleHead(err) { + // The head moved since enumeration: not a conflict, and not + // ours to merge onto. The next tick re-reads the head and the + // lift check sees the move as drift. + logger.Info("hold guard: held PR head moved since enumeration; skipped keeping it mergeable", + "repo", repo, "pr", pr.Number, "expected_head", head, "error", err) + continue + } switch github.ClassifyBranchUpdateError(err, pr.FromFork) { case github.BranchUpdateWorkflowPermission: stopHeldPRUpdate(ctx, writer, store, repo, pr, holdguard.UpdateStopWorkflowPermission, err, logger) diff --git a/src/cmd/hive/merge_eligibility.go b/src/cmd/hive/merge_eligibility.go index ae92389d74..14ec8dbf58 100644 --- a/src/cmd/hive/merge_eligibility.go +++ b/src/cmd/hive/merge_eligibility.go @@ -97,15 +97,17 @@ func contributorLanePolicyFunc(cfg *config.Config) func() automerge.ContributorL p := cfg.AutoMerge.ContributorLane tier := cfg.AutoMerge.ContributorTier return automerge.ContributorLanePolicy{ - Enabled: p.IsEnabled(), - HiveID: cfg.HiveID, - Soak: time.Duration(p.EffectiveSoakMinutes()) * time.Minute, - RequireReviewerVerdict: p.EffectiveRequireReviewerVerdict(), - Repos: p.RepoSet(), - ExcludeLabels: p.ExcludeLabelSet(), - RequireTiers: p.RequireTierSet(), - RegistryTiersSufficient: p.EffectiveRegistryTiersSufficient(), - AutoPromoteAfterMergedPR: tier.EffectiveAutoPromoteAfterMerged(), + Enabled: p.IsEnabled(), + HiveID: cfg.HiveID, + Soak: time.Duration(p.EffectiveSoakMinutes()) * time.Minute, + RequireReviewerVerdict: p.EffectiveRequireReviewerVerdict(), + Repos: p.RepoSet(), + ExcludeLabels: p.ExcludeLabelSet(), + RequireTiers: p.RequireTierSet(), + RegistryTiersSufficient: p.EffectiveRegistryTiersSufficient(), + AutoPromoteAfterMergedPR: tier.EffectiveAutoPromoteAfterMerged(), + AutoPromotionScope: p.EffectiveAutoPromotionScope(), + ApproveHeldRunsForRegistryTrusted: p.EffectiveApproveHeldRunsForRegistryTrusted(), } } } @@ -131,6 +133,7 @@ func stageLabelPolicyFunc(cfg *config.Config) func() automerge.StageLabelPolicy AdoptAfterUnregistered: stage.Adoption.EffectiveAfterUnregistered(), AdoptInPlace: stage.Adoption.InPlaceEnabled(), AdoptionStrictAttribution: stage.Adoption.StrictAttributionEnabled(), + AdoptionAutoMerge: stage.Adoption.AutoMergeEnabled(), HiveID: cfg.HiveID, AuthorKindLabels: cfg.Governor.Labels.Author.Names(), RelayKnown: contributorRelayKnown, diff --git a/src/cmd/hive/output_freshness_helpers_test.go b/src/cmd/hive/output_freshness_helpers_test.go index 2f2cd51e6e..fc8972f2f1 100644 --- a/src/cmd/hive/output_freshness_helpers_test.go +++ b/src/cmd/hive/output_freshness_helpers_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/governor" spoke "github.com/hivecommons/hive/pkg/hub/spoke" "github.com/hivecommons/hive/pkg/snapshot" diff --git a/src/cmd/hive/outreach_hold_test.go b/src/cmd/hive/outreach_hold_test.go index c604970b3d..721011f362 100644 --- a/src/cmd/hive/outreach_hold_test.go +++ b/src/cmd/hive/outreach_hold_test.go @@ -16,8 +16,8 @@ func TestShouldHoldAgentPR(t *testing.T) { ownsRepo func(agent, repo string) bool want bool }{ - {name: "cross-repo outreach is held at full autonomy", agent: "outreach", repo: "someone-else/awesome-list", level: 6, ownsRepo: ownsRepo, want: true}, - {name: "same-repo outreach is not held at full autonomy", agent: "outreach", repo: "hive", level: 6, ownsRepo: ownsRepo, want: false}, + {name: "cross-repo outreach is held at L6", agent: "outreach", repo: "someone-else/awesome-list", level: 6, ownsRepo: ownsRepo, want: true}, + {name: "same-repo outreach is not held at L6", agent: "outreach", repo: "hive", level: 6, ownsRepo: ownsRepo, want: false}, {name: "same-repo outreach accepts an org-qualified target", agent: "outreach", repo: "hivecommons/hive", level: 6, ownsRepo: ownsRepo, want: false}, {name: "same-repo outreach still follows the level gate", agent: "outreach", repo: "console", level: 4, ownsRepo: ownsRepo, want: true}, {name: "outreach identity is normalized", agent: " Outreach ", repo: "other/repo", level: 6, ownsRepo: ownsRepo, want: true}, diff --git a/src/cmd/hive/pause_provenance_boot_test.go b/src/cmd/hive/pause_provenance_boot_test.go index d1eef96a07..aa7c2ce45a 100644 --- a/src/cmd/hive/pause_provenance_boot_test.go +++ b/src/cmd/hive/pause_provenance_boot_test.go @@ -6,9 +6,10 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux" "github.com/hivecommons/hive/pkg/snapshot" ) @@ -47,7 +48,7 @@ func TestRestoreAgentRuntimeState_ReplaysPausedBy(t *testing.T) { if err != nil { t.Fatalf("LoadState: %v", err) } - m := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + m := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) restoreAgentRuntimeState(saved, cfg, m, restoreTestLogger()) proc, err := m.GetStatus("scanner") diff --git a/src/cmd/hive/persist_state_test.go b/src/cmd/hive/persist_state_test.go index 7a92dfaa9e..0a3b375b03 100644 --- a/src/cmd/hive/persist_state_test.go +++ b/src/cmd/hive/persist_state_test.go @@ -12,15 +12,16 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/watchdog" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/dashboard/collect" "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/snapshot" "github.com/hivecommons/hive/pkg/tracing" - "github.com/hivecommons/hive/pkg/watchdog" ) type persistStateFakeFleet struct { @@ -94,7 +95,7 @@ func TestPersistStateRoundTripUsesInjectedPaths(t *testing.T) { t.Fatalf("seed config: %v", err) } - mgr := agent.NewManager(cfg.Agents, logger, agent.ProjectContext{ACMMLevel: level}) + mgr := cruxtest.NewRuntime(cfg.Agents, logger, crux.ProjectContext{ACMMLevel: level}) // Anchor relative to now: SeedRestartTelemetry prunes restart events older // than 24h, so a fixed calendar date turns this test into a time bomb. base := time.Now().UTC().Truncate(time.Second).Add(-time.Hour) @@ -105,10 +106,10 @@ func TestPersistStateRoundTripUsesInjectedPaths(t *testing.T) { t.Fatalf("PauseBy: %v", err) } mgr.SeedPauseState("scanner", pausedAt, "dashboard-api", "owner maintenance", "owner@example") - if err := mgr.Pause("worker", agent.BreakerTrigger, "fleet breaker engaged"); err != nil { + if err := mgr.Pause("worker", crux.BreakerTrigger, "fleet breaker engaged"); err != nil { t.Fatalf("Pause worker: %v", err) } - mgr.SeedPauseState("worker", pausedAt, agent.BreakerTrigger, "fleet breaker engaged", "") + mgr.SeedPauseState("worker", pausedAt, crux.BreakerTrigger, "fleet breaker engaged", "") if err := mgr.PinCLI("scanner", "copilot-cli-v1"); err != nil { t.Fatalf("PinCLI: %v", err) } @@ -375,7 +376,7 @@ func TestPersistStateResolvesPathsThroughRuntimeSeam(t *testing.T) { if err := os.WriteFile(cfgPath, []byte("project:\n org: testorg\nagents:\n scanner:\n role: scanner\n"), 0o644); err != nil { t.Fatalf("seed config: %v", err) } - mgr := agent.NewManager(cfg.Agents, logger, agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(cfg.Agents, logger, crux.ProjectContext{}) gov := governor.New(cfg.Governor, cfg.Agents, logger) // One evaluation gives the governor a non-empty eval and mode history, so // the history files are the observable proof that the seam's path set — diff --git a/src/cmd/hive/plan_from_label_sink_test.go b/src/cmd/hive/plan_from_label_sink_test.go index 57a85353fd..1ce33416ec 100644 --- a/src/cmd/hive/plan_from_label_sink_test.go +++ b/src/cmd/hive/plan_from_label_sink_test.go @@ -7,7 +7,9 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/beads" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" @@ -115,7 +117,7 @@ func TestLabelPlanSinkQueuedPlanUnavailable(t *testing.T) { func TestPlanFromLabeledIssuesFallbackStore(t *testing.T) { fallback := newPlanTestStore(t) stores := map[string]*beads.Store{"scanner": fallback} - mgr := agent.NewManager(map[string]config.AgentConfig{}, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(map[string]config.AgentConfig{}, restoreTestLogger(), crux.ProjectContext{}) actionable := &github.ActionableResult{Issues: github.IssueResult{Items: []github.Issue{ planLabeledIssue(7, "hive-plan"), }}} @@ -267,7 +269,7 @@ func TestLabelPlanSinkAuditMethodsNilDashboard(t *testing.T) { // store": planFromLabeledIssues returns without panicking. func TestPlanFromLabeledIssuesNilStoreValue(t *testing.T) { stores := map[string]*beads.Store{"scanner": nil} - mgr := agent.NewManager(map[string]config.AgentConfig{}, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(map[string]config.AgentConfig{}, restoreTestLogger(), crux.ProjectContext{}) actionable := &github.ActionableResult{Issues: github.IssueResult{Items: []github.Issue{ planLabeledIssue(9, "plan"), }}} diff --git a/src/cmd/hive/plan_from_label_wiring_test.go b/src/cmd/hive/plan_from_label_wiring_test.go index 12cc83105e..562ec19374 100644 --- a/src/cmd/hive/plan_from_label_wiring_test.go +++ b/src/cmd/hive/plan_from_label_wiring_test.go @@ -3,7 +3,9 @@ package main import ( "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/beads" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" @@ -36,7 +38,7 @@ func planLabeledIssue(number int, labels ...string) github.Issue { func TestPlanFromLabeledIssuesGatedBelowPlanningLevel(t *testing.T) { store := newPlanTestStore(t) stores := map[string]*beads.Store{planning.ArchitectAgentName: store} - mgr := agent.NewManager(map[string]config.AgentConfig{}, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(map[string]config.AgentConfig{}, restoreTestLogger(), crux.ProjectContext{}) actionable := &github.ActionableResult{Issues: github.IssueResult{Items: []github.Issue{ planLabeledIssue(1, "hive-plan"), }}} @@ -60,7 +62,7 @@ func TestPlanFromLabeledIssuesRoutesToArchitectStore(t *testing.T) { planning.ArchitectAgentName: architectStore, "scanner": otherStore, } - mgr := agent.NewManager(map[string]config.AgentConfig{}, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(map[string]config.AgentConfig{}, restoreTestLogger(), crux.ProjectContext{}) actionable := &github.ActionableResult{Issues: github.IssueResult{Items: []github.Issue{ planLabeledIssue(1, "hive-plan"), planLabeledIssue(2, "kind/bug"), @@ -82,7 +84,7 @@ func TestPlanFromLabeledIssuesRoutesToArchitectStore(t *testing.T) { // runs this every tick, including before the first GitHub pass and on hives // with no bead stores at all. func TestPlanFromLabeledIssuesNilInputs(t *testing.T) { - mgr := agent.NewManager(map[string]config.AgentConfig{}, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(map[string]config.AgentConfig{}, restoreTestLogger(), crux.ProjectContext{}) planFromLabeledIssues(nil, map[string]*beads.Store{}, mgr, nil, nil, restoreTestLogger(), planLabelTestConfig(), planning.PlanningMinACMMLevel) planFromLabeledIssues(&github.ActionableResult{}, nil, mgr, nil, nil, restoreTestLogger(), planLabelTestConfig(), @@ -98,7 +100,7 @@ func TestPlanFromLabeledIssuesNilInputs(t *testing.T) { func TestPlanFromLabeledIssuesDoesNotCorruptActionableItemsWithSpektacular(t *testing.T) { store := newPlanTestStore(t) stores := map[string]*beads.Store{planning.ArchitectAgentName: store} - mgr := agent.NewManager(map[string]config.AgentConfig{}, restoreTestLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(map[string]config.AgentConfig{}, restoreTestLogger(), crux.ProjectContext{}) dashSrv := dashboard.NewServer(0, restoreTestLogger()) original := []github.Issue{ diff --git a/src/cmd/hive/prfollowupwire.go b/src/cmd/hive/prfollowupwire.go index 136afcefa0..cfa26f0456 100644 --- a/src/cmd/hive/prfollowupwire.go +++ b/src/cmd/hive/prfollowupwire.go @@ -8,7 +8,8 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/escalation" @@ -65,17 +66,17 @@ func qualifyPRFollowUpRepo(org, repo string) string { return repo } -// prFollowUpSessions is the slice of *agent.Manager the PR-opened hook needs. +// prFollowUpSessions is the slice of crux.Runtime the PR-opened hook needs. type prFollowUpSessions interface { SessionID(name string) (string, bool) } // prFollowUpResumeCapturer is the optional half: a session source that can // also name the agent's backend-native resume handle (hivecommons/hive#9606). -// *agent.Manager implements it; a source that does not simply records a +// crux.Runtime implements it; a source that does not simply records a // pointer with no handle, which is the pre-#9606 behaviour. type prFollowUpResumeCapturer interface { - ResumeHandle(name string) (agent.ResumeHandle, bool) + ResumeHandle(name string) (crux.ResumeHandle, bool) } // capturePRFollowUpResume reads the authoring agent's backend resume handle, @@ -118,9 +119,9 @@ func recordPRFollowUpPointer(cfg *config.Config, sessions prFollowUpSessions, d } } -// prFollowUpResumer adapts *agent.Manager to prfollowup.Resumer, translating +// prFollowUpResumer adapts crux.Runtime to prfollowup.Resumer, translating // the manager's transient refusal into prfollowup.ErrBusy. -type prFollowUpResumer struct{ mgr *agent.Manager } +type prFollowUpResumer struct{ mgr crux.Runtime } func (r prFollowUpResumer) SessionID(name string) (string, bool) { return r.mgr.SessionID(name) @@ -128,7 +129,7 @@ func (r prFollowUpResumer) SessionID(name string) (string, bool) { func (r prFollowUpResumer) SendResumeKick(name, message, sessionID string) error { err := r.mgr.SendResumeKick(name, message, sessionID) - if errors.Is(err, agent.ErrResumeBusy) { + if errors.Is(err, crux.ErrResumeBusy) { return fmt.Errorf("%w: %v", prfollowup.ErrBusy, err) } return err @@ -138,7 +139,7 @@ func (r prFollowUpResumer) SendResumeKick(name, message, sessionID string) error // (the enumerated PRs, the escalation verdicts, review-threads.json), plus // the human feedback on PRs with a pointer (a bounded, rate-limited GraphQL // read), then runs the pointer sweep. -func routePRFollowUps(ctx context.Context, cfg *config.Config, client *github.Client, actionable *github.ActionableResult, escalatedPRs map[string]bool, agentMgr *agent.Manager, logger *slog.Logger) []prfollowup.Outcome { +func routePRFollowUps(ctx context.Context, cfg *config.Config, client *github.Client, actionable *github.ActionableResult, escalatedPRs map[string]bool, agentMgr crux.Runtime, logger *slog.Logger) []prfollowup.Outcome { if actionable == nil || !cfg.PRFollowUpResumeEnabled() { return nil } @@ -310,7 +311,7 @@ func prLifecycleState(st github.PRState) string { // sweepPRFollowUps prunes pointers at most once per prFollowUpSweepInterval. // PRs in this tick's open list (and hold-gated PRs, which are open too) are // known open; any other pointer's PR is looked up, within budget. -func sweepPRFollowUps(ctx context.Context, cfg *config.Config, client *github.Client, dir string, prs []github.PullRequest, held []github.HoldItem, agentMgr *agent.Manager, now time.Time, logger *slog.Logger) *prfollowup.SweepResult { +func sweepPRFollowUps(ctx context.Context, cfg *config.Config, client *github.Client, dir string, prs []github.PullRequest, held []github.HoldItem, agentMgr crux.Runtime, now time.Time, logger *slog.Logger) *prfollowup.SweepResult { if !prFollowUpLastSweep.IsZero() && now.Sub(prFollowUpLastSweep) < prFollowUpSweepInterval { return nil } diff --git a/src/cmd/hive/prfollowupwire_test.go b/src/cmd/hive/prfollowupwire_test.go index 996c795e93..e068920267 100644 --- a/src/cmd/hive/prfollowupwire_test.go +++ b/src/cmd/hive/prfollowupwire_test.go @@ -12,7 +12,9 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/escalation" "github.com/hivecommons/hive/pkg/github" @@ -306,13 +308,13 @@ func TestPRLifecycleState(t *testing.T) { } func TestPRFollowUpResumer_AdaptsManager(t *testing.T) { - mgr := agent.NewManager(map[string]config.AgentConfig{}, discardLogger(), agent.ProjectContext{}) + mgr := cruxtest.NewRuntime(map[string]config.AgentConfig{}, discardLogger(), crux.ProjectContext{}) r := prFollowUpResumer{mgr: mgr} if _, ok := r.SessionID("ghost"); ok { t.Fatal("unknown agent must have no session") } err := r.SendResumeKick("ghost", "msg", "s1") - if !errors.Is(err, agent.ErrResumeSessionGone) || errors.Is(err, prfollowup.ErrBusy) { + if !errors.Is(err, crux.ErrResumeSessionGone) || errors.Is(err, prfollowup.ErrBusy) { t.Fatalf("err = %v, want session-gone (a fallback), not busy", err) } } @@ -337,14 +339,14 @@ func TestPRFollowUpMetricsCounters_ReadsStatsFile(t *testing.T) { } // fakePRFollowUpResumeSessions is a session source that also exposes a -// backend-native resume handle, like *agent.Manager does (#9606). +// backend-native resume handle, like crux.Runtime does (#9606). type fakePRFollowUpResumeSessions struct { fakePRFollowUpSessions - handle agent.ResumeHandle + handle crux.ResumeHandle ok bool } -func (f fakePRFollowUpResumeSessions) ResumeHandle(string) (agent.ResumeHandle, bool) { +func (f fakePRFollowUpResumeSessions) ResumeHandle(string) (crux.ResumeHandle, bool) { return f.handle, f.ok } @@ -359,7 +361,7 @@ func TestPRFollowUpWiring_CapturesBackendResumeHandle(t *testing.T) { } sessions := fakePRFollowUpResumeSessions{ fakePRFollowUpSessions: fakePRFollowUpSessions{"scanner": "s1"}, - handle: agent.ResumeHandle{ + handle: crux.ResumeHandle{ Backend: "copilot", SessionID: "sess-9606", Transcript: transcript, Command: "copilot --resume sess-9606", CapturedAt: time.Now(), }, diff --git a/src/cmd/hive/prior_art_gate_wire.go b/src/cmd/hive/prior_art_gate_wire.go index 8d8c08aa57..ca8a79f85a 100644 --- a/src/cmd/hive/prior_art_gate_wire.go +++ b/src/cmd/hive/prior_art_gate_wire.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/hivecommons/hive/pkg/beads" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/github" @@ -65,6 +66,62 @@ func newPriorArtSearcher(cfg *config.Config, knowledgeAPI func() *knowledge.Know }) } +// applyPriorArtFilingGate makes the issue relay enforce the hive-agent +// filing verdict (hivecommons/hive#13879) with the scheduler's Searcher and +// the live runs.triage.prior_art config. Until the Searcher is built the +// relay files as before. +func (b *boot) applyPriorArtFilingGate(client *github.Client) { + client.SetPriorArtFilingGate( + func() config.PriorArtGateConfig { return b.cfg.Runs.Triage.PriorArt }, + func(ctx context.Context, issue priorart.Issue) priorart.Report { + s := b.priorArtSearcher.Load() + if s == nil { + return priorart.Report{} + } + return s.Find(ctx, issue) + }, + b.recordPriorArtRefusal, + ) +} + +// beadStoreFor resolves the bead store an agent-attributed bead is written +// to: the agent's own, then the scanner's, then the supervisor's, then any +// store at all. Nil only when the hive has no bead store. Shared by the +// ioscan advisory writer and the prior-art refusal writer so the two +// resolve by one rule. +func (b *boot) beadStoreFor(agentName string) *beads.Store { + if store := b.beadStores[agentName]; store != nil { + return store + } + for _, name := range []string{"scanner", "supervisor"} { + if store := b.beadStores[name]; store != nil { + return store + } + } + for _, store := range b.beadStores { + return store + } + return nil +} + +// recordPriorArtRefusal writes the knowledge bead for a filing the relay +// refused as already existing, in the requesting agent's bead store (see +// beadStoreFor for the fallback order). +func (b *boot) recordPriorArtRefusal(title, detail, agentName, ref string) { + store := b.beadStoreFor(agentName) + if store == nil { + return + } + bead, err := store.Create(title, beads.TypeDecision, beads.PriorityLow, agentName, ref) + if err != nil { + if b.logger != nil { + b.logger.Warn("prior-art filing refusal bead failed", "agent", agentName, "error", err) + } + return + } + _ = store.SetMetadata(bead.ID, "prior_art", detail) +} + // knowledgePriorArt asks the knowledge layers for the issue's top terms. // Deprecated and superseded facts are left out: pointing an author at a // retired note would be worse than silence. diff --git a/src/cmd/hive/provider_gateways.go b/src/cmd/hive/provider_gateways.go index b09f2619c6..9d8cbc8f79 100644 --- a/src/cmd/hive/provider_gateways.go +++ b/src/cmd/hive/provider_gateways.go @@ -6,10 +6,10 @@ import ( "net/http" "time" + "github.com/hivecommons/hive/crux/openrouter" + "github.com/hivecommons/hive/crux/watsonx" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/linearagent" - "github.com/hivecommons/hive/pkg/openrouter" - "github.com/hivecommons/hive/pkg/watsonx" ) // watsonxGateway adapts pkg/watsonx to dashboard.WatsonxGateway. Stateless: diff --git a/src/cmd/hive/provider_gateways_test.go b/src/cmd/hive/provider_gateways_test.go index b5d7e3973e..810086172c 100644 --- a/src/cmd/hive/provider_gateways_test.go +++ b/src/cmd/hive/provider_gateways_test.go @@ -14,10 +14,10 @@ import ( "testing" "time" + "github.com/hivecommons/hive/crux/openrouter" + "github.com/hivecommons/hive/crux/watsonx" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/linearagent" - "github.com/hivecommons/hive/pkg/openrouter" - "github.com/hivecommons/hive/pkg/watsonx" ) // The adapters in provider_gateways.go are the only bridge between the diff --git a/src/cmd/hive/proxy_rest_accounting.go b/src/cmd/hive/proxy_rest_accounting.go new file mode 100644 index 0000000000..46cfa2d9ad --- /dev/null +++ b/src/cmd/hive/proxy_rest_accounting.go @@ -0,0 +1,29 @@ +package main + +import ( + "net/http" + "time" + + "github.com/hivecommons/hive/crux/proxy" + "github.com/hivecommons/hive/pkg/github" +) + +// githubRESTAccounting adapts pkg/github's package-level REST accounting to +// the proxy's GitHubRESTAccounting seam, so the proxy (a Crux ring package) +// records relayed GitHub traffic and reads the shared core-bucket snapshot +// without importing pkg/github (hivecommons/hive#13536). +type githubRESTAccounting struct{} + +var _ proxy.GitHubRESTAccounting = githubRESTAccounting{} + +func (githubRESTAccounting) RecordRESTRequest(caller, method, path string, status int, h http.Header) { + github.RecordRESTRequest(caller, method, path, status, h) +} + +func (githubRESTAccounting) RecordInjectedRESTRequest(caller, method, path string, status int, h http.Header) { + github.RecordInjectedRESTRequest(caller, method, path, status, h) +} + +func (githubRESTAccounting) CoreRateLimitSnapshot() (remaining int, reset time.Time, ok bool) { + return github.CoreRateLimitSnapshot() +} diff --git a/src/cmd/hive/proxy_rest_accounting_test.go b/src/cmd/hive/proxy_rest_accounting_test.go new file mode 100644 index 0000000000..1cdb1bdb3c --- /dev/null +++ b/src/cmd/hive/proxy_rest_accounting_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "net/http" + "strconv" + "testing" + "time" + + "github.com/hivecommons/hive/crux/proxy" + "github.com/hivecommons/hive/pkg/github" +) + +// The adapter is pkg/github's package-level REST accounting behind the +// proxy's seam: an injected (daemon-credential) record on the hive's own core +// bucket moves the shared snapshot, and the adapter reads back exactly what +// pkg/github reports. The store is process-global, so this asserts the +// injected observation is visible rather than resetting it. +func TestGitHubRESTAccountingAdapterTracksSharedCoreBucket(t *testing.T) { + var acct proxy.GitHubRESTAccounting = githubRESTAccounting{} + reset := time.Now().Add(time.Hour).Unix() + h := http.Header{} + h.Set("X-RateLimit-Resource", "core") + h.Set("X-RateLimit-Remaining", "1234") + h.Set("X-RateLimit-Reset", strconv.FormatInt(reset, 10)) + + acct.RecordRESTRequest("agent:scanner", http.MethodGet, "/repos/o/r/pulls", http.StatusOK, h) + acct.RecordInjectedRESTRequest("agent:scanner", http.MethodGet, "/repos/o/r/pulls", http.StatusOK, h) + remaining, resetAt, ok := acct.CoreRateLimitSnapshot() + if !ok || remaining != 1234 || resetAt.Unix() != reset { + t.Fatalf("snapshot = %d, %v, %v; want 1234 at %d from the injected record", remaining, resetAt, ok, reset) + } + if want, _, _ := github.CoreRateLimitSnapshot(); want != remaining { + t.Fatalf("adapter snapshot %d differs from pkg/github's %d", remaining, want) + } +} diff --git a/src/cmd/hive/release_sentinel.go b/src/cmd/hive/release_sentinel.go index d304da6a0d..2688c44961 100644 --- a/src/cmd/hive/release_sentinel.go +++ b/src/cmd/hive/release_sentinel.go @@ -12,7 +12,6 @@ import ( "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/notify" - "github.com/hivecommons/hive/pkg/pushbroker" "github.com/hivecommons/hive/pkg/releasesentinel" ) @@ -43,7 +42,7 @@ var newReleaseSentinelRetagger = func(cfg *config.Config, ghClient *github.Clien return &releasesentinel.GitRetagger{ RemoteURL: strings.TrimRight(cfg.GitHub.ResolvedBaseURL(), "/") + "/" + owner + "/" + repo + ".git", Dir: releaseSentinelRetagDir, - Minter: pushbroker.GitHubAppMinter{Auth: auth, Tier: releaseSentinelRetagTokenTier}, + Minter: appMinter(auth, releaseSentinelRetagTokenTier), }, nil } diff --git a/src/cmd/hive/restart_events_persist_test.go b/src/cmd/hive/restart_events_persist_test.go index e55150c26e..53db23e91e 100644 --- a/src/cmd/hive/restart_events_persist_test.go +++ b/src/cmd/hive/restart_events_persist_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/snapshot" ) diff --git a/src/cmd/hive/review_queue_test.go b/src/cmd/hive/review_queue_test.go index 39770bb7ca..408140cd4b 100644 --- a/src/cmd/hive/review_queue_test.go +++ b/src/cmd/hive/review_queue_test.go @@ -18,18 +18,41 @@ type fakePriorityLabeler struct { applied []string removed []string applyErr error + // ops is every write in order ("+key label" / "-key label"); asOf the + // observed state each RemoveLabelAsOf was journaled against. + ops []string + asOf []time.Time + // closed answers ClosedPRsWithLabel by label; searches counts calls. + closed map[string][]github.LabelledPR + searches int } func (f *fakePriorityLabeler) ApplyHumanDecisionLabel(_ context.Context, repo string, number int, label string) error { f.applied = append(f.applied, github.ReviewQueueKey(repo, number)+" "+label) + f.ops = append(f.ops, "+"+github.ReviewQueueKey(repo, number)+" "+label) return f.applyErr } func (f *fakePriorityLabeler) RemoveLabel(_ context.Context, repo string, number int, label string) error { f.removed = append(f.removed, github.ReviewQueueKey(repo, number)+" "+label) + f.ops = append(f.ops, "-"+github.ReviewQueueKey(repo, number)+" "+label) return nil } +func (f *fakePriorityLabeler) RemoveLabelAsOf(ctx context.Context, repo string, number int, label string, updatedAt time.Time) error { + f.asOf = append(f.asOf, updatedAt) + return f.RemoveLabel(ctx, repo, number, label) +} + +func (f *fakePriorityLabeler) ClosedPRsWithLabel(_ context.Context, _ string, label string, limit int) ([]github.LabelledPR, error) { + f.searches++ + out := f.closed[label] + if len(out) > limit { + out = out[:limit] + } + return out, nil +} + func priorityQueue() []github.ReviewQueueEntry { return []github.ReviewQueueEntry{ {Repo: "acme/a", Number: 1, Priority: github.ReviewPriorityHigh, Labels: []string{"review-priority/high"}}, diff --git a/src/cmd/hive/review_verdict_labels.go b/src/cmd/hive/review_verdict_labels.go new file mode 100644 index 0000000000..ebcf2ae80b --- /dev/null +++ b/src/cmd/hive/review_verdict_labels.go @@ -0,0 +1,221 @@ +package main + +import ( + "context" + "log/slog" + "os" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/review" +) + +// reviewVerdictLabelMaxChanges bounds verdict-label writes per cycle, so a +// hive turning the feature on for a deep queue converges over a few cycles +// instead of spending one burst of API calls. +const reviewVerdictLabelMaxChanges = 25 + +// reviewVerdictLabelSweepInterval spaces the closed-PR sweep: it costs one +// Search call per verdict label and owner, and the search API allows 30 a +// minute. +const reviewVerdictLabelSweepInterval = 10 * time.Minute + +// reviewVerdictLabeler is the slice of *github.Client the verdict label +// mirror writes through. Removals are journaled against the PR state they +// were planned from (RemoveLabelAsOf): the plain RemoveLabel journals +// "remove X on #N" as applied forever, so once a PR's in-progress label had +// come off it could never come off again, and the next approval left +// in-progress and approved side by side (hivecommons/hive#14361). +type reviewVerdictLabeler interface { + ApplyHumanDecisionLabel(ctx context.Context, repo string, number int, label string) error + RemoveLabelAsOf(ctx context.Context, repo string, number int, label string, updatedAt time.Time) error +} + +// closedPRVerdictLabeler adds the search the closed-PR sweep needs. +type closedPRVerdictLabeler interface { + reviewVerdictLabeler + ClosedPRsWithLabel(ctx context.Context, owner, label string, limit int) ([]github.LabelledPR, error) +} + +// applyReviewVerdictLabels mirrors each open PR's review state onto one +// review.verdict_labels label (hivecommons/hive#13833): reviewing while the +// current head is under review, then its settled verdict. It runs after the +// cycle's dispatch plan, so a head kicked this cycle already reads as +// reviewing. Only PRs whose labels are out of step are touched. Labels are +// never created (the same contract as human_decision_label and +// priority_labels): a missing label is logged and skipped. A PR's stale +// labels come off before its new one goes on, and the cap counts a PR's +// whole change, so two states never show at once. +func applyReviewVerdictLabels(ctx context.Context, cfg *config.Config, labeler reviewVerdictLabeler, actionable *github.ActionableResult, plan review.DispatchPlan, logger *slog.Logger) { + if cfg == nil || labeler == nil || actionable == nil || !cfg.Review.VerdictLabels.Enabled() { + return + } + set := reviewVerdictLabelSet(cfg.Review.VerdictLabels) + artifact, err := review.LoadArtifact("") + if err != nil && !os.IsNotExist(err) { + if logger != nil { + logger.Warn("review verdict labels: verdict artifact unavailable; labels left as they are", "error", err) + } + return + } + prs := verdictLabelPRs(cfg, actionable) + updatedAt := make(map[string]time.Time, len(prs)) + for _, pr := range prs { + updatedAt[github.ReviewQueueKey(pr.Repo, pr.Number)] = pr.UpdatedAt + } + for _, ch := range review.PlanVerdictLabels(set, prs, artifact, plan.State, reviewVerdictLabelMaxChanges) { + observed := updatedAt[github.ReviewQueueKey(ch.Repo, ch.Number)] + for _, stale := range ch.Remove { + if err := labeler.RemoveLabelAsOf(ctx, ch.Repo, ch.Number, stale, observed); err != nil && logger != nil { + logger.Warn("stale review verdict label not removed", "repo", ch.Repo, "pr", ch.Number, "label", stale, "error", err) + } + } + if ch.Add != "" { + if err := labeler.ApplyHumanDecisionLabel(ctx, ch.Repo, ch.Number, ch.Add); err != nil { + if logger != nil { + logger.Warn("review verdict label not applied", "repo", ch.Repo, "pr", ch.Number, "label", ch.Add, "error", err) + } + } else if logger != nil { + logger.Info("review verdict label applied", "repo", ch.Repo, "pr", ch.Number, "label", ch.Add) + } + } + } +} + +var verdictLabelSweep struct { + mu sync.Mutex + last time.Time +} + +// sweepClosedPRVerdictLabels takes the review verdict labels off PRs that +// merged or closed while carrying one. The open-PR mirror never sees those +// PRs again, so a PR merged mid-review kept hive-review/in-progress forever +// (hivecommons/hive#14361). It runs at most once per +// reviewVerdictLabelSweepInterval, searches each configured owner for closed +// PRs carrying each verdict label, and removes at most +// reviewVerdictLabelMaxChanges labels per run. A PR it cleaned no longer +// matches, so a settled hive pays one empty search per label and owner. +func sweepClosedPRVerdictLabels(ctx context.Context, cfg *config.Config, labeler closedPRVerdictLabeler, now time.Time, logger *slog.Logger) { + if cfg == nil || labeler == nil || !cfg.Review.VerdictLabels.Enabled() { + return + } + verdictLabelSweep.mu.Lock() + if !verdictLabelSweep.last.IsZero() && now.Sub(verdictLabelSweep.last) < reviewVerdictLabelSweepInterval { + verdictLabelSweep.mu.Unlock() + return + } + verdictLabelSweep.last = now + verdictLabelSweep.mu.Unlock() + + governed, owners := verdictLabelSweepScope(cfg) + removed := 0 + for _, owner := range owners { + for _, label := range verdictLabelNames(cfg.Review.VerdictLabels) { + if removed >= reviewVerdictLabelMaxChanges { + return + } + found, err := labeler.ClosedPRsWithLabel(ctx, owner, label, reviewVerdictLabelMaxChanges-removed) + if err != nil { + if logger != nil { + logger.Warn("review verdict labels: closed-PR sweep search failed", "owner", owner, "label", label, "error", err) + } + continue + } + for _, pr := range found { + if removed >= reviewVerdictLabelMaxChanges { + return + } + if len(governed) > 0 && !governed[strings.ToLower(pr.Repo)] { + continue + } + removed++ + if err := labeler.RemoveLabelAsOf(ctx, pr.Repo, pr.Number, label, pr.UpdatedAt); err != nil { + if logger != nil { + logger.Warn("review verdict label not removed from closed PR", "repo", pr.Repo, "pr", pr.Number, "label", label, "error", err) + } + } else if logger != nil { + logger.Info("review verdict label removed from closed PR", "repo", pr.Repo, "pr", pr.Number, "label", label) + } + } + } + } +} + +// verdictLabelSweepScope returns the governed repos (lower-case owner/name) +// and the owners to search. With no repos configured every PR under the +// project org is in scope. +func verdictLabelSweepScope(cfg *config.Config) (map[string]bool, []string) { + governed := map[string]bool{} + seen := map[string]bool{} + var owners []string + addOwner := func(o string) { + o = strings.ToLower(strings.TrimSpace(o)) + if o != "" && !seen[o] { + seen[o] = true + owners = append(owners, o) + } + } + for _, r := range cfg.Project.Repos { + full := strings.ToLower(fullRepoName(strings.TrimSpace(r), cfg.Project.Org)) + if owner, _, ok := strings.Cut(full, "/"); ok { + governed[full] = true + addOwner(owner) + } + } + if len(owners) == 0 { + addOwner(cfg.Project.Org) + } + return governed, owners +} + +func reviewVerdictLabelSet(v config.ReviewVerdictLabelsConfig) review.VerdictLabelSet { + return review.VerdictLabelSet{ + Reviewing: v.Reviewing, + Approve: v.Approve, + ChangesRequested: v.ChangesRequested, + RequiresHuman: v.RequiresHuman, + } +} + +// verdictLabelNames is the distinct configured verdict labels, in state order. +func verdictLabelNames(v config.ReviewVerdictLabelsConfig) []string { + seen := map[string]bool{} + var out []string + for _, l := range []string{v.Reviewing, v.Approve, v.ChangesRequested, v.RequiresHuman} { + l = strings.TrimSpace(l) + if l != "" && !seen[strings.ToLower(l)] { + seen[strings.ToLower(l)] = true + out = append(out, l) + } + } + return out +} + +// verdictLabelPRs is the cycle's open PRs (actionable and held) with the +// labels the enumeration already read, in the planner's PullRequest shape. +func verdictLabelPRs(cfg *config.Config, actionable *github.ActionableResult) []review.PullRequest { + seen := map[string]bool{} + var out []review.PullRequest + add := func(items []github.PullRequest) { + for _, pr := range items { + key := github.ReviewQueueKey(fullRepoName(pr.Repo, cfg.Project.Org), pr.Number) + if pr.Number <= 0 || seen[key] { + continue + } + seen[key] = true + out = append(out, review.PullRequest{ + Repo: pr.Repo, + Number: pr.Number, + HeadSHA: pr.HeadSHA, + Labels: append([]string(nil), pr.Labels...), + UpdatedAt: pr.UpdatedAt, + }) + } + } + add(actionable.PRs.Items) + add(actionable.PRs.Held) + return out +} diff --git a/src/cmd/hive/review_verdict_labels_test.go b/src/cmd/hive/review_verdict_labels_test.go new file mode 100644 index 0000000000..b9a820e813 --- /dev/null +++ b/src/cmd/hive/review_verdict_labels_test.go @@ -0,0 +1,170 @@ +package main + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/review" +) + +// The governor mirrors each PR head's review state onto one verdict label: +// in progress while dispatched, then the settled verdict (#13833). +func TestApplyReviewVerdictLabels(t *testing.T) { + dir := t.TempDir() + oldPath, oldLegacy := review.ReviewVerdictsPath, review.LegacyReviewVerdictsPath + review.ReviewVerdictsPath = filepath.Join(dir, review.ReviewVerdictsFile) + review.LegacyReviewVerdictsPath = filepath.Join(dir, "legacy-"+review.ReviewVerdictsFile) + t.Cleanup(func() { review.ReviewVerdictsPath, review.LegacyReviewVerdictsPath = oldPath, oldLegacy }) + if err := review.WriteArtifact("", review.Artifact{Items: []review.Aggregate{ + {Repo: "acme/a", Number: 1, HeadSHA: "h1", Verdict: review.VerdictApprove}, + }}); err != nil { + t.Fatal(err) + } + cfg := &config.Config{} + cfg.Project.Org = "acme" + cfg.Review.VerdictLabels = config.ReviewVerdictLabelsConfig{ + Reviewing: "hive-review/in-progress", + Approve: "hive-review/approved", + } + actionable := &github.ActionableResult{PRs: github.PRResult{Items: []github.PullRequest{ + {Repo: "acme/a", Number: 1, HeadSHA: "h1", Labels: []string{"hive-review/in-progress"}}, + {Repo: "acme/a", Number: 2, HeadSHA: "h2"}, + {Repo: "acme/a", Number: 3, HeadSHA: "h3", Labels: []string{"hive-review/approved"}}, + }}} + plan := review.DispatchPlan{State: review.DispatchState{Pending: []review.PendingReview{ + {Repo: "acme/a", Number: 2, HeadSHA: "h2", Perspective: review.PerspectiveCorrectness, Agent: "r"}, + }}} + labeler := &fakePriorityLabeler{} + applyReviewVerdictLabels(context.Background(), cfg, labeler, actionable, plan, nil) + if got := strings.Join(labeler.applied, "|"); got != "acme/a#1 hive-review/approved|acme/a#2 hive-review/in-progress" { + t.Fatalf("applied = %q", got) + } + if got := strings.Join(labeler.removed, "|"); got != "acme/a#1 hive-review/in-progress|acme/a#3 hive-review/approved" { + t.Fatalf("removed = %q", got) + } + + // Off: nothing is touched. + cfg.Review.VerdictLabels = config.ReviewVerdictLabelsConfig{} + off := &fakePriorityLabeler{} + applyReviewVerdictLabels(context.Background(), cfg, off, actionable, plan, nil) + if len(off.applied)+len(off.removed) != 0 { + t.Fatalf("disabled feature wrote labels: %+v", off) + } +} + +// A settled verdict on a PR still showing in-progress takes in-progress off +// before approved goes on, journaled against the PR state the plan read +// (hivecommons/hive#14361: #2315 showed both after its in-progress label had +// been removed once before, because the plain removal deduped forever). +func TestApplyReviewVerdictLabelsSwapsStateAsOfThePRState(t *testing.T) { + dir := t.TempDir() + oldPath, oldLegacy := review.ReviewVerdictsPath, review.LegacyReviewVerdictsPath + review.ReviewVerdictsPath = filepath.Join(dir, review.ReviewVerdictsFile) + review.LegacyReviewVerdictsPath = filepath.Join(dir, "legacy-"+review.ReviewVerdictsFile) + t.Cleanup(func() { review.ReviewVerdictsPath, review.LegacyReviewVerdictsPath = oldPath, oldLegacy }) + if err := review.WriteArtifact("", review.Artifact{Items: []review.Aggregate{ + {Repo: "acme/a", Number: 2315, HeadSHA: "h1", Verdict: review.VerdictApprove}, + }}); err != nil { + t.Fatal(err) + } + cfg := &config.Config{} + cfg.Project.Org = "acme" + cfg.Review.VerdictLabels = config.ReviewVerdictLabelsConfig{ + Reviewing: "hive-review/in-progress", + Approve: "hive-review/approved", + } + updated := time.Date(2026, 10, 26, 14, 21, 0, 0, time.UTC) + actionable := &github.ActionableResult{PRs: github.PRResult{Items: []github.PullRequest{ + {Repo: "acme/a", Number: 2315, HeadSHA: "h1", Labels: []string{"hive-review/in-progress"}, UpdatedAt: updated}, + }}} + labeler := &fakePriorityLabeler{} + + applyReviewVerdictLabels(context.Background(), cfg, labeler, actionable, review.DispatchPlan{}, nil) + + if got := strings.Join(labeler.ops, "|"); got != "-acme/a#2315 hive-review/in-progress|+acme/a#2315 hive-review/approved" { + t.Fatalf("ops = %q, want the stale state removed, then the new one applied", got) + } + if len(labeler.asOf) != 1 || !labeler.asOf[0].Equal(updated) { + t.Fatalf("removal journaled as of %v, want the PR's updated_at %v", labeler.asOf, updated) + } +} + +func resetVerdictLabelSweep() { + verdictLabelSweep.mu.Lock() + verdictLabelSweep.last = time.Time{} + verdictLabelSweep.mu.Unlock() +} + +// A PR that merged or closed while carrying a verdict label loses it: the +// open-PR mirror never sees it again (hivecommons/hive#14361: #2331 merged +// still showing in-progress). Only governed repos are touched, the sweep is +// throttled, and one run removes at most reviewVerdictLabelMaxChanges. +func TestSweepClosedPRVerdictLabels(t *testing.T) { + resetVerdictLabelSweep() + t.Cleanup(resetVerdictLabelSweep) + cfg := &config.Config{} + cfg.Project.Org = "acme" + cfg.Project.Repos = []string{"a"} + cfg.Review.VerdictLabels = config.ReviewVerdictLabelsConfig{ + Reviewing: "hive-review/in-progress", + Approve: "hive-review/approved", + } + merged := time.Date(2026, 10, 26, 15, 12, 0, 0, time.UTC) + labeler := &fakePriorityLabeler{closed: map[string][]github.LabelledPR{ + "hive-review/in-progress": { + {Repo: "acme/a", Number: 2331, UpdatedAt: merged}, + {Repo: "acme/other", Number: 9, UpdatedAt: merged}, + }, + "hive-review/approved": {{Repo: "ACME/a", Number: 2320, UpdatedAt: merged}}, + }} + now := time.Date(2026, 10, 26, 16, 0, 0, 0, time.UTC) + + sweepClosedPRVerdictLabels(context.Background(), cfg, labeler, now, nil) + + if got := strings.Join(labeler.removed, "|"); got != "acme/a#2331 hive-review/in-progress|acme/a#2320 hive-review/approved" { + t.Fatalf("removed = %q, want only the governed repo's closed PRs", got) + } + if len(labeler.applied) != 0 { + t.Fatalf("sweep applied labels: %v", labeler.applied) + } + for _, at := range labeler.asOf { + if !at.Equal(merged) { + t.Fatalf("removal journaled as of %v, want the closed PR's updated_at", at) + } + } + + // Throttled: a second run inside the interval searches nothing. + searches := labeler.searches + sweepClosedPRVerdictLabels(context.Background(), cfg, labeler, now.Add(time.Minute), nil) + if labeler.searches != searches { + t.Fatalf("sweep ran again inside its interval (%d searches, was %d)", labeler.searches, searches) + } + + // Bounded: one run removes at most reviewVerdictLabelMaxChanges. + var many []github.LabelledPR + for i := 1; i <= reviewVerdictLabelMaxChanges+10; i++ { + many = append(many, github.LabelledPR{Repo: "acme/a", Number: 1000 + i, UpdatedAt: merged}) + } + bounded := &fakePriorityLabeler{closed: map[string][]github.LabelledPR{ + "hive-review/in-progress": many, + "hive-review/approved": many, + }} + sweepClosedPRVerdictLabels(context.Background(), cfg, bounded, now.Add(reviewVerdictLabelSweepInterval), nil) + if len(bounded.removed) != reviewVerdictLabelMaxChanges { + t.Fatalf("one sweep removed %d labels, want at most %d", len(bounded.removed), reviewVerdictLabelMaxChanges) + } + + // Off: nothing is searched. + resetVerdictLabelSweep() + cfg.Review.VerdictLabels = config.ReviewVerdictLabelsConfig{} + off := &fakePriorityLabeler{} + sweepClosedPRVerdictLabels(context.Background(), cfg, off, now, nil) + if off.searches != 0 || len(off.removed) != 0 { + t.Fatalf("disabled feature swept: %+v", off) + } +} diff --git a/src/cmd/hive/rotation_check_test.go b/src/cmd/hive/rotation_check_test.go index a8f4026115..665f6b3235 100644 --- a/src/cmd/hive/rotation_check_test.go +++ b/src/cmd/hive/rotation_check_test.go @@ -5,9 +5,10 @@ import ( "log/slog" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux" "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/rotation" ) @@ -53,7 +54,7 @@ type rotationHarness struct { cfg *config.Config rotMgr *rotation.Manager gov *governor.Governor - agentMgr *agent.Manager + agentMgr cruxtest.Runtime } func newRotationHarness(t *testing.T) *rotationHarness { @@ -63,7 +64,7 @@ func newRotationHarness(t *testing.T) *rotationHarness { cfg: cfg, rotMgr: rotation.NewManager(cfg.Governor.Rotation), gov: governor.New(config.GovernorConfig{}, cfg.Agents, rotationTestLogger()), - agentMgr: agent.NewManager(cfg.Agents, rotationTestLogger(), agent.ProjectContext{}), + agentMgr: cruxtest.NewRuntime(cfg.Agents, rotationTestLogger(), crux.ProjectContext{}), } } @@ -253,7 +254,7 @@ func TestRunRotationCheck_StrandRecoveredAutoResumes(t *testing.T) { }, } h.cfg.Agents = agents - h.agentMgr = agent.NewManager(agents, rotationTestLogger(), agent.ProjectContext{}) + h.agentMgr = cruxtest.NewRuntime(agents, rotationTestLogger(), crux.ProjectContext{}) h.agentMgr.SetSandboxConfig(config.AgentSandboxConfig{Enabled: true}) if err := h.agentMgr.Pause("quality", rotationTrigger, "no provider has headroom (RFC #3958)"); err != nil { @@ -325,7 +326,7 @@ func TestRunRotationCheck_UnknownBackendUntouched(t *testing.T) { "quality": {Backend: "copilot", Enabled: true}, } h.cfg.Agents = agents - h.agentMgr = agent.NewManager(agents, rotationTestLogger(), agent.ProjectContext{}) + h.agentMgr = cruxtest.NewRuntime(agents, rotationTestLogger(), crux.ProjectContext{}) h.exhaust("anthropic") h.exhaust("openai") diff --git a/src/cmd/hive/run_loop_deps.go b/src/cmd/hive/run_loop_deps.go index 6ba2c28608..a4eaa02071 100644 --- a/src/cmd/hive/run_loop_deps.go +++ b/src/cmd/hive/run_loop_deps.go @@ -4,7 +4,8 @@ import ( "context" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/github" ) @@ -55,7 +56,7 @@ type runLoopDeps struct { runRotation func(b *boot) runSweeps func(b *boot) persist func(b *boot) - restartCrashed func(ctx context.Context, mgr *agent.Manager) []string + restartCrashed func(ctx context.Context, mgr crux.Runtime) []string } func defaultRunLoopDeps() runLoopDeps { @@ -94,6 +95,7 @@ func defaultRunLoopDeps() runLoopDeps { runSupersessionSweepIfDue(b.ctx, b.ghClient, b.cfg, b.dashSrv, &b.lastSupersessionSweep, b.logger) runSharedCIRerunSweepIfDue(b.ctx, b.ghClient, b.cfg, b.dashSrv, &b.lastSharedCIRerunSweep, b.logger) runRedMainSweepIfDue(b.ctx, b.ghClient, &b.lastRedMainSweep, b.logger) + runBlockedIssuesSweepIfDue(b.ctx, b.ghClient, b.cfg, b.dashSrv, &b.lastBlockedIssuesSweep, b.logger) } if decision.SkipOptional { return @@ -113,6 +115,6 @@ func defaultRunLoopDeps() runLoopDeps { persist: func(b *boot) { persistState(b.agentMgr, b.gov, b.cfg, spokeStatePath, b.logger, b.dashSrv, b.wd) }, - restartCrashed: func(ctx context.Context, mgr *agent.Manager) []string { return mgr.CheckAndRestartCrashedAgents(ctx) }, + restartCrashed: func(ctx context.Context, mgr crux.Runtime) []string { return mgr.CheckAndRestartCrashedAgents(ctx) }, } } diff --git a/src/cmd/hive/run_loop_deps_test.go b/src/cmd/hive/run_loop_deps_test.go index a001b38502..200f17db6b 100644 --- a/src/cmd/hive/run_loop_deps_test.go +++ b/src/cmd/hive/run_loop_deps_test.go @@ -7,11 +7,13 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/watchdog" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/governor" - "github.com/hivecommons/hive/pkg/watchdog" ) // fakeTicker is a loopTicker the test fires by hand. @@ -67,7 +69,7 @@ func (h *loopHarness) deps() runLoopDeps { h.persisted <- struct{}{} } }, - restartCrashed: func(context.Context, *agent.Manager) []string { + restartCrashed: func(context.Context, crux.Runtime) []string { out := h.crashed h.crashed = nil return out @@ -87,7 +89,7 @@ func newLoopBoot(t *testing.T, evalS, agentPollS int) (*boot, *strings.Builder, t.Cleanup(cancel) b.ctx = ctx b.gov = governor.New(cfg.Governor, cfg.EnabledAgents(), b.logger) - b.agentMgr = agent.NewManager(cfg.Agents, b.logger, agent.ProjectContext{}) + b.agentMgr = cruxtest.NewRuntime(cfg.Agents, b.logger, crux.ProjectContext{}) b.dashSrv = dashboard.NewServer(0, b.logger) return b, &sb, cancel } diff --git a/src/cmd/hive/self_upgrade_drain.go b/src/cmd/hive/self_upgrade_drain.go index f15348b466..929d8038e2 100644 --- a/src/cmd/hive/self_upgrade_drain.go +++ b/src/cmd/hive/self_upgrade_drain.go @@ -5,7 +5,9 @@ import ( "sync" "time" - "github.com/hivecommons/hive/pkg/crux" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/hub/spoke" ) // selfUpgradeDrainStaleAfter bounds how long a drain stays latched without a @@ -32,6 +34,10 @@ type upgradeDrain struct { checkedAt time.Time // deadline is startedAt plus the drain timeout of the open drain window. deadline time.Time + // inFlight and timeout are the last decide inputs, kept so Report can + // tell the hub WHAT the drain is waiting on and for how much longer. + inFlight []string + timeout time.Duration // released is set once decide lets the upgrade proceed. Every path after // that point exits the process or blocks until SIGTERM, so kicks stay // withheld for good: a kick started now would be cut by the restart. @@ -63,6 +69,8 @@ func (d *upgradeDrain) decide(target string, inFlight []string, timeout time.Dur if timeout <= 0 { d.target, d.startedAt, d.checkedAt, d.deadline = "", time.Time{}, time.Time{}, time.Time{} + d.target, d.startedAt, d.checkedAt = "", time.Time{}, time.Time{} + d.inFlight, d.timeout = nil, 0 return upgradeDrainDecision{Proceed: true} } if len(inFlight) == 0 { @@ -80,6 +88,8 @@ func (d *upgradeDrain) decide(target string, inFlight []string, timeout time.Dur d.target = target d.checkedAt = now d.deadline = d.startedAt.Add(timeout) + d.inFlight = append([]string(nil), inFlight...) + d.timeout = timeout waited := now.Sub(d.startedAt) if waited >= timeout { @@ -102,11 +112,42 @@ func (d *upgradeDrain) Active(now time.Time) (target string, active bool) { } if now.Sub(d.checkedAt) >= selfUpgradeDrainStaleAfter { d.target, d.startedAt, d.checkedAt, d.deadline = "", time.Time{}, time.Time{}, time.Time{} + d.target, d.startedAt, d.checkedAt = "", time.Time{}, time.Time{} + d.inFlight, d.timeout = nil, 0 return "", false } return d.target, true } +// Report describes the open drain for the hub heartbeat, or nil when no +// upgrade is being held. The hub's orphan sweep otherwise cannot tell a +// deliberately draining spoke from one whose attempt died: both are alive, +// heartbeating and still on the old SHA. A released drain reports nil too — +// the process is about to exit and the hub sees the restart directly. The +// same staleness rule as Active applies, so a drain the hub stopped asking +// about is not reported either. +func (d *upgradeDrain) Report(now time.Time) *spoke.UpgradeDrainReport { + d.mu.Lock() + defer d.mu.Unlock() + if d.released || d.target == "" || d.startedAt.IsZero() { + return nil + } + if now.Sub(d.checkedAt) >= selfUpgradeDrainStaleAfter { + return nil + } + waited := now.Sub(d.startedAt) + remaining := d.timeout - waited + if remaining < 0 { + remaining = 0 + } + return &spoke.UpgradeDrainReport{ + Target: d.target, + InFlight: append([]string(nil), d.inFlight...), + WaitedS: int(waited / time.Second), + RemainingS: int(remaining / time.Second), + } +} + // releaseLocked closes the drain window and latches the kick hold until the // process exits. func (d *upgradeDrain) releaseLocked(target string, now time.Time) { @@ -114,6 +155,7 @@ func (d *upgradeDrain) releaseLocked(target string, now time.Time) { d.startedAt = time.Time{} d.checkedAt = now d.deadline = time.Time{} + d.inFlight, d.timeout = nil, 0 d.released = true } @@ -132,13 +174,39 @@ func (d *upgradeDrain) Deadline(now time.Time) (deadline time.Time, ok bool) { return d.deadline, true } +// wedgedKick is a kick agentKicksInFlight stopped counting as in flight +// because it has gone unsettled for longer than the agent's turn bound. +type wedgedKick struct { + Agent string + KickAt time.Time + Age time.Duration + Ceiling time.Duration +} + +// inFlightKickCeiling is the age past which an unsettled kick no longer holds +// the drain: the agent's continuous_turn_ceiling (the same bound continuous +// mode uses to presume a silent turn over), or DefaultContinuousTurnCeiling +// when the agent is unknown or the ceiling is disabled. The drain needs a +// bound either way — a turn-end signal that is never observed would otherwise +// hold every upgrade for the full drain window (#14323). +func inFlightKickCeiling(ceiling func(agent string) time.Duration, agent string) time.Duration { + if ceiling != nil { + if c := ceiling(agent); c > 0 { + return c + } + } + return config.DefaultContinuousTurnCeiling +} + // agentKicksInFlight names the running, unpaused agents whose most recent kick was // delivered by THIS process and whose turn has not been classified as ended // (crux.KickOutcome is settled by the pane poller / stall watchdog when the // CLI returns to its prompt). Kicks seeded from persisted state predate -// processStart and cannot still be running here, so they are ignored. -func agentKicksInFlight(statuses map[string]*crux.Process, processStart time.Time) []string { - var names []string +// processStart and cannot still be running here, so they are ignored. A kick +// left unsettled for longer than its agent's turn ceiling (see +// inFlightKickCeiling) is presumed wedged: it is returned in wedged instead of +// inFlight so a lost turn-end signal cannot hold the upgrade drain (#14323). +func agentKicksInFlight(statuses map[string]*crux.Process, processStart, now time.Time, ceiling func(agent string) time.Duration) (inFlight []string, wedged []wedgedKick) { for name, proc := range statuses { if proc == nil || proc.State != crux.StateRunning || proc.Paused || proc.LastKick == nil { continue @@ -149,9 +217,56 @@ func agentKicksInFlight(statuses map[string]*crux.Process, processStart time.Tim if proc.KickOutcome.Settled(proc.LastKick) { continue } - names = append(names, name) + bound := inFlightKickCeiling(ceiling, name) + if age := now.Sub(*proc.LastKick); age >= bound { + wedged = append(wedged, wedgedKick{Agent: name, KickAt: *proc.LastKick, Age: age, Ceiling: bound}) + continue + } + inFlight = append(inFlight, name) + } + sort.Strings(inFlight) + sort.Slice(wedged, func(i, j int) bool { return wedged[i].Agent < wedged[j].Agent }) + return inFlight, wedged +} + +// wedgedKickNotices remembers which wedged kicks were already logged, so the +// per-heartbeat upgrade check warns once per kick instead of every beat. +type wedgedKickNotices struct { + mu sync.Mutex + logged map[string]time.Time +} + +// selfUpgradeWedgedKicks is the process-wide wedged-kick log de-duplicator. +var selfUpgradeWedgedKicks = &wedgedKickNotices{} + +// fresh returns the wedged kicks not reported before (keyed on agent and +// kick time, so a later kick that also wedges is reported again). +func (n *wedgedKickNotices) fresh(wedged []wedgedKick) []wedgedKick { + n.mu.Lock() + defer n.mu.Unlock() + if n.logged == nil { + n.logged = make(map[string]time.Time) + } + var out []wedgedKick + for _, w := range wedged { + if at, ok := n.logged[w.Agent]; ok && at.Equal(w.KickAt) { + continue + } + n.logged[w.Agent] = w.KickAt + out = append(out, w) + } + return out +} + +// wedgedKickAgents names the agents in wedged, for log attributes. +func wedgedKickAgents(wedged []wedgedKick) []string { + if len(wedged) == 0 { + return nil + } + names := make([]string, 0, len(wedged)) + for _, w := range wedged { + names = append(names, w.Agent) } - sort.Strings(names) return names } diff --git a/src/cmd/hive/self_upgrade_drain_test.go b/src/cmd/hive/self_upgrade_drain_test.go index 153eefa9cc..7aad691705 100644 --- a/src/cmd/hive/self_upgrade_drain_test.go +++ b/src/cmd/hive/self_upgrade_drain_test.go @@ -5,8 +5,9 @@ import ( "testing" "time" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux" + "github.com/hivecommons/hive/pkg/hub/spoke" ) func TestUpgradeDrain_ProceedsImmediatelyWithNoKicksInFlight(t *testing.T) { @@ -132,11 +133,78 @@ func TestAgentKicksInFlight(t *testing.T) { "nil-proc": nil, } - got := agentKicksInFlight(statuses, processStart) + got, wedged := agentKicksInFlight(statuses, processStart, kick.Add(5*time.Minute), nil) want := []string{"architect", "quality"} if !reflect.DeepEqual(got, want) { t.Fatalf("agentKicksInFlight = %v, want %v", got, want) } + if len(wedged) != 0 { + t.Fatalf("wedged = %+v, want none for fresh kicks", wedged) + } +} + +// TestAgentKicksInFlight_IgnoresWedgedKickPastCeiling pins #14323: a kick whose +// turn-end signal never arrives stops holding the drain once it is older than +// the agent's continuous turn ceiling (default when unset or disabled). +func TestAgentKicksInFlight_IgnoresWedgedKickPastCeiling(t *testing.T) { + processStart := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) + now := processStart.Add(72 * time.Hour) + wedgedAt := processStart.Add(time.Minute) + recent := now.Add(-10 * time.Minute) + shortAt := now.Add(-20 * time.Minute) + disabledAt := now.Add(-50 * time.Minute) + + statuses := map[string]*crux.Process{ + "brainstorm": {State: crux.StateRunning, LastKick: &wedgedAt}, + "architect": {State: crux.StateRunning, LastKick: &recent}, + "short": {State: crux.StateRunning, LastKick: &shortAt}, + "disabled": {State: crux.StateRunning, LastKick: &disabledAt}, + } + ceilings := map[string]time.Duration{"short": 15 * time.Minute, "disabled": 0} + + inFlight, wedged := agentKicksInFlight(statuses, processStart, now, func(name string) time.Duration { + return ceilings[name] + }) + if want := []string{"architect"}; !reflect.DeepEqual(inFlight, want) { + t.Fatalf("inFlight = %v, want %v", inFlight, want) + } + want := []wedgedKick{ + {Agent: "brainstorm", KickAt: wedgedAt, Age: now.Sub(wedgedAt), Ceiling: config.DefaultContinuousTurnCeiling}, + {Agent: "disabled", KickAt: disabledAt, Age: 50 * time.Minute, Ceiling: config.DefaultContinuousTurnCeiling}, + {Agent: "short", KickAt: shortAt, Age: 20 * time.Minute, Ceiling: 15 * time.Minute}, + } + if !reflect.DeepEqual(wedged, want) { + t.Fatalf("wedged = %+v, want %+v", wedged, want) + } + if got := wedgedKickAgents(wedged); !reflect.DeepEqual(got, []string{"brainstorm", "disabled", "short"}) { + t.Fatalf("wedgedKickAgents = %v", got) + } + + // With only the wedged kick left, the drain proceeds immediately instead + // of waiting out the full drain window. + var d upgradeDrain + onlyWedged := map[string]*crux.Process{"brainstorm": statuses["brainstorm"]} + inFlight, _ = agentKicksInFlight(onlyWedged, processStart, now, nil) + if dec := d.decide("feedbeef", inFlight, 45*time.Minute, now); !dec.Proceed || dec.TimedOut { + t.Fatalf("decision = %+v, want immediate proceed past a wedged kick", dec) + } +} + +func TestWedgedKickNotices_LogsEachKickOnce(t *testing.T) { + var n wedgedKickNotices + k1 := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) + k2 := k1.Add(time.Hour) + w := []wedgedKick{{Agent: "brainstorm", KickAt: k1}} + + if got := n.fresh(w); len(got) != 1 { + t.Fatalf("first fresh = %+v, want the wedged kick", got) + } + if got := n.fresh(w); len(got) != 0 { + t.Fatalf("repeat fresh = %+v, want nothing", got) + } + if got := n.fresh([]wedgedKick{{Agent: "brainstorm", KickAt: k2}}); len(got) != 1 { + t.Fatalf("new kick fresh = %+v, want it reported again", got) + } } func TestHubConfigEffectiveUpgradeDrainTimeout(t *testing.T) { @@ -187,3 +255,41 @@ func TestUpgradeDrain_DeadlineReportedOnlyWhileWaiting(t *testing.T) { }) } } + +// TestUpgradeDrain_ReportReflectsDeferralForHub pins the heartbeat view of the +// drain: nil until a deferral starts, the in-flight kicks and remaining budget +// while deferring, nil again once the drain releases or goes stale. The hub's +// orphan sweep relies on this to tell a deliberate hold from a dead attempt. +func TestUpgradeDrain_ReportReflectsDeferralForHub(t *testing.T) { + d := &upgradeDrain{} + start := time.Date(2026, 10, 23, 17, 16, 0, 0, time.UTC) + timeout := 45 * time.Minute + + if r := d.Report(start); r != nil { + t.Fatalf("no drain yet, Report = %+v, want nil", r) + } + + // Re-confirmed every heartbeat, as the hub re-delivers the instruction. + at := start + for i := 0; i <= 5; i++ { + at = start.Add(time.Duration(i) * 2 * time.Minute) + d.decide("4f8be0b", []string{"brainstorm"}, timeout, at) + } + r := d.Report(at) + if r == nil { + t.Fatal("deferring drain must report") + } + want := &spoke.UpgradeDrainReport{Target: "4f8be0b", InFlight: []string{"brainstorm"}, WaitedS: 600, RemainingS: 2100} + if !reflect.DeepEqual(r, want) { + t.Fatalf("Report = %+v, want %+v", r, want) + } + + if r := d.Report(at.Add(selfUpgradeDrainStaleAfter)); r != nil { + t.Fatalf("stale drain must report nil, got %+v", r) + } + + d.decide("4f8be0b", nil, timeout, at.Add(time.Minute)) + if r := d.Report(at.Add(time.Minute)); r != nil { + t.Fatalf("released drain must report nil, got %+v", r) + } +} diff --git a/src/cmd/hive/selfreport_publish.go b/src/cmd/hive/selfreport_publish.go index c34b100542..ed5803bb63 100644 --- a/src/cmd/hive/selfreport_publish.go +++ b/src/cmd/hive/selfreport_publish.go @@ -53,6 +53,7 @@ const selfReportObservationClass = "observation" // selfReportCreateDeferredReason is the hub's answer to a join_only request // with no existing issue; must match selfReportCreateDeferredReason in pkg/hub. const selfReportCreateDeferredReason = "create_deferred" + // selfReportFailures remembers the last write failure per (kind, fingerprint) // so a persistent outage warns once, then logs at Debug until a write // succeeds or the error changes. @@ -142,6 +143,9 @@ type selfReportRelayResponse struct { URL string `json:"url"` Joined bool `json:"joined"` Reason string `json:"reason,omitempty"` + // ClosedAt is set when the hub found the fingerprint's issue closed and + // wrote nothing; the spoke records it like a direct closed match. + ClosedAt time.Time `json:"closed_at,omitempty"` } type selfReportPublishDeps struct { @@ -259,7 +263,14 @@ func publishFleetReportsWith(ctx context.Context, logger *slog.Logger, dashSrv * if isObservation && open.Recovered { open = fleetreport.OpenIssue{} } - if fleetreport.ClosedRecently(open.ClosedAt, now, deps.limits) { + if fleetreport.ClosedSuppressed(open, now, deps.limits) { + // Still observed on this build while the issue is closed: refresh + // the suppression so it outlives the build (#14256). + dashSrv.UpdateFleetReportState(func(st *fleetreport.State) { + o := st.Open[report.Fingerprint] + o.ClosedObservedAt = now.UTC() + st.Open[report.Fingerprint] = o + }) continue } seenAllowed := fleetreport.AllowSeenComment(open, now, deps.limits) @@ -319,7 +330,7 @@ func publishFleetReportsWith(ctx context.Context, logger *slog.Logger, dashSrv * } else { opts := github.FleetReportOptions{ Repo: deps.repo, HiveSlug: deps.hiveSlug, Version: deps.version, Cluster: deps.cluster, - Now: deps.now, JoinExisting: true, SeenCommentAllowed: seenAllowed, + Now: deps.now, JoinExisting: true, SeenCommentAllowed: seenAllowed, ClosedCooldown: deps.limits.ClosedCooldown, } // The cap and gap apply to NEW issues only; joining never counts. // We cannot know before the search whether this will be a create, so @@ -341,7 +352,7 @@ func publishFleetReportsWith(ctx context.Context, logger *slog.Logger, dashSrv * JoinOnly: !allowNew, }) if err == nil { - write = github.FleetReportResult{Number: relayed.Number, URL: relayed.URL, Created: !relayed.Joined, Joined: relayed.Joined, Commented: relayed.Joined} + write = github.FleetReportResult{Number: relayed.Number, URL: relayed.URL, Created: !relayed.Joined, Joined: relayed.Joined, Commented: relayed.Joined, ClosedAt: relayed.ClosedAt} } } if err != nil && errors.Is(err, errSelfReportCreateDeferred) { @@ -360,10 +371,10 @@ func publishFleetReportsWith(ctx context.Context, logger *slog.Logger, dashSrv * } clearSelfReportFailure(selfReportWriteFailedMsg, report.Fingerprint) if !write.ClosedAt.IsZero() { - logger.Info("fleet report: fingerprint closed upstream recently; not re-filing", "fingerprint", report.Fingerprint, "issue", write.Number, "closed_at", write.ClosedAt) + logger.Info("fleet report: fingerprint closed upstream; not re-filing", "fingerprint", report.Fingerprint, "issue", write.Number, "closed_at", write.ClosedAt) dashSrv.UpdateFleetReportState(func(st *fleetreport.State) { o := st.Open[report.Fingerprint] - o.Number, o.URL, o.ClosedAt = write.Number, write.URL, write.ClosedAt + o.Number, o.URL, o.ClosedAt, o.ClosedObservedAt = write.Number, write.URL, write.ClosedAt, now.UTC() st.Open[report.Fingerprint] = o }) continue @@ -459,7 +470,28 @@ func publishFleetReportsWith(ctx context.Context, logger *slog.Logger, dashSrv * if open.Recovered || (!open.ClearedNextAttemptAt.IsZero() && now.Before(open.ClearedNextAttemptAt)) { continue } + if !fleetreport.AllowClearedComment(open, now, deps.limits) { + // Closed upstream, or a cleared comment went out inside the + // gap: the condition is recovered either way, silently. + dashSrv.MarkFleetReportRecovered(report.Fingerprint) + continue + } if err := deps.gh.PostFleetReportCleared(ctx, deps.repo, open.Number, fleetreport.ClearedComment(report.Fingerprint, hive, deps.version, now)); err != nil { + var closed *github.FleetReportClosedError + if errors.As(err, &closed) { + logger.Info("fleet report: issue closed upstream; cleared comment withheld", "fingerprint", report.Fingerprint, "issue", open.Number, "closed_at", closed.ClosedAt) + clearSelfReportFailure(selfReportClearedFailedMsg, report.Fingerprint) + dashSrv.UpdateFleetReportState(func(st *fleetreport.State) { + o, ok := st.Open[report.Fingerprint] + if !ok { + return + } + o.ClosedAt, o.Recovered = closed.ClosedAt, true + o.ClearedAttempts, o.ClearedNextAttemptAt, o.ClearedNotifyFailed = 0, time.Time{}, "" + st.Open[report.Fingerprint] = o + }) + continue + } recordSelfReportClearedFailure(logger, dashSrv, report.Fingerprint, open, err, now) continue } @@ -470,6 +502,7 @@ func publishFleetReportsWith(ctx context.Context, logger *slog.Logger, dashSrv * return } o.ClearedAttempts, o.ClearedNextAttemptAt, o.ClearedNotifyFailed = 0, time.Time{}, "" + o.ClearedPostedAt = now.UTC() st.Open[report.Fingerprint] = o }) dashSrv.MarkFleetReportRecovered(report.Fingerprint) diff --git a/src/cmd/hive/selfreport_publish_test.go b/src/cmd/hive/selfreport_publish_test.go index a01b31a5ad..f1d3fd55b5 100644 --- a/src/cmd/hive/selfreport_publish_test.go +++ b/src/cmd/hive/selfreport_publish_test.go @@ -190,9 +190,19 @@ func TestPublishFleetReportsHonoursClosedCooldownAndClearedComment(t *testing.T) if calls != 1 || st.Open["fp-closed"].ClosedAt.IsZero() || st.Open["fp-closed"].OpenedByHive || len(st.Filings) != 0 { t.Fatalf("closed cooldown: calls=%d state=%#v", calls, st.Open["fp-closed"]) } - // After the 7-day cooldown the fingerprint is searched again. + if len(w.ensures) == 0 || w.ensures[0].ClosedCooldown != deps.limits.ClosedCooldown { + t.Fatalf("writer not told the closed cooldown: %#v", w.ensures) + } + // The cooldown runs from the last tick that still observed the + // condition (#14256), so it is searched again only once the condition + // has been absent for the whole 7 days. now = now.Add(6 * 24 * time.Hour) publishFleetReportsWith(context.Background(), slog.Default(), srv, res, deps) + if calls != 1 { + t.Fatalf("re-searched while still inside the cooldown: calls=%d", calls) + } + now = now.Add(8 * 24 * time.Hour) + publishFleetReportsWith(context.Background(), slog.Default(), srv, res, deps) if calls != 2 { t.Fatalf("cooldown expiry did not re-search: calls=%d", calls) } @@ -643,3 +653,176 @@ func TestSelfReportClearedPermanent(t *testing.T) { } } } + +// The weekend loop William hit (#14256, #14255, #14270): a spoke files a +// selfreport, a maintainer closes it, the condition is still there on the +// spoke's build at the next tick (and the next daily build), and a new +// issue appears. Once closed, the reporter must stay quiet for that +// fingerprint for as long as the condition keeps being observed, through +// version bumps, and must not pile "Cleared on" comments onto the closed +// issue when the condition flaps. +func TestPublishFleetReportsNeverRefilesAClosedFingerprintWhileConditionPersists(t *testing.T) { + srv := selfReportTestServer(t) + now := time.Date(2026, 10, 25, 17, 10, 0, 0, time.UTC) + closedAt := time.Time{} + searches := 0 + w := &fakeSelfReportWriter{} + w.ensure = func(report fleetreport.Report, opts github.FleetReportOptions) (github.FleetReportResult, error) { + searches++ + if closedAt.IsZero() { + return github.FleetReportResult{Number: 14256, URL: "u", Created: true}, nil + } + return github.FleetReportResult{Number: 14256, URL: "u", ClosedAt: closedAt}, nil + } + deps := selfReportDeps(w, &now) + deps.limits = fleetreport.DefaultFilingLimits + report := runtimeReport("fp-seed", fleetreport.ClassUnclassifiedRepeated) + res := &fleetreport.Result{Reports: []fleetreport.Report{report}} + publishFleetReportsWith(context.Background(), slog.Default(), srv, res, deps) + if searches != 1 || !srv.FleetReportState().Open["fp-seed"].OpenedByHive { + t.Fatalf("initial filing: searches=%d state=%#v", searches, srv.FleetReportState().Open["fp-seed"]) + } + + // The scanner closes it as "already fixed on v5" two hours later; the + // spoke learns of the close on its next report attempt (its state still + // says open, so it re-searches once). + now = now.Add(2 * time.Hour) + closedAt = now + srv.UpdateFleetReportState(func(st *fleetreport.State) { + o := st.Open["fp-seed"] + o.PostedAt = time.Time{} + st.Open["fp-seed"] = o + }) + now = now.Add(time.Minute) + publishFleetReportsWith(context.Background(), slog.Default(), srv, res, deps) + o := srv.FleetReportState().Open["fp-seed"] + if searches != 2 || !o.ClosedAt.Equal(closedAt) || !o.ClosedObservedAt.Equal(now) { + t.Fatalf("close not recorded: searches=%d state=%#v", searches, o) + } + + // Three weeks of daily channel rollouts, each restarting the spoke on a + // new build with the condition still firing every tick: no search, no + // create, no relay — and the suppression keeps sliding forward. + for day := 1; day <= 21; day++ { + now = now.Add(24 * time.Hour) + deps.version = "5.186.4-" + string(rune('a'+day)) + publishFleetReportsWith(context.Background(), slog.Default(), srv, res, deps) + } + o = srv.FleetReportState().Open["fp-seed"] + if searches != 2 || !o.ClosedObservedAt.Equal(now) || len(srv.FleetReportState().Filings) != 1 { + t.Fatalf("closed fingerprint was re-filed: searches=%d filings=%d state=%#v", searches, len(srv.FleetReportState().Filings), o) + } + + // A flap while closed: the condition clears and comes back. Nothing is + // written under the closed issue. + rec := &fleetreport.Result{Recoveries: []fleetreport.Report{{Fingerprint: "fp-seed", Trigger: fleetreport.TriggerRuntimeError, Class: fleetreport.ClassUnclassifiedRepeated, Body: "recovered"}}} + publishFleetReportsWith(context.Background(), slog.Default(), srv, rec, deps) + if w.clearCalls != 0 || len(w.recovered) != 0 { + t.Fatalf("wrote under a closed issue: cleared=%d recovered=%d", w.clearCalls, len(w.recovered)) + } + + // Only once the condition has been gone for the whole cooldown does a + // recurrence count as a regression and get searched (and, with the + // upstream issue still closed but stale, re-filed by the writer). + now = now.Add(8 * 24 * time.Hour) + publishFleetReportsWith(context.Background(), slog.Default(), srv, res, deps) + if searches != 3 { + t.Fatalf("regression after a quiet cooldown was not re-searched: searches=%d", searches) + } +} + +// A "Cleared on" comment is posted at most once per seen-gap for a flapping +// condition, and a comment attempt that finds the issue closed records the +// close instead of retrying (#13825 drew 79 such comments in a day). +func TestPublishFleetReportsPacesClearedCommentsAndLearnsOfClose(t *testing.T) { + srv := selfReportTestServer(t) + now := time.Date(2026, 10, 23, 17, 10, 0, 0, time.UTC) + w := &fakeSelfReportWriter{} + deps := selfReportDeps(w, &now) + deps.limits = fleetreport.DefaultFilingLimits + srv.UpdateFleetReportState(func(st *fleetreport.State) { + st.Open["fp-flap"] = fleetreport.OpenIssue{Number: 13825, OpenedByHive: true, Trigger: fleetreport.TriggerRuntimeError, Class: fleetreport.ClassUnclassifiedRepeated, PostedAt: now} + }) + rec := &fleetreport.Result{Recoveries: []fleetreport.Report{{Fingerprint: "fp-flap", Trigger: fleetreport.TriggerRuntimeError, Class: fleetreport.ClassUnclassifiedRepeated, Body: "recovered"}}} + for i := 0; i < 6; i++ { + now = now.Add(20 * time.Minute) + // Evaluate flips Recovered back to false when the condition re-fires. + srv.UpdateFleetReportState(func(st *fleetreport.State) { + o := st.Open["fp-flap"] + o.Recovered = false + st.Open["fp-flap"] = o + }) + publishFleetReportsWith(context.Background(), slog.Default(), srv, rec, deps) + } + if len(w.cleared) != 1 || !srv.FleetReportState().Open["fp-flap"].Recovered { + t.Fatalf("flapping condition posted %d cleared comments", len(w.cleared)) + } + now = now.Add(deps.limits.SeenGap) + srv.UpdateFleetReportState(func(st *fleetreport.State) { + o := st.Open["fp-flap"] + o.Recovered = false + st.Open["fp-flap"] = o + }) + publishFleetReportsWith(context.Background(), slog.Default(), srv, rec, deps) + if len(w.cleared) != 2 { + t.Fatalf("cleared comment refused after the seen gap: %d", len(w.cleared)) + } + + // The maintainer closed it meanwhile: the writer reports the close, the + // publisher records it and stops. + closedAt := now.Add(-5 * time.Minute) + w.clearedErr = func() error { return &github.FleetReportClosedError{Number: 13825, ClosedAt: closedAt} } + now = now.Add(deps.limits.SeenGap) + srv.UpdateFleetReportState(func(st *fleetreport.State) { + o := st.Open["fp-flap"] + o.Recovered = false + st.Open["fp-flap"] = o + }) + publishFleetReportsWith(context.Background(), slog.Default(), srv, rec, deps) + o := srv.FleetReportState().Open["fp-flap"] + if len(w.cleared) != 2 || !o.ClosedAt.Equal(closedAt) || !o.Recovered { + t.Fatalf("close via cleared comment not recorded: cleared=%d state=%#v", len(w.cleared), o) + } + before := w.clearCalls + srv.UpdateFleetReportState(func(st *fleetreport.State) { + o := st.Open["fp-flap"] + o.Recovered = false + st.Open["fp-flap"] = o + }) + now = now.Add(deps.limits.SeenGap) + publishFleetReportsWith(context.Background(), slog.Default(), srv, rec, deps) + if w.clearCalls != before { + t.Fatal("kept trying to comment under a closed issue") + } +} + +// The hub relay answers closed_upstream for a recently closed fingerprint; +// the spoke records the close like a direct closed match and does not retry. +func TestPublishFleetReportsRecordsCloseFromRelay(t *testing.T) { + srv := selfReportTestServer(t) + now := time.Date(2026, 10, 25, 17, 10, 0, 0, time.UTC) + w := &fakeSelfReportWriter{ensure: func(fleetreport.Report, github.FleetReportOptions) (github.FleetReportResult, error) { + return github.FleetReportResult{}, &gh.ErrorResponse{Response: &http.Response{StatusCode: http.StatusForbidden}, Message: "Resource not accessible by integration"} + }} + closedAt := now.Add(-time.Hour) + relays := 0 + deps := selfReportDeps(w, &now) + deps.limits = fleetreport.DefaultFilingLimits + deps.relay = func(_ context.Context, req selfReportRelayRequest) (selfReportRelayResponse, error) { + relays++ + return selfReportRelayResponse{OK: true, Number: 14256, URL: "u", Reason: "closed_upstream", ClosedAt: closedAt}, nil + } + res := &fleetreport.Result{Reports: []fleetreport.Report{runtimeReport("fp-relay-closed", fleetreport.ClassUnclassifiedRepeated)}} + publishFleetReportsWith(context.Background(), slog.Default(), srv, res, deps) + o := srv.FleetReportState().Open["fp-relay-closed"] + if relays != 1 || o.Number != 14256 || !o.ClosedAt.Equal(closedAt) || o.OpenedByHive || !o.ClosedObservedAt.Equal(now) { + t.Fatalf("relay close not recorded: relays=%d state=%#v", relays, o) + } + for i := 0; i < 3; i++ { + now = now.Add(24 * time.Hour) + publishFleetReportsWith(context.Background(), slog.Default(), srv, res, deps) + } + if relays != 1 || len(w.ensures) != 1 { + t.Fatalf("closed fingerprint retried: relays=%d ensures=%d", relays, len(w.ensures)) + } +} diff --git a/src/cmd/hive/selfupgrade.go b/src/cmd/hive/selfupgrade.go index b898178122..b9c4b4e662 100644 --- a/src/cmd/hive/selfupgrade.go +++ b/src/cmd/hive/selfupgrade.go @@ -25,10 +25,17 @@ const ( // selfUpgradeFailureExitCode marks a process exit caused by a FAILED // self-upgrade. Distinct from 0 so the failure is visible in the container's // termination state instead of looking like a clean shutdown. - selfUpgradeFailureExitCode = 17 - selfUpgradeImageCheckTimeout = 5 * time.Second - selfUpgradeGHCRBase = "https://ghcr.io" - selfUpgradeSpokeRepo = "hivecommons/hive" + selfUpgradeFailureExitCode = 17 + // selfUpgradePendingRestartExitCode marks a process exit taken because the + // Deployment PATCH failed but the Deployment tracks a floating tag the + // kubelet re-pulls on start, so the restart itself is expected to land the + // upgrade. Distinct from 17 so the termination state says "fell back to a + // restart", not "failed", and from 18/19 (duplicate-process, proxy-inject + // guards in main.go). + selfUpgradePendingRestartExitCode = 20 + selfUpgradeImageCheckTimeout = 5 * time.Second + selfUpgradeGHCRBase = "https://ghcr.io" + selfUpgradeSpokeRepo = "hivecommons/hive" ) // upgradeMarker is the on-PVC record at /data/upgrade-requested. It survives diff --git a/src/cmd/hive/spektacularwire.go b/src/cmd/hive/spektacularwire.go index 1aaa7c82aa..d3cd6b5e43 100644 --- a/src/cmd/hive/spektacularwire.go +++ b/src/cmd/hive/spektacularwire.go @@ -13,10 +13,10 @@ import ( "sync" "time" + "github.com/hivecommons/hive/crux/pushbroker" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/planengine" - "github.com/hivecommons/hive/pkg/pushbroker" // Linking pkg/spektacular registers the default "spektacular" engine. _ "github.com/hivecommons/hive/pkg/spektacular" ) diff --git a/src/cmd/hive/spektacularwire_test.go b/src/cmd/hive/spektacularwire_test.go index 2826dd9729..22fc1efe2c 100644 --- a/src/cmd/hive/spektacularwire_test.go +++ b/src/cmd/hive/spektacularwire_test.go @@ -8,10 +8,10 @@ import ( "testing" "time" + "github.com/hivecommons/hive/crux/pushbroker" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/planengine" - "github.com/hivecommons/hive/pkg/pushbroker" ) func TestWireSpektacularRunner_DefaultOffAndOptIn(t *testing.T) { diff --git a/src/cmd/hive/spoke_dashboard_url_10282_test.go b/src/cmd/hive/spoke_dashboard_url_10282_test.go index ea0403c5a0..e50619ec03 100644 --- a/src/cmd/hive/spoke_dashboard_url_10282_test.go +++ b/src/cmd/hive/spoke_dashboard_url_10282_test.go @@ -3,8 +3,8 @@ package main import ( "testing" + "github.com/hivecommons/hive/crux/taskmcp" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/taskmcp" ) // #10282: the task MCP URL handed to hub-launched agents must name a host this diff --git a/src/cmd/hive/state_restore.go b/src/cmd/hive/state_restore.go index 5093ac7c3a..e036ab928d 100644 --- a/src/cmd/hive/state_restore.go +++ b/src/cmd/hive/state_restore.go @@ -7,9 +7,8 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux" "github.com/hivecommons/hive/pkg/snapshot" ) @@ -84,7 +83,7 @@ func pausedRestoreDetail(enabled map[string]config.AgentConfig, onDemandFromPack return detail } -func restoreAgentRuntimeState(saved *snapshot.PersistedState, cfg *config.Config, agentMgr *agent.Manager, logger *slog.Logger) { +func restoreAgentRuntimeState(saved *snapshot.PersistedState, cfg *config.Config, agentMgr crux.Runtime, logger *slog.Logger) { for name, as := range saved.Agents { if _, inConfig := cfg.Agents[name]; !inConfig { logger.Info("skipping saved state for agent not in config", "agent", name) diff --git a/src/cmd/hive/state_restore_branches_test.go b/src/cmd/hive/state_restore_branches_test.go index 2044f97cec..e6886dd3e7 100644 --- a/src/cmd/hive/state_restore_branches_test.go +++ b/src/cmd/hive/state_restore_branches_test.go @@ -4,7 +4,9 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/snapshot" ) @@ -27,7 +29,7 @@ func TestRestoreAgentRuntimeState_SkipsAgentNotInConfig(t *testing.T) { }, }, } - m := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + m := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) wireGatewayChecker(m, cfg) restoreAgentRuntimeState(&snapshot.PersistedState{ @@ -56,7 +58,7 @@ func TestRestoreAgentRuntimeState_DefaultPauseProvenance(t *testing.T) { }, }, } - m := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + m := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) wireGatewayChecker(m, cfg) pausedAt := time.Now().Add(-48 * time.Hour).Truncate(time.Second) @@ -95,7 +97,7 @@ func TestRestoreAgentRuntimeState_PinsAndTelemetry(t *testing.T) { }, }, } - m := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + m := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) wireGatewayChecker(m, cfg) lastKick := time.Now().Add(-time.Hour).Truncate(time.Second) @@ -187,7 +189,7 @@ func TestRestoreAgentRuntimeState_ConfigFieldMerge(t *testing.T) { }, }, } - m := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + m := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) wireGatewayChecker(m, cfg) enabled := false diff --git a/src/cmd/hive/state_restore_test.go b/src/cmd/hive/state_restore_test.go index 104b72171c..352ae9dfa7 100644 --- a/src/cmd/hive/state_restore_test.go +++ b/src/cmd/hive/state_restore_test.go @@ -6,7 +6,9 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/cruxtest" + + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/snapshot" ) @@ -27,7 +29,7 @@ func restoreTestLogger() *slog.Logger { // wireGatewayChecker mirrors main()'s (post-fix) wiring: the predicate is // attached right after the manager is built, BEFORE any state replay. -func wireGatewayChecker(m *agent.Manager, cfg *config.Config) { +func wireGatewayChecker(m crux.Host, cfg *config.Config) { m.SetGatewayBackendChecker(func(backend string) bool { return cfg.Governor.ResolveGateway(backend) != nil && backend != "" }) @@ -76,7 +78,7 @@ func TestRestoreAgentRuntimeState_SurvivesRestart(t *testing.T) { if err != nil { t.Fatalf("LoadState: %v", err) } - m := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + m := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) wireGatewayChecker(m, cfg) restoreAgentRuntimeState(saved, cfg, m, restoreTestLogger()) @@ -113,7 +115,7 @@ func TestRestoreAgentRuntimeState_CLIBackendOverride(t *testing.T) { }, }, } - m := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + m := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) wireGatewayChecker(m, cfg) restoreAgentRuntimeState(&snapshot.PersistedState{ @@ -143,7 +145,7 @@ func TestRestoreAgentRuntimeState_DeletedGatewayNotRestored(t *testing.T) { // No gateways configured: "corp-litellm" is unresolvable. (Deliberately // NOT a CLI backend name — those pass validation without any gateway.) } - m := agent.NewManager(cfg.Agents, restoreTestLogger(), agent.ProjectContext{}) + m := cruxtest.NewRuntime(cfg.Agents, restoreTestLogger(), crux.ProjectContext{}) wireGatewayChecker(m, cfg) restoreAgentRuntimeState(&snapshot.PersistedState{ diff --git a/src/cmd/hive/trajectory_metrics.go b/src/cmd/hive/trajectory_metrics.go index 46709df8aa..99e03da137 100644 --- a/src/cmd/hive/trajectory_metrics.go +++ b/src/cmd/hive/trajectory_metrics.go @@ -3,8 +3,8 @@ package main import ( "time" + "github.com/hivecommons/hive/crux/trajectory" "github.com/hivecommons/hive/pkg/dashboard" - "github.com/hivecommons/hive/pkg/trajectory" ) // trajectoryMetricsCounters adapts the trajectory lane's Stats to the diff --git a/src/cmd/hive/turn_loss_persist_test.go b/src/cmd/hive/turn_loss_persist_test.go index 6f575a6a86..8c190b1aeb 100644 --- a/src/cmd/hive/turn_loss_persist_test.go +++ b/src/cmd/hive/turn_loss_persist_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux" + "github.com/hivecommons/hive/crux" ) // The turn-loss measurement (#4002 open question 3) is worthless unless it diff --git a/src/cmd/hive/watchdog_auth_probe_test.go b/src/cmd/hive/watchdog_auth_probe_test.go new file mode 100644 index 0000000000..af52844832 --- /dev/null +++ b/src/cmd/hive/watchdog_auth_probe_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/crux/watchdog" + "github.com/hivecommons/hive/pkg/rotation" +) + +// These two tests used to live in pkg/watchdog against rotation.ClaudeProber +// directly; they moved here with the usageProber adapter, since the watchdog +// no longer imports pkg/rotation (hivecommons/hive#13536). + +func claudeCredentialsFile(t *testing.T, refreshToken string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "credentials.json") + body := fmt.Sprintf(`{"claudeAiOauth":{"accessToken":"stale-token","refreshToken":%q,"expiresAt":%d}}`, + refreshToken, time.Now().Add(-time.Minute).UnixMilli()) + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +type unauthorizedTransport struct{} + +func (unauthorizedTransport) RoundTrip(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusUnauthorized, Body: http.NoBody, Header: make(http.Header)}, nil +} + +func claudeAuthProbeForCredentials(path string) watchdog.RotationAuthProbe { + return watchdog.RotationAuthProbe{Prober: usageProber{rotation.ClaudeProber{ + ThresholdPct: 80, + BaseURL: "https://example.invalid", + Client: &http.Client{Transport: unauthorizedTransport{}}, + CredentialsPath: path, + }}} +} + +func TestRotationAuthProbeExpiredClaudeTokenWithRefreshIsNotFailed(t *testing.T) { + p := claudeAuthProbeForCredentials(claudeCredentialsFile(t, "refresh-token")) + if p.Provider() != "anthropic" { + t.Fatalf("provider = %q", p.Provider()) + } + got, detail := p.ProbeAuth(context.Background()) + if got != watchdog.AuthUnknown { + t.Fatalf("ProbeAuth = %s (%s), want %s: Claude can refresh this credential on use", got, detail, watchdog.AuthUnknown) + } + if detail == "" { + t.Fatal("the inconclusive verdict must explain the refreshable expiry") + } +} + +func TestRotationAuthProbeExpiredClaudeTokenWithoutRefreshStillFails(t *testing.T) { + p := claudeAuthProbeForCredentials(claudeCredentialsFile(t, "")) + got, detail := p.ProbeAuth(context.Background()) + if got != watchdog.AuthFailed { + t.Fatalf("ProbeAuth = %s (%s), want %s: this credential has no recovery grant", got, detail, watchdog.AuthFailed) + } +} diff --git a/src/cmd/hive/watchdog_queued_refs.go b/src/cmd/hive/watchdog_queued_refs.go index b9e9e0b308..015cdb6600 100644 --- a/src/cmd/hive/watchdog_queued_refs.go +++ b/src/cmd/hive/watchdog_queued_refs.go @@ -5,8 +5,8 @@ import ( "strconv" "strings" + "github.com/hivecommons/hive/crux/watchdog" "github.com/hivecommons/hive/pkg/github" - "github.com/hivecommons/hive/pkg/watchdog" ) // watchdogQueuedRefs names up to limit items of the actionable queue the diff --git a/src/cmd/hive/watchdog_queued_refs_test.go b/src/cmd/hive/watchdog_queued_refs_test.go index a1b36183de..7f2738195c 100644 --- a/src/cmd/hive/watchdog_queued_refs_test.go +++ b/src/cmd/hive/watchdog_queued_refs_test.go @@ -3,8 +3,8 @@ package main import ( "testing" + "github.com/hivecommons/hive/crux/watchdog" "github.com/hivecommons/hive/pkg/github" - "github.com/hivecommons/hive/pkg/watchdog" ) func TestWatchdogQueuedRefs(t *testing.T) { diff --git a/src/crux/Makefile b/src/crux/Makefile new file mode 100644 index 0000000000..280e867cf0 --- /dev/null +++ b/src/crux/Makefile @@ -0,0 +1,37 @@ +# Makefile for the Crux agent runtime module (github.com/hivecommons/hive/crux). +# Everything here works from this directory alone, with or without the +# src/go.work workspace: GOWORK=off is set so the targets prove the module is +# self-contained (hivecommons/hive#14138). +# +# make build -> bin/crux (version from `git describe`, or VERSION=...) +# make test -> go test ./... (crux/agent runs tmux tests; never run it +# from inside an agent pane — parts of crux/agent exercise +# tmux process management and can kill the agent's own session; +# see https://github.com/hivecommons/hive/blob/v5/AGENTS.md) +# make vet -> go vet ./... +# make smoke -> build, start `crux serve` on a private unix socket, drive +# it with `crux --server`, stop it (scripts/smoke.sh) + +GO ?= go +BIN ?= bin/crux +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +LDFLAGS ?= -X main.version=$(VERSION) + +export GOWORK = off + +.PHONY: build test vet smoke clean + +build: + $(GO) build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN) ./cmd/crux + +test: + $(GO) test ./... + +vet: + $(GO) vet ./... + +smoke: build + CRUX_BIN=$(abspath $(BIN)) bash scripts/smoke.sh + +clean: + rm -rf bin .smoke diff --git a/src/crux/README.md b/src/crux/README.md new file mode 100644 index 0000000000..ff1b7d2ae5 --- /dev/null +++ b/src/crux/README.md @@ -0,0 +1,147 @@ +# Crux + +Crux is an agent runtime: it starts coding agents (Copilot, Claude, Codex, +Gemini and other CLIs) in managed tmux sessions, feeds them work ("kicks"), +watches their output, restarts or stops them, and exposes all of that as a +Go interface, a command line and a local HTTP+JSON API. It is the runtime +inside [Hive](https://github.com/hivecommons/hive) and is built to run on +its own: this directory is the Go module `github.com/hivecommons/hive/crux` +and depends on nothing else in the Hive repository. + +The design, the boundary rules and the roadmap are in +[`src/docs/design/crux.md`](../docs/design/crux.md); the operator pages are +[`crux-cli.md`](../docs/crux-cli.md) and [`crux-serve.md`](../docs/crux-serve.md). + +## What is in the module + +| Package | Role | +|---|---| +| `crux` | the public surface: `crux.Runtime` (`Definitions`, `Lifecycle`, `Observer`, `Kicker`), `crux.Process`, `crux.ProjectContext` | +| `crux/agent` | the manager: tmux sessions, backends, restarts, kick delivery, the only place a process is touched | +| `crux/config` | the configuration leaf: `config.AgentConfig` and the other YAML-shaped types, stdlib only | +| `crux/cli` | the `crux` command: `run`, `ps`, `logs`, `kick`, `stop`, `serve`, `agy-turn` (the pane-side agy subprocess), and `--server` remote mode | +| `crux/api`, `crux/serve` | the wire contract of `crux serve` (stdlib-only client and types, `api.RuntimeRoutes`: every `crux.Runtime` method → route) and the server | +| `crux/api/client` | `client.Runtime`: `crux.Runtime` over the API; what `crux/runtime.New` returns for `CRUX_MODE=sidecar` | +| `crux/api/callback`, `crux/hivecap` | the hive callback contract (capabilities hive serves the runtime: scoped tokens, agent mint, Bob credential, PRs, Linear, scanners, conditions, host policy, inference routes, effects, usage) and the runtime-side adapters `crux serve --hive-callback` installs; `api.HostSetters` is the table that accounts for every `crux.Host` setter (route / callback / config key / in-process-only) | +| `crux/runtime` | `runtime.New` (mode selection: `CRUX_MODE=inprocess` default, `sidecar`) and `runtime.Supervisor`, the child supervisor hive uses for `crux serve` when `CRUX_SUPERVISE=child` | +| `crux/defs` | the file-based agent-definition loader the module's binary uses | +| `crux/cmd/crux` | the binary | +| the rest | observation and policy: `watchdog`, `trajectory`, `logscrub`, `proxy`, `sandbox`, `inference`, … | + +## Build + +```sh +make build # -> bin/crux, version from `git describe` +GOWORK=off go build ./cmd/crux # the same, without make +go install github.com/hivecommons/hive/crux/cmd/crux@latest +``` + +The Makefile sets `GOWORK=off`, so every target proves the module builds +without the `src/go.work` workspace it sits in. `make test` runs the unit +tests (the `crux/agent` tests need `tmux`), `make vet` vets, `make smoke` +builds and exercises `crux serve` end to end on a private unix socket (no +agents are started; CI runs the same `scripts/smoke.sh`). + +## Run an agent + +Agent definitions are YAML files. Either the portable `AgentDefinition` +document (the format Hive's dashboard imports and exports): + +```yaml +# defs/scout.yaml +apiVersion: hive.kubestellar.io/v1 +kind: AgentDefinition +metadata: + name: scout + displayName: Scout +spec: + backend: copilot + model: claude-sonnet-4-6 + role: scout + clearOnKick: true +``` + +or a bare per-agent file, named by its file name (`defs/scout.yaml` defines +`scout`): + +```yaml +backend: copilot +model: claude-sonnet-4-6 +``` + +Then: + +```sh +export CRUX_DEFS=./defs # or --defs ./defs on every command; a single file works too +crux run scout # start scout, stream its output, stop it on Ctrl-C +crux run --detach scout # start it and return +crux ps # what is running +crux logs scout -n 50 # its recent pane output (--buffer, --full, --json) +crux kick scout "look at issue 42" # send it work +crux stop scout +crux run ./defs/scout.yaml # a definition file can be named directly +``` + +Agents live under `--workdir` (default `$HIVE_WORK_DIR`, then `$HOME/.crux`). +While a foreground `crux run` is up it serves a control socket in the work +dir, and `ps`, `logs`, `kick` and `stop` talk to it so they see the live +runtime; `--local` builds a fresh one instead. + +## Serve the runtime + +```sh +crux serve & # unix socket: $CRUX_SOCKET, else $XDG_RUNTIME_DIR/crux/crux.sock +crux serve --socket /run/crux/crux.sock +crux serve --listen 127.0.0.1:7777 --token "$T" # TCP too; loopback only, token required +``` + +`crux serve` exposes the runtime over HTTP+JSON under `/v1`: list and get +agents, run from a definition, logs (pane, buffer, full, or followed as +server-sent events), kick, stop, health, and an event stream of starts, +stops, kicks and state changes. The contract is `crux/api`, which also has +a Go client (`api.NewClient`). [`crux-serve.md`](../docs/crux-serve.md) +has the routes. + +## Remote mode + +Every verb except `serve` can drive a `crux serve` instead of an in-process +runtime: + +```sh +export CRUX_SERVER=/run/crux/crux.sock # or --server; unix:///path, host:port and http://… also work +export CRUX_TOKEN=... # or --token, when the server has one +crux ps +crux run ./defs/scout.yaml # the file is read here and sent to the server +crux logs -f scout # follows over SSE +``` + +`crux --version` prints the version embedded at build time +(`-ldflags "-X main.version=…"`, `dev` otherwise). + +## Using Crux from Go + +```go +import ( + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/defs" +) + +defs, _ := defs.Load("./defs") +var rt crux.Runtime = agent.AsRuntime(agent.NewManager(defs, logger, crux.ProjectContext{})) +``` + +Everything a host needs is on `crux.Runtime`; `*agent.Manager` is the +constructor and nothing more. Hive itself is such a host: its `hive` binary +builds the manager the same way, and its own `crux` build +(`src/cmd/crux` in the root module) is the binary here with Hive's +`hive.yaml` loader plugged into `cli.DefinitionLoader` in place of +`crux/defs`. + +## Boundaries + +Nothing under this module imports the Hive root module; a test in the root +repository (`src/internal/testutil/crux_outbound_ratchet_test.go`) keeps it +that way. `crux/config` and `crux/api` are stdlib-only. The tmux-backed +tests in `crux/agent` create and kill real sessions: do not run them from +inside an agent's own tmux pane. diff --git a/src/crux/activity.go b/src/crux/activity.go new file mode 100644 index 0000000000..a3d365cbbe --- /dev/null +++ b/src/crux/activity.go @@ -0,0 +1,13 @@ +package crux + +import "time" + +// ActivityObserver exposes the liveness and capability evidence used by the +// spoke's heartbeat. It does not launch or modify agent processes. +type ActivityObserver interface { + SessionMissing(name string) bool + AgentCapabilities(name string) (canOpenIssue, canOpenPR, canMerge, ok bool) + EffectiveBackend(name string) (backend string, ok bool) + AgentCredentialRefreshable(name string) (refreshable, known bool) + RestartTelemetry(name string) (total, last24h int, lastRestartAt time.Time, lastReason string, ok bool) +} diff --git a/src/crux/advisorhook/advisorhook.go b/src/crux/advisorhook/advisorhook.go new file mode 100644 index 0000000000..291632fd83 --- /dev/null +++ b/src/crux/advisorhook/advisorhook.go @@ -0,0 +1,29 @@ +// Package advisorhook holds the advisor lane's launch-side contract: the +// `hive advisor-hook` subcommand name, its output dialects and the advise +// endpoint variable. crux/agent projects these into agent hook files at +// launch; the hub side (pkg/advisor) serves the endpoint and implements the +// subcommand, and re-exports these names so its callers are unchanged. +package advisorhook + +// HookSubcommand is the `hive advisor-hook` subcommand every supported +// backend invokes at its turn boundary. It reads the transcript reference the +// backend provides on stdin, asks the hub's loopback advise endpoint for the +// verdict, and prints the answer in the calling backend's dialect. Keeping +// the backend-specific part to formatting is what makes each adapter thin. +const HookSubcommand = "advisor-hook" + +// HookFormatCopilot renders the Copilot CLI `agentStop` dialect (camelCase +// payload, `decision: block` with a reason that becomes the forced next +// prompt). HookFormatCodex is the Codex CLI `Stop` dialect, which matches +// Claude Code's payload and answer. +const ( + HookFormatCopilot = "copilot" + HookFormatCodex = "codex" +) + +// EndpointEnvVar carries the advise endpoint base URL to advised agents, the +// way HIVE_JEV_ENDPOINT carries the Jev endpoint. +const EndpointEnvVar = "HIVE_ADVISOR_ENDPOINT" + +// DefaultEndpoint is what EndpointEnvVar carries to agents. +const DefaultEndpoint = "http://127.0.0.1:18447" diff --git a/src/crux/advisorhook/advisorhook_test.go b/src/crux/advisorhook/advisorhook_test.go new file mode 100644 index 0000000000..61952f95b8 --- /dev/null +++ b/src/crux/advisorhook/advisorhook_test.go @@ -0,0 +1,43 @@ +package advisorhook + +import ( + "net/url" + "strings" + "testing" +) + +// The contract is a set of names shared with the hub side (pkg/advisor +// re-exports them); pin the wire values so a rename on either side is a +// deliberate, visible change rather than a silent hook-file drift. +func TestContractValues(t *testing.T) { + if HookSubcommand != "advisor-hook" { + t.Fatalf("HookSubcommand = %q, want advisor-hook", HookSubcommand) + } + if HookFormatCopilot != "copilot" || HookFormatCodex != "codex" { + t.Fatalf("hook formats = %q/%q, want copilot/codex", HookFormatCopilot, HookFormatCodex) + } + if HookFormatCopilot == HookFormatCodex { + t.Fatal("hook format dialects must be distinct") + } + if EndpointEnvVar != "HIVE_ADVISOR_ENDPOINT" || !strings.HasPrefix(EndpointEnvVar, "HIVE_") { + t.Fatalf("EndpointEnvVar = %q, want HIVE_ADVISOR_ENDPOINT", EndpointEnvVar) + } +} + +// The default endpoint must be a loopback http URL an agent can hand to its +// hook without further parsing. +func TestDefaultEndpointIsLoopbackURL(t *testing.T) { + u, err := url.Parse(DefaultEndpoint) + if err != nil { + t.Fatalf("DefaultEndpoint %q does not parse: %v", DefaultEndpoint, err) + } + if u.Scheme != "http" { + t.Fatalf("DefaultEndpoint scheme = %q, want http", u.Scheme) + } + if u.Hostname() != "127.0.0.1" { + t.Fatalf("DefaultEndpoint host = %q, want 127.0.0.1 (loopback only)", u.Hostname()) + } + if u.Port() == "" { + t.Fatalf("DefaultEndpoint %q has no port", DefaultEndpoint) + } +} diff --git a/src/pkg/agent/acmm_gate.go b/src/crux/agent/acmm_gate.go similarity index 57% rename from src/pkg/agent/acmm_gate.go rename to src/crux/agent/acmm_gate.go index 9d880f589e..83b50c0f51 100644 --- a/src/pkg/agent/acmm_gate.go +++ b/src/crux/agent/acmm_gate.go @@ -1,14 +1,10 @@ package agent -const operabilityAgentMinACMMLevel = 5 +import "github.com/hivecommons/hive/crux" // AgentAvailableAtACMMLevel is the hard ACMM roster gate for agents whose // existence is level-scoped, not just whose GitHub write authority is scoped. +// The gate lives in the crux root package (crux.AgentAvailableAtACMMLevel). func AgentAvailableAtACMMLevel(name string, level int) bool { - switch name { - case "telemetry", "operations": - return level >= operabilityAgentMinACMMLevel - default: - return true - } + return crux.AgentAvailableAtACMMLevel(name, level) } diff --git a/src/pkg/agent/advisor_hook.go b/src/crux/agent/advisor_hook.go similarity index 98% rename from src/pkg/agent/advisor_hook.go rename to src/crux/agent/advisor_hook.go index ce3e519c49..030f5d7ab4 100644 --- a/src/pkg/agent/advisor_hook.go +++ b/src/crux/agent/advisor_hook.go @@ -14,8 +14,8 @@ import ( "sync" "time" - "github.com/hivecommons/hive/pkg/advisor" - "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/crux/advisorhook" + "github.com/hivecommons/hive/crux/config" ) // This file is the Claude Code adapter's launch-side half of the advisor lane @@ -67,7 +67,7 @@ var advisorHookRunnerBinary = func() string { // advisor's Stop hook for one Claude Code launch. The timeout gives the hook // room for the hub-side review timeout plus the round trip. func claudeAdvisorSettings() string { - command := fmt.Sprintf("%s %s", advisorHookRunnerBinary(), advisor.HookSubcommand) + command := fmt.Sprintf("%s %s", advisorHookRunnerBinary(), advisorhook.HookSubcommand) settings := map[string]any{ "hooks": map[string]any{ "Stop": []map[string]any{ @@ -102,7 +102,7 @@ const ( // copilotAdvisorHooksJSON renders the Copilot CLI hook file registering the // advisor on `agentStop`. The command asks for the Copilot dialect. func copilotAdvisorHooksJSON() []byte { - command := fmt.Sprintf("%s %s --format %s", advisorHookRunnerBinary(), advisor.HookSubcommand, advisor.HookFormatCopilot) + command := fmt.Sprintf("%s %s --format %s", advisorHookRunnerBinary(), advisorhook.HookSubcommand, advisorhook.HookFormatCopilot) doc := map[string]any{ "version": 1, "hooks": map[string]any{ @@ -121,7 +121,7 @@ func copilotAdvisorHooksJSON() []byte { // codexAdvisorHooksJSON renders the Codex CLI hooks.json registering the // advisor's `Stop` hook. Codex speaks the same dialect as Claude Code here. func codexAdvisorHooksJSON() []byte { - command := fmt.Sprintf("%s %s --format %s", advisorHookRunnerBinary(), advisor.HookSubcommand, advisor.HookFormatCodex) + command := fmt.Sprintf("%s %s --format %s", advisorHookRunnerBinary(), advisorhook.HookSubcommand, advisorhook.HookFormatCodex) doc := map[string]any{ "hooks": map[string]any{ "Stop": []map[string]any{ @@ -199,7 +199,7 @@ func (m *Manager) provisionAdvisorHooks(agent *AgentProcess, backend string) { } user := m.agentExecUserSpec(agent) if !m.advisorHooksActive(agent, backend) { - if existing, err := readFileAsUser(user, path); err == nil && strings.Contains(string(existing), advisor.HookSubcommand) { + if existing, err := readFileAsUser(user, path); err == nil && strings.Contains(string(existing), advisorhook.HookSubcommand) { if err := exec.Command("su-exec", user, "rm", "-f", path).Run(); err != nil { m.logger.Warn("failed to remove stale advisor hook file", "agent", agent.Name, "path", path, "error", err) } diff --git a/src/pkg/agent/advisor_hook_test.go b/src/crux/agent/advisor_hook_test.go similarity index 97% rename from src/pkg/agent/advisor_hook_test.go rename to src/crux/agent/advisor_hook_test.go index edafb3ee86..3f1b6e63dc 100644 --- a/src/pkg/agent/advisor_hook_test.go +++ b/src/crux/agent/advisor_hook_test.go @@ -12,8 +12,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/advisor" - "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/crux/advisorhook" + "github.com/hivecommons/hive/crux/config" ) func advisorTestManager() *Manager { @@ -65,7 +65,7 @@ func TestClaudeAdvisorSettingsShape(t *testing.T) { if h.Type != "command" { t.Errorf("hook type = %q", h.Type) } - if h.Command != "/usr/local/bin/hive "+advisor.HookSubcommand { + if h.Command != "/usr/local/bin/hive "+advisorhook.HookSubcommand { t.Errorf("hook command = %q", h.Command) } if h.Timeout <= 0 { @@ -92,7 +92,7 @@ func TestAdvisorLaunchFlag(t *testing.T) { if !strings.HasPrefix(flag, " --settings '") { t.Fatalf("flag = %q", flag) } - if !strings.Contains(flag, `"Stop"`) || !strings.Contains(flag, advisor.HookSubcommand) { + if !strings.Contains(flag, `"Stop"`) || !strings.Contains(flag, advisorhook.HookSubcommand) { t.Errorf("flag must carry the Stop hook: %q", flag) } @@ -132,7 +132,7 @@ func TestAdvisorCopilotHooksJSONShape(t *testing.T) { t.Fatalf("want exactly one agentStop hook: %+v", doc) } h := doc.Hooks.AgentStop[0] - want := "/usr/local/bin/hive " + advisor.HookSubcommand + " --format copilot" + want := "/usr/local/bin/hive " + advisorhook.HookSubcommand + " --format copilot" if h.Type != "command" || h.Bash != want || h.TimeoutSec <= 0 { t.Errorf("hook = %+v, want command %q", h, want) } @@ -161,7 +161,7 @@ func TestAdvisorCodexHooksJSONShape(t *testing.T) { t.Fatalf("want exactly one Stop hook: %+v", doc) } h := doc.Hooks.Stop[0].Hooks[0] - want := "/usr/local/bin/hive " + advisor.HookSubcommand + " --format codex" + want := "/usr/local/bin/hive " + advisorhook.HookSubcommand + " --format codex" if h.Type != "command" || h.Command != want || h.Timeout <= 0 { t.Errorf("hook = %+v, want command %q", h, want) } diff --git a/src/pkg/agent/advisor_omp.go b/src/crux/agent/advisor_omp.go similarity index 97% rename from src/pkg/agent/advisor_omp.go rename to src/crux/agent/advisor_omp.go index 31e31344c0..58476f652b 100644 --- a/src/pkg/agent/advisor_omp.go +++ b/src/crux/agent/advisor_omp.go @@ -8,8 +8,8 @@ import ( "sort" "strings" - "github.com/hivecommons/hive/pkg/advisor" - "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/crux/advisorhook" + "github.com/hivecommons/hive/crux/config" ) // This file is the OMP adapter's launch-side half of the advisor lane @@ -61,7 +61,7 @@ func (m *Manager) advisorRoles() map[string]config.ModelRole { // fails open: an advisor outage must never stall the agent. func ompAdvisorExtension() string { binary, _ := json.Marshal(advisorHookRunnerBinary()) - subcommand, _ := json.Marshal(advisor.HookSubcommand) + subcommand, _ := json.Marshal(advisorhook.HookSubcommand) return fmt.Sprintf(`import { spawn } from "node:child_process"; const HOOK_BINARY = %s; diff --git a/src/pkg/agent/advisor_omp_test.go b/src/crux/agent/advisor_omp_test.go similarity index 95% rename from src/pkg/agent/advisor_omp_test.go rename to src/crux/agent/advisor_omp_test.go index 2f41ecafce..26465ccd2f 100644 --- a/src/pkg/agent/advisor_omp_test.go +++ b/src/crux/agent/advisor_omp_test.go @@ -6,8 +6,8 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/advisor" - "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/crux/advisorhook" + "github.com/hivecommons/hive/crux/config" ) func withOMPAdvisorDir(t *testing.T) string { @@ -28,7 +28,7 @@ func TestOMPAdvisorExtensionShape(t *testing.T) { for _, want := range []string{ `pi.on("session_stop"`, `"/usr/local/bin/hive"`, - `"` + advisor.HookSubcommand + `"`, + `"` + advisorhook.HookSubcommand + `"`, `decision: "block"`, `transcript_path: event.session_file`, } { diff --git a/src/pkg/agent/agent_bootstrap_test.go b/src/crux/agent/agent_bootstrap_test.go similarity index 99% rename from src/pkg/agent/agent_bootstrap_test.go rename to src/crux/agent/agent_bootstrap_test.go index 65d2592f5d..8253f17023 100644 --- a/src/pkg/agent/agent_bootstrap_test.go +++ b/src/crux/agent/agent_bootstrap_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func testManager(level int) *Manager { diff --git a/src/pkg/agent/agent_capabilities_test.go b/src/crux/agent/agent_capabilities_test.go similarity index 98% rename from src/pkg/agent/agent_capabilities_test.go rename to src/crux/agent/agent_capabilities_test.go index fe5e106d4b..ca20f2fdc8 100644 --- a/src/pkg/agent/agent_capabilities_test.go +++ b/src/crux/agent/agent_capabilities_test.go @@ -3,7 +3,7 @@ package agent import ( "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // AgentCapabilities must return exactly the mode's CanCreateIssues/CanPush/ diff --git a/src/pkg/agent/agent_repo_scope_test.go b/src/crux/agent/agent_repo_scope_test.go similarity index 98% rename from src/pkg/agent/agent_repo_scope_test.go rename to src/crux/agent/agent_repo_scope_test.go index 212dcccb0e..be41be52bc 100644 --- a/src/pkg/agent/agent_repo_scope_test.go +++ b/src/crux/agent/agent_repo_scope_test.go @@ -3,7 +3,7 @@ package agent import ( "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func scopedProject(scopes map[string][]string) ProjectContext { diff --git a/src/pkg/agent/agent_state_file_test.go b/src/crux/agent/agent_state_file_test.go similarity index 100% rename from src/pkg/agent/agent_state_file_test.go rename to src/crux/agent/agent_state_file_test.go diff --git a/src/pkg/agent/agent_unit_test.go b/src/crux/agent/agent_unit_test.go similarity index 99% rename from src/pkg/agent/agent_unit_test.go rename to src/crux/agent/agent_unit_test.go index 053285748d..02134ebcd3 100644 --- a/src/pkg/agent/agent_unit_test.go +++ b/src/crux/agent/agent_unit_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestSetAndGetACMMLevel(t *testing.T) { @@ -27,7 +27,7 @@ func TestSetAndGetACMMLevel(t *testing.T) { func TestOperabilityAgentsRequireACMML5(t *testing.T) { for _, name := range []string{"telemetry", "operations"} { - for level := 0; level < operabilityAgentMinACMMLevel; level++ { + for level := 0; level < 5; level++ { if AgentAvailableAtACMMLevel(name, level) { t.Fatalf("%s should be unavailable at L%d", name, level) } diff --git a/src/pkg/agent/agent_working_test.go b/src/crux/agent/agent_working_test.go similarity index 99% rename from src/pkg/agent/agent_working_test.go rename to src/crux/agent/agent_working_test.go index 9be82e54e4..5bfb1d4962 100644 --- a/src/pkg/agent/agent_working_test.go +++ b/src/crux/agent/agent_working_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestPaneShowsAgentWorking is the regression guard for #7085: modern TUI diff --git a/src/pkg/agent/agentspecwire.go b/src/crux/agent/agentspecwire.go similarity index 96% rename from src/pkg/agent/agentspecwire.go rename to src/crux/agent/agentspecwire.go index 8437e63bbe..b647ad08e7 100644 --- a/src/pkg/agent/agentspecwire.go +++ b/src/crux/agent/agentspecwire.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/skillreg" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/skillreg" ) func (m *Manager) applyAgentSpec(agent *AgentProcess) error { diff --git a/src/pkg/agent/agentspecwire_test.go b/src/crux/agent/agentspecwire_test.go similarity index 99% rename from src/pkg/agent/agentspecwire_test.go rename to src/crux/agent/agentspecwire_test.go index 1215c72807..ac40faa41b 100644 --- a/src/pkg/agent/agentspecwire_test.go +++ b/src/crux/agent/agentspecwire_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestApplyAgentSpecConfigChangesLaunchCommand(t *testing.T) { diff --git a/src/pkg/agent/agy_launch_test.go b/src/crux/agent/agy_launch_test.go similarity index 100% rename from src/pkg/agent/agy_launch_test.go rename to src/crux/agent/agy_launch_test.go diff --git a/src/pkg/agent/agy_restart_gate_test.go b/src/crux/agent/agy_restart_gate_test.go similarity index 97% rename from src/pkg/agent/agy_restart_gate_test.go rename to src/crux/agent/agy_restart_gate_test.go index a3ab1dba14..6d2a891231 100644 --- a/src/pkg/agent/agy_restart_gate_test.go +++ b/src/crux/agent/agy_restart_gate_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestTokenRestartHealIsBackendScoped pins the gate that stopped agents being diff --git a/src/pkg/agent/agy_turn.go b/src/crux/agent/agy_turn.go similarity index 98% rename from src/pkg/agent/agy_turn.go rename to src/crux/agent/agy_turn.go index c5ed03959a..eb873ac9f7 100644 --- a/src/pkg/agent/agy_turn.go +++ b/src/crux/agent/agy_turn.go @@ -48,8 +48,14 @@ var agyTurnStatusInterval = 10 * time.Second // Ctrl+C before the runner kills it. const agyTurnInterruptGrace = 5 * time.Second +// AgyTurnSubcommand is the subcommand the manager types into a headless agy +// pane, on whichever binary hosts the manager: `hive agy-turn` in-process, +// `crux agy-turn` under `crux serve` (#14138 stage 3). crux/cli dispatches +// it to RunAgyTurn. +const AgyTurnSubcommand = "agy-turn" + const ( - agyTurnSubcommand = "agy-turn" + agyTurnSubcommand = AgyTurnSubcommand // agyTurnToolOutputLines caps how much of each tool's output is echoed // into the pane. The agent sees the full output; the pane only needs diff --git a/src/pkg/agent/agy_turn_test.go b/src/crux/agent/agy_turn_test.go similarity index 100% rename from src/pkg/agent/agy_turn_test.go rename to src/crux/agent/agy_turn_test.go diff --git a/src/pkg/agent/antigravity_credential_test.go b/src/crux/agent/antigravity_credential_test.go similarity index 100% rename from src/pkg/agent/antigravity_credential_test.go rename to src/crux/agent/antigravity_credential_test.go diff --git a/src/pkg/agent/antigravity_onboarding_test.go b/src/crux/agent/antigravity_onboarding_test.go similarity index 82% rename from src/pkg/agent/antigravity_onboarding_test.go rename to src/crux/agent/antigravity_onboarding_test.go index 436570f51f..3cbdef62f6 100644 --- a/src/pkg/agent/antigravity_onboarding_test.go +++ b/src/crux/agent/antigravity_onboarding_test.go @@ -1,14 +1,23 @@ package agent import ( + "errors" + "io/fs" "os" "path/filepath" "strings" "testing" + + "github.com/hivecommons/hive/crux/internal/testutil" ) func TestEntrypointRepairsAgyOnboardingFastPath(t *testing.T) { body, err := os.ReadFile("../../deploy/entrypoint.sh") + if errors.Is(err, fs.ErrNotExist) { + // deploy/ belongs to the hive root module; absent when the crux module + // is built standalone (hivecommons/crux). The parity is hive's to keep. + testutil.SkipUnlessRequired(t, "deploy/entrypoint.sh is not part of the crux module (standalone build)") + } if err != nil { t.Fatalf("read entrypoint: %v", err) } diff --git a/src/pkg/agent/antigravity_perm_guard_test.go b/src/crux/agent/antigravity_perm_guard_test.go similarity index 73% rename from src/pkg/agent/antigravity_perm_guard_test.go rename to src/crux/agent/antigravity_perm_guard_test.go index f55bc72a67..001e91ee9c 100644 --- a/src/pkg/agent/antigravity_perm_guard_test.go +++ b/src/crux/agent/antigravity_perm_guard_test.go @@ -1,13 +1,22 @@ package agent import ( + "errors" + "io/fs" "os" "strings" "testing" + + "github.com/hivecommons/hive/crux/internal/testutil" ) func TestEntrypointGeminiGuardIsBoundedToAntigravityCLI(t *testing.T) { body, err := os.ReadFile("../../deploy/entrypoint.sh") + if errors.Is(err, fs.ErrNotExist) { + // deploy/ belongs to the hive root module; absent when the crux module + // is built standalone (hivecommons/crux). The parity is hive's to keep. + testutil.SkipUnlessRequired(t, "deploy/entrypoint.sh is not part of the crux module (standalone build)") + } if err != nil { t.Fatalf("read entrypoint: %v", err) } diff --git a/src/pkg/agent/audit.go b/src/crux/agent/audit.go similarity index 75% rename from src/pkg/agent/audit.go rename to src/crux/agent/audit.go index 49b8de31c6..b0572db5ba 100644 --- a/src/pkg/agent/audit.go +++ b/src/crux/agent/audit.go @@ -1,32 +1,32 @@ package agent import ( - "github.com/hivecommons/hive/pkg/crux/types" + "github.com/hivecommons/hive/crux" ) // The audit vocabulary this runtime emits — the lifecycle action names, the -// AuditSink interface and the Fields helper — is DEFINED in the stdlib-only -// leaf pkg/crux/types, so policy-layer packages can emit audit events without +// AuditSink interface and the Fields helper — is DEFINED in the crux root +// package, so policy-layer packages can emit audit events without // importing this runtime package and the runtime does not import the // governor-side pkg/agentaudit (which aliases the same names for the // recorder). The governor-side vocabulary (lease stages, commit trailers, // artifact links, FormatAuditDetail) is agentaudit's alone and is referenced // there directly (hivecommons/hive#13536). const ( - AuditAgentStarted = types.AuditAgentStarted - AuditAgentStartFailed = types.AuditAgentStartFailed - AuditAgentStopped = types.AuditAgentStopped - AuditAgentKicked = types.AuditAgentKicked - AuditAgentPaused = types.AuditAgentPaused - AuditAgentResumed = types.AuditAgentResumed - AuditAgentAdded = types.AuditAgentAdded - AuditAgentRemoved = types.AuditAgentRemoved - AuditAgentBackendSet = types.AuditAgentBackendSet - AuditAgentModelSet = types.AuditAgentModelSet - AuditToolApproval = types.AuditToolApproval + AuditAgentStarted = crux.AuditAgentStarted + AuditAgentStartFailed = crux.AuditAgentStartFailed + AuditAgentStopped = crux.AuditAgentStopped + AuditAgentKicked = crux.AuditAgentKicked + AuditAgentPaused = crux.AuditAgentPaused + AuditAgentResumed = crux.AuditAgentResumed + AuditAgentAdded = crux.AuditAgentAdded + AuditAgentRemoved = crux.AuditAgentRemoved + AuditAgentBackendSet = crux.AuditAgentBackendSet + AuditAgentModelSet = crux.AuditAgentModelSet + AuditToolApproval = crux.AuditToolApproval - AuditCopilotTokenMissing = types.AuditCopilotTokenMissing - AuditClaudeTokenMissing = types.AuditClaudeTokenMissing + AuditCopilotTokenMissing = crux.AuditCopilotTokenMissing + AuditClaudeTokenMissing = crux.AuditClaudeTokenMissing ) // auditActorSystem attributes an event to the hive process itself rather than @@ -40,7 +40,7 @@ const auditActorSystem = "system" // AuditSink receives agent lifecycle events for durable, queryable recording. // See agentaudit.AuditSink for the contract; the alias exists because it is // the parameter type of SetAuditSink. -type AuditSink = types.AuditSink +type AuditSink = crux.AuditSink // SetAuditSink installs the durable audit sink. Safe to leave unset: a nil // sink makes every audit call a no-op, which is what unit tests and any @@ -73,7 +73,7 @@ func (m *Manager) auditWithActor(actor, action, agentName string, fields map[str } // auditFields builds a fields map from alternating key/value pairs; see -// types.Fields. +// crux.Fields. func auditFields(kv ...any) map[string]any { - return types.Fields(kv...) + return crux.Fields(kv...) } diff --git a/src/pkg/agent/audit_test.go b/src/crux/agent/audit_test.go similarity index 99% rename from src/pkg/agent/audit_test.go rename to src/crux/agent/audit_test.go index 8e8157151e..581e047717 100644 --- a/src/pkg/agent/audit_test.go +++ b/src/crux/agent/audit_test.go @@ -7,7 +7,7 @@ import ( "sync" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // recordedAudit is one captured audit event. diff --git a/src/pkg/agent/authorize_issue_open_test.go b/src/crux/agent/authorize_issue_open_test.go similarity index 98% rename from src/pkg/agent/authorize_issue_open_test.go rename to src/crux/agent/authorize_issue_open_test.go index 3221b13316..8b75d98888 100644 --- a/src/pkg/agent/authorize_issue_open_test.go +++ b/src/crux/agent/authorize_issue_open_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // AuthorizeIssueOpen is the gate for the file-based issue-request relay diff --git a/src/pkg/agent/authorize_push_branch_test.go b/src/crux/agent/authorize_push_branch_test.go similarity index 98% rename from src/pkg/agent/authorize_push_branch_test.go rename to src/crux/agent/authorize_push_branch_test.go index 88859b0bb5..dc5502eb9f 100644 --- a/src/pkg/agent/authorize_push_branch_test.go +++ b/src/crux/agent/authorize_push_branch_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // AuthorizePushBranch is the gate for the file-based push-branch relay diff --git a/src/pkg/agent/authorize_review_request_test.go b/src/crux/agent/authorize_review_request_test.go similarity index 99% rename from src/pkg/agent/authorize_review_request_test.go rename to src/crux/agent/authorize_review_request_test.go index 1085499c83..85cf21dc89 100644 --- a/src/pkg/agent/authorize_review_request_test.go +++ b/src/crux/agent/authorize_review_request_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // boolPtr is a local helper: AgentConfig.Converse is a *bool so that "unset" diff --git a/src/pkg/agent/authprobe.go b/src/crux/agent/authprobe.go similarity index 94% rename from src/pkg/agent/authprobe.go rename to src/crux/agent/authprobe.go index 53723e1dd8..cfd46d40d8 100644 --- a/src/pkg/agent/authprobe.go +++ b/src/crux/agent/authprobe.go @@ -7,32 +7,16 @@ import ( "sort" "strings" - "github.com/hivecommons/hive/pkg/claude" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/config" ) -// interactiveAuthBackends are the CLI backends whose credentials come from an -// INTERACTIVE login flow (OAuth / device flow) that a human must complete. Only -// these can ever legitimately show a login badge. -var interactiveAuthBackends = map[string]bool{ - "claude": true, - "copilot": true, - "codex": true, - "gemini": true, - "omp": true, -} - -// BackendRequiresInteractiveAuth reports whether a backend authenticates via an -// interactive login the operator must perform. Model-gateway backends -// (config.InferenceBackends: litellm, vllm, llm-d, watsonx) authenticate with -// an API key supplied by config — watsonx with an IAM bearer minted from that -// key — and therefore NEVER require a login. Showing them a 🔑 badge is always -// a false alarm. +// BackendRequiresInteractiveAuth is crux.BackendRequiresInteractiveAuth; the +// table lives in the root so the dashboard decides whether to show a login +// badge without importing the runtime. func BackendRequiresInteractiveAuth(backend string) bool { - if config.IsInferenceBackend(backend) { - return false - } - return interactiveAuthBackends[backend] + return crux.BackendRequiresInteractiveAuth(backend) } // AgentHome returns the HOME directory an agent's CLI runs with. It mirrors diff --git a/src/pkg/agent/authprobe_credential_gate_test.go b/src/crux/agent/authprobe_credential_gate_test.go similarity index 99% rename from src/pkg/agent/authprobe_credential_gate_test.go rename to src/crux/agent/authprobe_credential_gate_test.go index 1811d32192..94a4098588 100644 --- a/src/pkg/agent/authprobe_credential_gate_test.go +++ b/src/crux/agent/authprobe_credential_gate_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // writeExpiredClaudeCreds writes a credentials.json whose OAuth token expired diff --git a/src/pkg/agent/authprobe_omp_test.go b/src/crux/agent/authprobe_omp_test.go similarity index 97% rename from src/pkg/agent/authprobe_omp_test.go rename to src/crux/agent/authprobe_omp_test.go index 6dc8983932..90d5d4ef74 100644 --- a/src/pkg/agent/authprobe_omp_test.go +++ b/src/crux/agent/authprobe_omp_test.go @@ -5,7 +5,7 @@ import ( "reflect" "testing" - "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/crux/config" ) func ompAgent(name string, uid int, backend, override string) *AgentProcess { diff --git a/src/pkg/agent/authprobe_test.go b/src/crux/agent/authprobe_test.go similarity index 99% rename from src/pkg/agent/authprobe_test.go rename to src/crux/agent/authprobe_test.go index ec4782ef8b..6e8c3a17b7 100644 --- a/src/pkg/agent/authprobe_test.go +++ b/src/crux/agent/authprobe_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // writeClaudeCreds writes a credentials.json with a non-expired OAuth token at diff --git a/src/pkg/agent/backend_auth.go b/src/crux/agent/backend_auth.go similarity index 89% rename from src/pkg/agent/backend_auth.go rename to src/crux/agent/backend_auth.go index 67df0214f6..9c7b6b9eda 100644 --- a/src/pkg/agent/backend_auth.go +++ b/src/crux/agent/backend_auth.go @@ -4,20 +4,20 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/crux/types" + "github.com/hivecommons/hive/crux" ) -// BackendAuth status values live in pkg/crux/types as part of the Crux +// BackendAuth status values live in pkg/crux as part of the Crux // runtime surface (hivecommons/hive#13536); these aliases keep pkg/agent's -// names valid. See types.BackendAuthForbidden for why a bare 403 is its own +// names valid. See crux.BackendAuthForbidden for why a bare 403 is its own // state (#6500). const ( - BackendAuthOK = types.BackendAuthOK - BackendAuthUnlicensed = types.BackendAuthUnlicensed - BackendAuthTokenExpired = types.BackendAuthTokenExpired - BackendAuthUnreachable = types.BackendAuthUnreachable - BackendAuthQuota = types.BackendAuthQuota - BackendAuthForbidden = types.BackendAuthForbidden + BackendAuthOK = crux.BackendAuthOK + BackendAuthUnlicensed = crux.BackendAuthUnlicensed + BackendAuthTokenExpired = crux.BackendAuthTokenExpired + BackendAuthUnreachable = crux.BackendAuthUnreachable + BackendAuthQuota = crux.BackendAuthQuota + BackendAuthForbidden = crux.BackendAuthForbidden ) // credentialRejectionSignals are the substrings in an auth-class pane line that @@ -57,8 +57,8 @@ func lineIndicatesCredentialRejection(lower string) bool { const backendAuthErrorLimit = 200 // BackendAuthState is the per-agent backend-auth canary (#6558); see -// types.BackendAuthState. -type BackendAuthState = types.BackendAuthState +// crux.BackendAuthState. +type BackendAuthState = crux.BackendAuthState // classifyBackendAuthStatus maps a classifyProviderError verdict onto the // coarser BackendAuth states. ok=false means the class is transient (or diff --git a/src/pkg/agent/backend_auth_test.go b/src/crux/agent/backend_auth_test.go similarity index 99% rename from src/pkg/agent/backend_auth_test.go rename to src/crux/agent/backend_auth_test.go index 3398920786..5124a4ce2d 100644 --- a/src/pkg/agent/backend_auth_test.go +++ b/src/crux/agent/backend_auth_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestClassifyBackendAuthStatus(t *testing.T) { diff --git a/src/pkg/agent/backend_coverage_test.go b/src/crux/agent/backend_coverage_test.go similarity index 99% rename from src/pkg/agent/backend_coverage_test.go rename to src/crux/agent/backend_coverage_test.go index b41f0bcfe9..4fc30fb065 100644 --- a/src/pkg/agent/backend_coverage_test.go +++ b/src/crux/agent/backend_coverage_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestAgentCopilotConfigPaths_PerUIDFirst locks down the ORDERING contract of diff --git a/src/pkg/agent/backend_launch_cmd_test.go b/src/crux/agent/backend_launch_cmd_test.go similarity index 100% rename from src/pkg/agent/backend_launch_cmd_test.go rename to src/crux/agent/backend_launch_cmd_test.go diff --git a/src/pkg/agent/backend_resume.go b/src/crux/agent/backend_resume.go similarity index 93% rename from src/pkg/agent/backend_resume.go rename to src/crux/agent/backend_resume.go index d597712807..c1dc3e11be 100644 --- a/src/pkg/agent/backend_resume.go +++ b/src/crux/agent/backend_resume.go @@ -6,12 +6,13 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" - "fmt" "io" "os" "path/filepath" "strings" "time" + + "github.com/hivecommons/hive/crux" ) // Backend-native resume handles (hivecommons/hive#9606). @@ -59,26 +60,9 @@ const ( gooseSessionIDKey = `"session.id":"` ) -// ResumeHandle names a backend-native conversation that can be resumed after -// the authoring tmux pane is gone. -type ResumeHandle struct { - // Backend is the CLI the handle belongs to ("claude", "copilot", ...). - Backend string - // SessionID is the backend's own conversation id. - SessionID string - // Transcript is the file the backend persists that conversation in, so a - // fresh session can read the reasoning even if resume itself fails. For - // goose, whose conversations live in a SQLite store rather than a file - // named by id, it is the per-launch CLI log that names the id instead. - Transcript string - // Command resumes the conversation from a shell in the agent's pane. - Command string - // LastActive is the transcript's modification time when it was captured. - LastActive time.Time - // CapturedAt is when the hive captured the handle (PR-open time); the - // staleness clock runs from here. - CapturedAt time.Time -} +// ResumeHandle is crux.ResumeHandle: a backend-native conversation that +// can be resumed after the authoring tmux pane is gone. +type ResumeHandle = crux.ResumeHandle // backendResumeLayout describes where one backend keeps its transcripts and // how to turn a path into a resume id and command. @@ -578,20 +562,3 @@ func (m *Manager) ResumeHandle(name string) (ResumeHandle, bool) { m.mu.RUnlock() return CaptureResumeHandleIn(name, uid, backend, workDir, time.Now()) } - -// String renders a handle for logs. -func (h ResumeHandle) String() string { - if h.SessionID == "" { - return "" - } - return fmt.Sprintf("%s session %s (%s)", h.Backend, h.SessionID, h.Transcript) -} - -// TranscriptExists reports whether the captured transcript is still on disk. -func (h ResumeHandle) TranscriptExists() bool { - if h.Transcript == "" { - return false - } - _, err := os.Stat(h.Transcript) - return err == nil -} diff --git a/src/pkg/agent/backend_resume_test.go b/src/crux/agent/backend_resume_test.go similarity index 99% rename from src/pkg/agent/backend_resume_test.go rename to src/crux/agent/backend_resume_test.go index 1d6a9ce853..06ae812a9d 100644 --- a/src/pkg/agent/backend_resume_test.go +++ b/src/crux/agent/backend_resume_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // Backend resume-handle capture (hivecommons/hive#9606). Each test lays out diff --git a/src/pkg/agent/blocking_prompt_test.go b/src/crux/agent/blocking_prompt_test.go similarity index 100% rename from src/pkg/agent/blocking_prompt_test.go rename to src/crux/agent/blocking_prompt_test.go diff --git a/src/pkg/agent/bob_display_name_test.go b/src/crux/agent/bob_display_name_test.go similarity index 97% rename from src/pkg/agent/bob_display_name_test.go rename to src/crux/agent/bob_display_name_test.go index 46a47ad485..c149cb8717 100644 --- a/src/pkg/agent/bob_display_name_test.go +++ b/src/crux/agent/bob_display_name_test.go @@ -3,7 +3,7 @@ package agent import ( "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestBobSessionLabelEnvPair pins #11273: bob-backed agents get an optional diff --git a/src/pkg/agent/bob_key_relaunch_loop_test.go b/src/crux/agent/bob_key_relaunch_loop_test.go similarity index 98% rename from src/pkg/agent/bob_key_relaunch_loop_test.go rename to src/crux/agent/bob_key_relaunch_loop_test.go index 850ad18056..cdf46f5ffe 100644 --- a/src/pkg/agent/bob_key_relaunch_loop_test.go +++ b/src/crux/agent/bob_key_relaunch_loop_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/internal/testutil" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" ) // These tests cover the relaunch LOOP body of RelaunchBobAgentsAwaitingKey — diff --git a/src/pkg/agent/bob_key_relaunch_test.go b/src/crux/agent/bob_key_relaunch_test.go similarity index 99% rename from src/pkg/agent/bob_key_relaunch_test.go rename to src/crux/agent/bob_key_relaunch_test.go index 6336712c24..d97af62472 100644 --- a/src/pkg/agent/bob_key_relaunch_test.go +++ b/src/crux/agent/bob_key_relaunch_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // newRelaunchManager builds a bare Manager with the given agents registered. diff --git a/src/crux/agent/bob_launch_shell_test.go b/src/crux/agent/bob_launch_shell_test.go new file mode 100644 index 0000000000..25a8c95150 --- /dev/null +++ b/src/crux/agent/bob_launch_shell_test.go @@ -0,0 +1,110 @@ +package agent + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// TestBobLaunchCmdRunsBehindEnvPrefix executes the real tmux launch shape — +// KEY='value' assignments followed by bobLaunchCmd — through bash against a +// fake bob. Before the sh -c wrap, `A='1' case ... esac` was a bash syntax +// error and bob never launched on any hive. +func TestBobLaunchCmdRunsBehindEnvPrefix(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Skip("bash not available") + } + + if _, err := os.Stat("../../../config/backends.conf"); err != nil { + t.Skip("monorepo file config/backends.conf not present (standalone crux checkout)") + } + + cases := []struct { + version string + wantArgs string + }{ + {"2.0.6", "chat --accept-license --auto-approve --trust"}, + {"1.0.6", "--accept-license --auth-method api-key --approval-mode yolo --trust"}, + {"unknown", "chat --accept-license --auto-approve --trust"}, + } + for _, tc := range cases { + t.Run(tc.version, func(t *testing.T) { + fake := filepath.Join(t.TempDir(), "bob") + script := "#!/bin/sh\n" + + "if [ \"$1\" = --version ]; then echo " + tc.version + "; echo commit: x; exit 0; fi\n" + + "echo \"ARGS=$* HIVE_AGENT=$HIVE_AGENT\"\n" + if err := os.WriteFile(fake, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + + // Exercise the contributor's actual flag helper against the same + // fake CLI and expected argv as the hub's pod-agent launcher. + conf, err := filepath.Abs("../../../config/backends.conf") + if err != nil { + t.Fatal(err) + } + contributor := exec.Command(bash, "-c", `source "$1"; eval "bob $(backend_perm_flag_shell bob)"`, "bash", conf) + contributor.Env = append(os.Environ(), "PATH="+filepath.Dir(fake)+":"+os.Getenv("PATH"), "HIVE_AGENT=quality") + out, err := contributor.CombinedOutput() + want := "ARGS=" + tc.wantArgs + " HIVE_AGENT=quality" + if err != nil || strings.TrimSpace(string(out)) != want { + t.Fatalf("contributor launch: err=%v output=%q, want %q", err, out, want) + } + + line := shellEnvVar("HIVE_AGENT", "quality") + " " + bobLaunchCmd(fake) + out, err = exec.Command(bash, "-c", line).CombinedOutput() + if err != nil { + t.Fatalf("launch line failed: %v\nline: %s\noutput: %s", err, line, out) + } + if got := strings.TrimSpace(string(out)); got != want { + t.Errorf("output = %q, want %q", got, want) + } + }) + } +} + +// Run the real contributor startup key check without starting tmux or a relay. +func TestContributorBobKeyCompatibility(t *testing.T) { + startup, err := os.ReadFile("../../../bin/contributor-agent.sh") + if os.IsNotExist(err) { + t.Skip("monorepo file bin/contributor-agent.sh not present (standalone crux checkout)") + } + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat("../../../config/backends.conf"); err != nil { + t.Skip("monorepo file config/backends.conf not present (standalone crux checkout)") + } + start := strings.Index(string(startup), "# Fail before an unattended Bob") + end := strings.Index(string(startup), "# Ensure metrics directory") + if start < 0 || end <= start { + t.Fatal("contributor Bob startup check not found") + } + for _, tc := range []struct { + name, current, legacy, want string + wantFailure bool + }{ + {"current", "current-key", "", "current-key/current-key", false}, + {"legacy", "", "legacy-key", "legacy-key/legacy-key", false}, + {"both", "current-key", "legacy-key", "current-key/legacy-key", false}, + {"missing", "", "", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + script := `set -eu; source ../../../config/backends.conf; AGENT_BACKEND=bob; BOB_API_KEY=$1; BOBSHELL_API_KEY=$2; ` + + string(startup[start:end]) + `bash -c 'printf "%s/%s" "$BOB_API_KEY" "$BOBSHELL_API_KEY"'` + out, err := exec.Command("bash", "-c", script, "bash", tc.current, tc.legacy).CombinedOutput() + if tc.wantFailure { + if err == nil || !strings.Contains(string(out), "requires BOB_API_KEY") { + t.Fatalf("missing key: err=%v output=%q", err, out) + } + return + } + if err != nil || string(out) != tc.want { + t.Fatalf("key aliases: err=%v output=%q, want %q", err, out, tc.want) + } + }) + } +} diff --git a/src/pkg/agent/bob_settings.go b/src/crux/agent/bob_settings.go similarity index 99% rename from src/pkg/agent/bob_settings.go rename to src/crux/agent/bob_settings.go index 822af0ede3..ed8d29f380 100644 --- a/src/pkg/agent/bob_settings.go +++ b/src/crux/agent/bob_settings.go @@ -6,7 +6,7 @@ import ( "os" "path/filepath" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // Unix permission bits for the agent-UID readability probe. Named rather than diff --git a/src/pkg/agent/bob_settings_test.go b/src/crux/agent/bob_settings_test.go similarity index 99% rename from src/pkg/agent/bob_settings_test.go rename to src/crux/agent/bob_settings_test.go index cad232ee6c..ae7ce0dac8 100644 --- a/src/pkg/agent/bob_settings_test.go +++ b/src/crux/agent/bob_settings_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // readBobSettings decodes the settings file written under home. diff --git a/src/pkg/agent/bob_startup_kick_test.go b/src/crux/agent/bob_startup_kick_test.go similarity index 99% rename from src/pkg/agent/bob_startup_kick_test.go rename to src/crux/agent/bob_startup_kick_test.go index e1aec77ffb..01ec351e6c 100644 --- a/src/pkg/agent/bob_startup_kick_test.go +++ b/src/crux/agent/bob_startup_kick_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestBackendDefersStartupKick pins WHICH backends get their bootstrap prompt diff --git a/src/pkg/agent/bob_test.go b/src/crux/agent/bob_test.go similarity index 99% rename from src/pkg/agent/bob_test.go rename to src/crux/agent/bob_test.go index c958078163..892d318752 100644 --- a/src/pkg/agent/bob_test.go +++ b/src/crux/agent/bob_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestWatchedHomeDirsIncludesBob pins the fix for bob's startup failure: diff --git a/src/pkg/agent/branches_coverage_test.go b/src/crux/agent/branches_coverage_test.go similarity index 99% rename from src/pkg/agent/branches_coverage_test.go rename to src/crux/agent/branches_coverage_test.go index 92db51d670..ad97677a7e 100644 --- a/src/pkg/agent/branches_coverage_test.go +++ b/src/crux/agent/branches_coverage_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // --------------------------------------------------------------------------- diff --git a/src/pkg/agent/busy_visibility.go b/src/crux/agent/busy_visibility.go similarity index 98% rename from src/pkg/agent/busy_visibility.go rename to src/crux/agent/busy_visibility.go index 560811ea27..ab02dd70be 100644 --- a/src/pkg/agent/busy_visibility.go +++ b/src/crux/agent/busy_visibility.go @@ -7,7 +7,7 @@ import ( "path/filepath" "time" - "github.com/hivecommons/hive/pkg/fleetreport" + "github.com/hivecommons/hive/crux" ) const ( @@ -183,7 +183,7 @@ func (m *Manager) markKickUndeliverableLocked(agent *AgentProcess, now time.Time return } // Self-report (#11868): the tracker applies the ≥3-consecutive threshold. - fleetreport.Observe(fleetreport.ClassKickFailedRepeatedly, agent.Name, + m.observeCondition(crux.ClassKickFailedRepeatedly, agent.Name, busyNoActivityRestartFailedMessage(agent, now, reason), "") } @@ -221,7 +221,7 @@ func (m *Manager) reportBusyNoActivityRestartFailed(name, reason string, err err if !ok { return } - fleetreport.Observe(fleetreport.ClassKickFailedRepeatedly, name, + m.observeCondition(crux.ClassKickFailedRepeatedly, name, fmt.Sprintf("kick undeliverable %d times in a row and busy-no-activity restart failed: %v (%s; %d restart(s) this episode)", kicks, err, reason, restarts), "") } @@ -262,7 +262,7 @@ func (m *Manager) markCopilotQuestionFormLocked(agent *AgentProcess, now time.Ti // dropped. Caller holds m.mu. func (m *Manager) resetBusyVisibilityLocked(agent *AgentProcess) { if agent.KicksUndeliverable > 0 || !agent.busyNoActivityRestartAt.IsZero() { - fleetreport.Clear(fleetreport.ClassKickFailedRepeatedly, agent.Name) + m.clearCondition(crux.ClassKickFailedRepeatedly, agent.Name) } agent.busyNoActivityRestartAt = time.Time{} agent.busyNoActivityRestarts = 0 diff --git a/src/pkg/agent/busy_visibility_test.go b/src/crux/agent/busy_visibility_test.go similarity index 90% rename from src/pkg/agent/busy_visibility_test.go rename to src/crux/agent/busy_visibility_test.go index e043dba30a..a593473a33 100644 --- a/src/pkg/agent/busy_visibility_test.go +++ b/src/crux/agent/busy_visibility_test.go @@ -12,8 +12,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/fleetreport" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" ) func busyVisibilityTestManager() *Manager { @@ -130,14 +130,10 @@ func writeEventsFile(t *testing.T, home, id string, mod time.Time) { } } -func kickFailedEvidence(tr *fleetreport.RuntimeTracker) []fleetreport.Evidence { - var out []fleetreport.Evidence - for _, ev := range tr.Evidence() { - if ev.Class == fleetreport.ClassKickFailedRepeatedly { - out = append(out, ev) - } - } - return out +// kickFailedEvidence is what the runtime has told its ConditionReporter +// about kick_failed_repeatedly and not cleared since. +func kickFailedEvidence(rec *conditionRecorder) []conditionObservation { + return rec.active(crux.ClassKickFailedRepeatedly) } // Decision B on #12787: kicks that fail before busyNoActivityStall has had @@ -146,10 +142,8 @@ func kickFailedEvidence(tr *fleetreport.RuntimeTracker) []fleetreport.Evidence { // not self-report. func TestKickUndeliverableBeforeStallRestartIsNotSelfReported(t *testing.T) { clock := time.Date(2026, 10, 17, 16, 25, 0, 0, time.UTC) - tr := fleetreport.NewRuntimeTracker(func() time.Time { return clock }) - fleetreport.SetDefaultTrackerForTest(t, tr) - m := busyVisibilityTestManager() + tr := recordConditions(t, m) agent := m.agents["scanner"] agent.LastTranscriptActivity = clock.Add(-config.DefaultBusyNoActivityThreshold - time.Minute) for i := 0; i < 8; i++ { @@ -178,10 +172,8 @@ func TestKickUndeliverableBeforeStallRestartIsNotSelfReported(t *testing.T) { func TestKickUndeliverableDuringLiveTurnIsNotSelfReported(t *testing.T) { clock := time.Date(2026, 10, 16, 3, 26, 26, 0, time.UTC) - tr := fleetreport.NewRuntimeTracker(func() time.Time { return clock }) - fleetreport.SetDefaultTrackerForTest(t, tr) - m := busyVisibilityTestManager() + tr := recordConditions(t, m) agent := m.agents["scanner"] // Even after a stall restart, a transcript written to seconds ago means // the pane is busy, not stuck (#12268). @@ -202,10 +194,8 @@ func TestKickUndeliverableDuringLiveTurnIsNotSelfReported(t *testing.T) { // delivered kick clears the marker and the tracker, and nothing is filed. func TestStallRestartThatRecoversClearsWithoutSelfReport(t *testing.T) { clock := time.Date(2026, 10, 17, 17, 0, 0, 0, time.UTC) - tr := fleetreport.NewRuntimeTracker(func() time.Time { return clock }) - fleetreport.SetDefaultTrackerForTest(t, tr) - m := busyVisibilityTestManager() + tr := recordConditions(t, m) agent := m.agents["scanner"] agent.LastTranscriptActivity = clock.Add(-65 * time.Minute) for i := 0; i < busyNoActivityRecoveryKicks; i++ { @@ -222,19 +212,22 @@ func TestStallRestartThatRecoversClearsWithoutSelfReport(t *testing.T) { t.Fatalf("restart reset dropped the stall-restart marker: at=%v n=%d", agent.busyNoActivityRestartAt, agent.busyNoActivityRestarts) } - // One more undeliverable kick while the new CLI boots is still below the - // tracker's streak, then the kick lands. + // One more undeliverable kick while the new CLI boots is observed (the + // host applies its streak before anything is filed), then the kick lands + // and the recovery clears it. m.markKickUndeliverableLocked(agent, clock.Add(time.Minute), "pane not at a ready input prompt") + if obs := tr.observations(crux.ClassKickFailedRepeatedly); len(obs) != 1 { + t.Fatalf("post-restart kick failure observations = %d, want 1", len(obs)) + } m.recordDeliveredKickLocked(agent, "go", "governor") if !agent.busyNoActivityRestartAt.IsZero() || agent.busyNoActivityRestarts != 0 { t.Fatalf("delivered kick did not clear the stall-restart marker: %+v", agent) } if evs := kickFailedEvidence(tr); len(evs) != 0 { - t.Fatalf("recovered stall was self-reported: %q", evs[0].Message) + t.Fatalf("recovered stall left kick_failed_repeatedly open: %q", evs[0].Message) } - if cleared := tr.TakeCleared(); len(cleared) != 0 { - // Nothing qualified, so nothing should surface as a clear either. - t.Fatalf("recovered stall produced %d cleared conditions, want 0", len(cleared)) + if cleared := tr.clears(crux.ClassKickFailedRepeatedly); len(cleared) != 1 { + t.Fatalf("recovered stall produced %d cleared conditions, want 1", len(cleared)) } } @@ -242,10 +235,8 @@ func TestStallRestartThatRecoversClearsWithoutSelfReport(t *testing.T) { // exactly one kick_failed_repeatedly report, saying the restart did not help. func TestStallRestartThatDoesNotRecoverSelfReportsOnce(t *testing.T) { clock := time.Date(2026, 10, 17, 17, 0, 0, 0, time.UTC) - tr := fleetreport.NewRuntimeTracker(func() time.Time { return clock }) - fleetreport.SetDefaultTrackerForTest(t, tr) - m := busyVisibilityTestManager() + tr := recordConditions(t, m) agent := m.agents["scanner"] agent.LastTranscriptActivity = clock.Add(-65 * time.Minute) for i := 0; i < busyNoActivityRecoveryKicks; i++ { @@ -260,11 +251,13 @@ func TestStallRestartThatDoesNotRecoverSelfReportsOnce(t *testing.T) { for i := 0; i < 4; i++ { m.markKickUndeliverableLocked(agent, clock, "pane not at a ready input prompt") } + // Every failing kick after the restart is observed; the host's streak + // turns them into one report. evs := kickFailedEvidence(tr) - if len(evs) != 1 { - t.Fatalf("kick_failed_repeatedly evidence = %d, want exactly 1", len(evs)) + if len(evs) != 4 { + t.Fatalf("kick_failed_repeatedly observations = %d, want 4", len(evs)) } - msg := evs[0].Message + msg := evs[len(evs)-1].Message for _, want := range []string{ "after busy-no-activity restart did not recover the session", "kick undeliverable 4 times in a row", @@ -282,8 +275,8 @@ func TestStallRestartThatDoesNotRecoverSelfReportsOnce(t *testing.T) { if evs := kickFailedEvidence(tr); len(evs) != 0 { t.Fatalf("evidence still present after recovery: %q", evs[0].Message) } - if cleared := tr.TakeCleared(); len(cleared) != 1 || cleared[0].Class != fleetreport.ClassKickFailedRepeatedly { - t.Fatalf("cleared conditions = %+v, want one kick_failed_repeatedly clear", cleared) + if cleared := tr.clears(crux.ClassKickFailedRepeatedly); len(cleared) != 1 || cleared[0].Key != "scanner" { + t.Fatalf("cleared conditions = %+v, want one kick_failed_repeatedly clear for scanner", cleared) } } @@ -291,33 +284,27 @@ func TestStallRestartThatDoesNotRecoverSelfReportsOnce(t *testing.T) { // kicks to pile up on a session that was never restarted. func TestStallRestartErrorIsSelfReported(t *testing.T) { clock := time.Date(2026, 10, 17, 17, 0, 0, 0, time.UTC) - tr := fleetreport.NewRuntimeTracker(func() time.Time { return clock }) - fleetreport.SetDefaultTrackerForTest(t, tr) - m := busyVisibilityTestManager() + tr := recordConditions(t, m) agent := m.agents["scanner"] agent.KicksUndeliverable = busyNoActivityRecoveryKicks m.markBusyNoActivityRestartLocked(agent, clock) reason := busyNoActivityCondition + ": pane shows Working with no transcript activity for 1h5m; 6 kicks undeliverable" m.reportBusyNoActivityRestartFailed("scanner", reason, errors.New("tmux kill-session: no server running")) - // The tracker still applies its streak: one observation is not yet - // evidence, two more failing kicks finish it and the count carries all - // three. - if evs := kickFailedEvidence(tr); len(evs) != 0 { - t.Fatalf("single restart error already qualified: %q", evs[0].Message) + // The restart error is observed at once; two more failing kicks follow + // it so the host's streak sees all three. + if evs := kickFailedEvidence(tr); len(evs) != 1 { + t.Fatalf("restart error observations = %d, want 1", len(evs)) } for i := 0; i < 2; i++ { m.markKickUndeliverableLocked(agent, clock, "pane not at a ready input prompt") } evs := kickFailedEvidence(tr) - if len(evs) != 1 { - t.Fatalf("kick_failed_repeatedly evidence = %d, want exactly 1", len(evs)) - } - if evs[0].Count != 3 { - t.Fatalf("count = %d, want 3 (restart error + 2 kicks)", evs[0].Count) + if len(evs) != 3 { + t.Fatalf("kick_failed_repeatedly observations = %d, want 3 (restart error + 2 kicks)", len(evs)) } - if !strings.Contains(evs[0].Message, "after busy-no-activity restart did not recover the session") { - t.Fatalf("message = %q, want the post-restart wording", evs[0].Message) + if !strings.Contains(evs[2].Message, "after busy-no-activity restart did not recover the session") { + t.Fatalf("message = %q, want the post-restart wording", evs[2].Message) } m.reportBusyNoActivityRestartFailed("no-such-agent", reason, errors.New("x")) } diff --git a/src/pkg/agent/capabilities.go b/src/crux/agent/capabilities.go similarity index 100% rename from src/pkg/agent/capabilities.go rename to src/crux/agent/capabilities.go diff --git a/src/pkg/agent/caveman_install_test.go b/src/crux/agent/caveman_install_test.go similarity index 100% rename from src/pkg/agent/caveman_install_test.go rename to src/crux/agent/caveman_install_test.go diff --git a/src/pkg/agent/caveman_pin_test.go b/src/crux/agent/caveman_pin_test.go similarity index 100% rename from src/pkg/agent/caveman_pin_test.go rename to src/crux/agent/caveman_pin_test.go diff --git a/src/crux/agent/ci_poll_log_read_test.go b/src/crux/agent/ci_poll_log_read_test.go new file mode 100644 index 0000000000..c1a6263f7c --- /dev/null +++ b/src/crux/agent/ci_poll_log_read_test.go @@ -0,0 +1,50 @@ +package agent + +import ( + "strings" + "testing" + "time" +) + +func TestCIPollCommandsLogReads(t *testing.T) { + for _, tt := range []struct { + command string + want int + }{ + {"gh run view 123 --log", 0}, + {"hive-gh run view 123 --job 456 --log-failed", 0}, + {"gh run view --log=true 123", 0}, + {"gh run view 123 --log-failed | tail -100", 0}, + {"gh run view 123", 1}, + {"gh run view 123 --json status,conclusion", 1}, + {"gh run view 123 --log=false", 1}, + {"gh run view 123 --log-failed=false", 1}, + {"gh pr checks 123", 1}, + {"gh run view 123; gh run view 456 --log", 1}, + } { + t.Run(tt.command, func(t *testing.T) { + if got := countCIPollCommands(tt.command); got != tt.want { + t.Fatalf("count = %d, want %d", got, tt.want) + } + }) + } +} + +func TestNudgeIfPollingCILogDiagnosis(t *testing.T) { + m, a := ciPollManager(t, ciPollIdlePane, 11*time.Minute) + tr := recordConditions(t, m) + m.nudgeIfPollingCI(a, "", ciPollIdlePane) + logs := strings.Repeat("gh run view 123 --log-failed\n", ciPollNudgeThreshold+1) + m.nudgeIfPollingCI(a, logs, ciPollIdlePane) + if a.ciPollNudgeSent || a.AwaitingCI || a.CIPollNudges != 0 { + t.Fatalf("log diagnosis treated as polling: nudged=%v awaiting=%v count=%d", a.ciPollNudgeSent, a.AwaitingCI, a.CIPollNudges) + } + if ev := ciPollEvidence(tr); len(ev) != 0 { + t.Fatalf("log diagnosis produced fleet evidence: %#v", ev) + } + // Log reads must not hide actual repeated status polling in the same kick. + m.nudgeIfPollingCI(a, logs+strings.Repeat("gh pr checks 123\n", ciPollNudgeThreshold+1), ciPollIdlePane) + if !a.ciPollNudgeSent || len(ciPollEvidence(tr)) != 1 { + t.Fatal("status polling mixed with log reads must still nudge and report") + } +} diff --git a/src/pkg/agent/ci_poll_pr_view_test.go b/src/crux/agent/ci_poll_pr_view_test.go similarity index 100% rename from src/pkg/agent/ci_poll_pr_view_test.go rename to src/crux/agent/ci_poll_pr_view_test.go diff --git a/src/pkg/agent/ci_poll_prohibition_test.go b/src/crux/agent/ci_poll_prohibition_test.go similarity index 100% rename from src/pkg/agent/ci_poll_prohibition_test.go rename to src/crux/agent/ci_poll_prohibition_test.go diff --git a/src/crux/agent/ci_poll_sweep_test.go b/src/crux/agent/ci_poll_sweep_test.go new file mode 100644 index 0000000000..9bc6e31012 --- /dev/null +++ b/src/crux/agent/ci_poll_sweep_test.go @@ -0,0 +1,206 @@ +package agent + +import ( + "fmt" + "slices" + "strings" + "testing" + "time" +) + +// ci_poll_sweep_test.go pins #14016: the scanner's merge sweep and CI-repair +// pass read `gh pr checks` once for each of many PRs. That is a sweep, not a +// wait on one PR, so only command lines run more than once count toward the +// nudge threshold and the fleet self-report, and the wall-clock trigger +// leaves a sweep alone (#14066). + +func TestCountRepeatedCIPollCommands(t *testing.T) { + tests := []struct { + name string + pane string + want int + }{ + { + name: "distinct PRs read once each", + pane: "gh pr checks 1\ngh pr checks 2\ngh pr checks 3\ngh pr checks 4", + want: 0, + }, + { + name: "same PR polled counts every read", + pane: "gh pr checks 7\ngh pr checks 8\ngh pr checks 7\ngh pr checks 7", + want: 3, + }, + { + name: "pane padding and box rule do not split one target", + pane: "● gh run watch 42 ┃\n gh run watch 42\n", + want: 2, + }, + { + name: "quoted prohibition is not a read", + pane: "no `gh pr checks` loops\ngh pr checks 5", + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := countRepeatedCIPollCommands(tt.pane, nil); got != tt.want { + t.Errorf("countRepeatedCIPollCommands(%q) = %d, want %d", tt.pane, got, tt.want) + } + }) + } +} + +func TestCountRepeatedCIPollCommandsSinceBaseline(t *testing.T) { + const poll = "gh pr checks 7\n" + for _, tt := range []struct { + name string + baseline string + pane string + want int + }{ + {name: "one old and one new read", baseline: poll, pane: poll + poll, want: 0}, + {name: "old repeated reads and one new read", baseline: poll + poll, pane: poll + poll + poll, want: 0}, + {name: "two new reads after old singleton", baseline: poll, pane: strings.Repeat(poll, 3), want: 2}, + {name: "two new reads after old repeats", baseline: poll + poll, pane: strings.Repeat(poll, 4), want: 2}, + {name: "unchanged scrollback", baseline: poll + poll, pane: poll + poll, want: 0}, + {name: "old target scrolled out, new target repeats", baseline: poll + poll, pane: "gh pr checks 8\ngh pr checks 8\n", want: 2}, + {name: "old reads scrolled out, same target read again", baseline: poll + poll + "gh pr checks 8\n", pane: poll + poll, want: 2}, + {name: "partial scroll-out keeps only new reads", baseline: "gh pr checks 1\ngh pr checks 2\ngh pr checks 3\n", pane: "gh pr checks 3\ngh pr checks 4\ngh pr checks 4\n", want: 2}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := countRepeatedCIPollCommands(tt.pane, ciPollCommands(tt.baseline)); got != tt.want { + t.Errorf("repeated reads since baseline = %d, want %d", got, tt.want) + } + }) + } +} + +// A scanner revisits the same PRs on successive kicks. Each sweep reads each +// PR only once; historical reads must not cause a count nudge or fleet report. +func TestNudgeIfPollingCISuccessiveSweeps(t *testing.T) { + var sweep string + for pr := range ciPollNudgeThreshold + 3 { + sweep += fmt.Sprintf("gh pr checks %d --repo acme/widgets\n", 100+pr) + } + for _, tt := range []struct { + name string + kickAge time.Duration + wantNudged bool + }{ + {name: "young kick", kickAge: 2 * time.Minute}, + {name: "past wall clock", kickAge: 11 * time.Minute}, + } { + t.Run(tt.name, func(t *testing.T) { + m, a := ciPollManager(t, ciPollIdlePane, tt.kickAge) + tr := recordConditions(t, m) + scrollback := sweep + for kick := range 3 { + m.mu.Lock() + m.recordDeliveredKickLocked(a, "inspect PRs", "test") + *a.LastKick = time.Now().Add(-tt.kickAge) + m.mu.Unlock() + + m.nudgeIfPollingCI(a, scrollback, ciPollIdlePane) + scrollback += sweep + m.nudgeIfPollingCI(a, scrollback, ciPollIdlePane) + + if a.ciPollNudgeSent != tt.wantNudged || a.AwaitingCI != tt.wantNudged { + t.Fatalf("kick %d: nudged=%v awaiting=%v, want %v", kick, a.ciPollNudgeSent, a.AwaitingCI, tt.wantNudged) + } + if ev := ciPollEvidence(tr); len(ev) != 0 { + t.Fatalf("kick %d: sweep produced CI-poll evidence: %#v", kick, ev) + } + } + }) + } +} + +func TestNudgeIfPollingCISweepIsNotPolling(t *testing.T) { + sweep := "" + for pr := range ciPollNudgeThreshold + 3 { + sweep += fmt.Sprintf("gh pr checks %d --repo acme/widgets\n", 100+pr) + } + for _, tt := range []struct { + name string + kickAge time.Duration + wantNudged bool + }{ + {name: "sweep, kick young: no nudge", kickAge: 2 * time.Minute, wantNudged: false}, + {name: "sweep past wall clock: no nudge (#14066)", kickAge: 11 * time.Minute, wantNudged: false}, + } { + t.Run(tt.name, func(t *testing.T) { + m, a := ciPollManager(t, ciPollIdlePane, tt.kickAge) + tr := recordConditions(t, m) + + m.nudgeIfPollingCI(a, "", ciPollIdlePane) + m.nudgeIfPollingCI(a, sweep, ciPollIdlePane) + + if a.AwaitingCI != tt.wantNudged { + t.Errorf("AwaitingCI = %v, want %v", a.AwaitingCI, tt.wantNudged) + } + + if a.ciPollNudgeSent != tt.wantNudged { + t.Errorf("ciPollNudgeSent = %v, want %v", a.ciPollNudgeSent, tt.wantNudged) + } + if ev := ciPollEvidence(tr); len(ev) != 0 { + t.Errorf("ci_poll_nudge evidence = %d for a sweep, want 0: %#v", len(ev), ev) + } + }) + } +} + +// TestNudgeIfPollingCIWallClockSweepWithRepeat pins #14066's boundary: past +// the wall clock a sweep is left alone, but one PR read twice inside it is a +// wait on that PR, so the wall-clock nudge still fires. +func TestNudgeIfPollingCIWallClockSweepWithRepeat(t *testing.T) { + sweep := "gh pr checks 100\ngh pr checks 101\ngh pr checks 102\n" + for _, tt := range []struct { + name string + pane string + wantNudged bool + }{ + {name: "distinct reads", pane: sweep, wantNudged: false}, + {name: "one target re-read", pane: sweep + "gh pr checks 101\n", wantNudged: true}, + } { + t.Run(tt.name, func(t *testing.T) { + m, a := ciPollManager(t, ciPollIdlePane, 11*time.Minute) + + m.nudgeIfPollingCI(a, "", ciPollIdlePane) + m.nudgeIfPollingCI(a, tt.pane, ciPollIdlePane) + + if a.ciPollNudgeSent != tt.wantNudged || a.AwaitingCI != tt.wantNudged { + t.Errorf("nudged=%v awaiting=%v, want %v", a.ciPollNudgeSent, a.AwaitingCI, tt.wantNudged) + } + }) + } +} + +func TestCIPollSweep(t *testing.T) { + for _, tt := range []struct { + polls, repeats int + want bool + }{ + {polls: 0, repeats: 0, want: false}, + {polls: 1, repeats: 0, want: false}, + {polls: 2, repeats: 0, want: true}, + {polls: 3, repeats: 2, want: false}, + } { + if got := ciPollSweep(tt.polls, tt.repeats); got != tt.want { + t.Errorf("ciPollSweep(%d, %d) = %v, want %v", tt.polls, tt.repeats, got, tt.want) + } + } +} + +func TestRepeatedCIPollCommandLines(t *testing.T) { + pane := "hive-gh pr checks 7\ngh pr checks 8\ngh pr checks 9\nhive-gh pr checks 7\ngh pr checks 9\nhive-gh pr checks 7\ngh pr view 1" + got := repeatedCIPollCommandLines(pane, nil) + // ciPollCommands captures from the regex match, so the hive-gh wrapper + // prefix is not part of the recorded command line. + want := []string{"3× gh pr checks 7", "2× gh pr checks 9"} + if !slices.Equal(got, want) { + t.Fatalf("got %q, want %q", got, want) + } + if got := repeatedCIPollCommandLines("gh pr checks 1\ngh pr checks 2", nil); len(got) != 0 { + t.Fatalf("sweep listed as repeated: %q", got) + } +} diff --git a/src/pkg/agent/ci_poll_wall_clock_test.go b/src/crux/agent/ci_poll_wall_clock_test.go similarity index 74% rename from src/pkg/agent/ci_poll_wall_clock_test.go rename to src/crux/agent/ci_poll_wall_clock_test.go index 81c750b5e9..c37c0e9270 100644 --- a/src/pkg/agent/ci_poll_wall_clock_test.go +++ b/src/crux/agent/ci_poll_wall_clock_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/fleetreport" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" ) // ci_poll_wall_clock_test.go covers the second half of #9673's harness-guard @@ -119,21 +119,28 @@ func TestNudgeIfPollingCIWallClockSecondNudgeSuppressed(t *testing.T) { } } -// TestNudgeIfPollingCISkipsReviewer pins #13860: a reviewer reading check -// status on the PRs it reviews must not be nudged or reported. -func TestNudgeIfPollingCISkipsReviewer(t *testing.T) { +// TestNudgeIfPollingCISkipsMonitoringLanes pins #13860, #13898 and #14188: +// reviewers, CI maintainers and quality agents read check status as evidence +// for the PRs and repositories they monitor, so that work must not be mistaken +// for polling their own PR. +func TestNudgeIfPollingCISkipsMonitoringLanes(t *testing.T) { for _, tt := range []struct { name string agentName string role string wantNudged bool }{ - {name: "role", agentName: "scanner", role: ReviewerRole, wantNudged: false}, - {name: "name only", agentName: "code-reviewer-sonnet-1", role: "", wantNudged: false}, + {name: "reviewer role", agentName: "scanner", role: ReviewerRole, wantNudged: false}, + {name: "reviewer name only", agentName: "code-reviewer-sonnet-1", role: "", wantNudged: false}, + {name: "CI maintainer role", agentName: "worker", role: "ci-maintainer", wantNudged: false}, + {name: "CI maintainer name only", agentName: "ci-maintainer", role: "", wantNudged: false}, + {name: "quality role", agentName: "worker", role: "quality", wantNudged: false}, + {name: "quality name only", agentName: "quality", role: "", wantNudged: false}, {name: "developer", agentName: "scanner", role: "developer", wantNudged: true}, } { t.Run(tt.name, func(t *testing.T) { m, a := ciPollManager(t, ciPollIdlePane, 2*time.Minute) + tr := recordConditions(t, m) a.Name = tt.agentName a.Config.Role = tt.role sent := 0 @@ -141,8 +148,8 @@ func TestNudgeIfPollingCISkipsReviewer(t *testing.T) { m.nudgeIfPollingCI(a, "", ciPollIdlePane) scrollback := "" - for i := 0; i < ciPollNudgeThreshold+2; i++ { - scrollback += "gh pr checks 123\n" + for range ciPollNudgeThreshold + 2 { + scrollback += "gh run list --repo acme/widgets\n" } m.nudgeIfPollingCI(a, scrollback, ciPollIdlePane) @@ -155,8 +162,13 @@ func TestNudgeIfPollingCISkipsReviewer(t *testing.T) { if (a.CIPollNudges > 0) != tt.wantNudged { t.Errorf("CIPollNudges = %d, wantNudged %v", a.CIPollNudges, tt.wantNudged) } - if !tt.wantNudged && sent != 0 { - t.Errorf("sent %d literals to reviewer, want 0", sent) + if !tt.wantNudged { + if sent != 0 { + t.Errorf("sent %d literals to monitoring lane, want 0", sent) + } + if ev := ciPollEvidence(tr); len(ev) != 0 { + t.Errorf("ci_poll_nudge evidence = %d for monitoring lane, want 0: %#v", len(ev), ev) + } } }) } @@ -171,15 +183,17 @@ func TestNudgeIfPollingCISelfReportGate(t *testing.T) { kickAge time.Duration polls int wantEvidence int + sweepReads int }{ {name: "one poll past wall clock: nudged, not reported", kickAge: 11 * time.Minute, polls: 1, wantEvidence: 0}, + {name: "two polls past wall clock: nudged, not reported (#14327)", kickAge: 11 * time.Minute, polls: 2, wantEvidence: 0}, {name: "repeated polls past wall clock: nudged and reported", kickAge: 11 * time.Minute, polls: ciPollReportMinPolls, wantEvidence: 1}, + {name: "sweep plus repeated target", kickAge: 11 * time.Minute, polls: ciPollReportMinPolls, sweepReads: 3, wantEvidence: 1}, {name: "count threshold crossed: nudged and reported", kickAge: 2 * time.Minute, polls: ciPollNudgeThreshold + 1, wantEvidence: 1}, } { t.Run(tt.name, func(t *testing.T) { - tr := fleetreport.NewRuntimeTracker(nil) - fleetreport.SetDefaultTrackerForTest(t, tr) m, a := ciPollManager(t, ciPollIdlePane, tt.kickAge) + tr := recordConditions(t, m) sent := 0 termSeams(m).sendLiteral = func(*AgentProcess, string) { sent++ } @@ -188,6 +202,9 @@ func TestNudgeIfPollingCISelfReportGate(t *testing.T) { for i := 0; i < tt.polls; i++ { scrollback += "gh pr checks 123\n" } + for i := 0; i < tt.sweepReads; i++ { + scrollback += fmt.Sprintf("gh pr checks %d\n", 200+i) + } m.nudgeIfPollingCI(a, scrollback, ciPollIdlePane) if !a.ciPollNudgeSent || sent != 1 { @@ -198,7 +215,10 @@ func TestNudgeIfPollingCISelfReportGate(t *testing.T) { t.Fatalf("ci_poll_nudge evidence = %d, want %d: %#v", len(ev), tt.wantEvidence, ev) } if tt.wantEvidence == 1 { - for _, want := range []string{"agent=scanner", fmt.Sprintf("polls_since_kick=%d", tt.polls), fmt.Sprintf("report_min_polls=%d", ciPollReportMinPolls)} { + if strings.Contains(ev[0].Message, "own PR") || !strings.Contains(ev[0].Message, fmt.Sprintf("%d repeated of %d", tt.polls, tt.polls+tt.sweepReads)) { + t.Errorf("report must describe observed commands without inferring ownership: %s", ev[0].Message) + } + for _, want := range []string{"agent=scanner", fmt.Sprintf("polls_since_kick=%d", tt.polls+tt.sweepReads), fmt.Sprintf("report_min_polls=%d", ciPollReportMinPolls)} { if !strings.Contains(ev[0].Diagnostics, want) { t.Errorf("diagnostics missing %q:\n%s", want, ev[0].Diagnostics) } @@ -208,12 +228,6 @@ func TestNudgeIfPollingCISelfReportGate(t *testing.T) { } } -func ciPollEvidence(tr *fleetreport.RuntimeTracker) []fleetreport.Evidence { - var out []fleetreport.Evidence - for _, e := range tr.Evidence() { - if e.Class == fleetreport.ClassCIPollNudge { - out = append(out, e) - } - } - return out +func ciPollEvidence(rec *conditionRecorder) []conditionObservation { + return rec.observations(crux.ClassCIPollNudge) } diff --git a/src/pkg/agent/claude_credential_paths_test.go b/src/crux/agent/claude_credential_paths_test.go similarity index 99% rename from src/pkg/agent/claude_credential_paths_test.go rename to src/crux/agent/claude_credential_paths_test.go index e9261a0395..e871e22323 100644 --- a/src/pkg/agent/claude_credential_paths_test.go +++ b/src/crux/agent/claude_credential_paths_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // claude_credential_paths_test.go covers ClaudeCredentialCandidatePaths, the diff --git a/src/pkg/agent/claude_launch_test.go b/src/crux/agent/claude_launch_test.go similarity index 99% rename from src/pkg/agent/claude_launch_test.go rename to src/crux/agent/claude_launch_test.go index fdf8432eba..e387d569d7 100644 --- a/src/pkg/agent/claude_launch_test.go +++ b/src/crux/agent/claude_launch_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // Launch-command tests for the per-agent reasoning effort on the claude diff --git a/src/pkg/agent/claude_oauth_env_test.go b/src/crux/agent/claude_oauth_env_test.go similarity index 99% rename from src/pkg/agent/claude_oauth_env_test.go rename to src/crux/agent/claude_oauth_env_test.go index 0f55e25c25..7f20b174a8 100644 --- a/src/pkg/agent/claude_oauth_env_test.go +++ b/src/crux/agent/claude_oauth_env_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // stageSharedClaudeCredential points the shared credential path at a temp file diff --git a/src/pkg/agent/claude_remote_control.go b/src/crux/agent/claude_remote_control.go similarity index 85% rename from src/pkg/agent/claude_remote_control.go rename to src/crux/agent/claude_remote_control.go index 1c5536fca6..d383a8e836 100644 --- a/src/pkg/agent/claude_remote_control.go +++ b/src/crux/agent/claude_remote_control.go @@ -21,12 +21,22 @@ const remoteControlUsedKey = "hasUsedRemoteControl" // claudeRemoteControlSeed returns the key hive pins into the shared Claude // user settings for interactive claude-backend agents. Kept minimal on // purpose: the shared /data/home/.claude/settings.json also carries -// fleet-critical keys hive does not own (skipDangerousModePermissionPrompt, -// enabledPlugins, ...), and the seed must never be a reason to touch them. +// fleet-critical keys hive does not own (enabledPlugins, ...), and the seed +// must never be a reason to touch them. func claudeRemoteControlSeed() map[string]any { return map[string]any{remoteControlSettingKey: false} } +// claudeBypassConsentSeed returns the key that suppresses Claude Code's +// "Bypass Permissions mode" consent screen (see inferenceSettingsSeed for how +// it was verified). Interactive claude agents launch with +// --dangerously-skip-permissions too, and nothing else provisions this key in +// the shared settings, so without it every launch stops on a menu whose +// default answer is "No, exit" (#14307). +func claudeBypassConsentSeed() map[string]any { + return map[string]any{"skipDangerousModePermissionPrompt": true} +} + // sharedClaudeSettingsFile is the userSettings file every interactive // claude-backend agent reads: the per-agent homes symlink ~/.claude to // sharedAgentHome/.claude (interactiveHomeBridgeDirs), so one file governs @@ -39,7 +49,8 @@ func sharedClaudeSettingsFile() string { // that Remote Control does not auto-start for the fleet. Called on EVERY // launch so a relaunch can never fall back to the server-side rollout default, // and so an out-of-band deletion of the key self-heals on the next launch — -// the same posture as ensureBobAuthSettings for bob's shared settings. +// the same posture as ensureBobAuthSettings for bob's shared settings. The +// bypass-consent suppressor (claudeBypassConsentSeed) is seeded the same way. // // Semantics: // - key absent -> written as false (the pin; this is the whole fix) @@ -63,6 +74,7 @@ func (m *Manager) ensureClaudeRemoteControlDefault(agent *AgentProcess) { return } m.seedJSONFile(agent.Name, path, claudeRemoteControlSeed()) + m.seedJSONFile(agent.Name, path, claudeBypassConsentSeed()) if enabled, explicit := readRemoteControlSetting(path); explicit && enabled { // Receipt, not an error: the operator opted in and hive honored it. m.logger.Info("claude remote control left ON by explicit operator setting", diff --git a/src/pkg/agent/claude_remote_control_test.go b/src/crux/agent/claude_remote_control_test.go similarity index 86% rename from src/pkg/agent/claude_remote_control_test.go rename to src/crux/agent/claude_remote_control_test.go index 56090dc698..366ff7ba54 100644 --- a/src/pkg/agent/claude_remote_control_test.go +++ b/src/crux/agent/claude_remote_control_test.go @@ -14,7 +14,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func readSettingsMap(t *testing.T, path string) map[string]any { @@ -99,6 +99,28 @@ func TestEnsureClaudeRemoteControlDefault_SeedsFalse(t *testing.T) { assertRemoteControlOff(t, path) } +// TestEnsureClaudeRemoteControlDefault_SeedsBypassConsentSuppressor is the +// #14307 guard: interactive claude agents launch with +// --dangerously-skip-permissions, so the shared settings must carry the key +// that keeps Claude Code's "Bypass Permissions mode" screen from appearing. +func TestEnsureClaudeRemoteControlDefault_SeedsBypassConsentSuppressor(t *testing.T) { + path := withSharedClaudeDir(t) + if err := os.WriteFile(path, []byte(`{"theme":"dark"}`), 0o666); err != nil { + t.Fatal(err) + } + + quietManager().ensureClaudeRemoteControlDefault(rcTestAgent()) + + parsed := readSettingsMap(t, path) + if parsed["skipDangerousModePermissionPrompt"] != true { + t.Fatalf("skipDangerousModePermissionPrompt = %v, want true", parsed["skipDangerousModePermissionPrompt"]) + } + if parsed["theme"] != "dark" { + t.Fatalf("sibling key clobbered: theme = %v", parsed["theme"]) + } + assertRemoteControlOff(t, path) +} + func TestEnsureClaudeRemoteControlDefault_PreservesOperatorTrue(t *testing.T) { path := withSharedClaudeDir(t) if err := os.WriteFile(path, []byte(`{"remoteControlAtStartup":true,"theme":"dark"}`), 0o666); err != nil { diff --git a/src/pkg/agent/claude_session_adopt.go b/src/crux/agent/claude_session_adopt.go similarity index 100% rename from src/pkg/agent/claude_session_adopt.go rename to src/crux/agent/claude_session_adopt.go diff --git a/src/pkg/agent/claude_session_adopt_fallback_test.go b/src/crux/agent/claude_session_adopt_fallback_test.go similarity index 100% rename from src/pkg/agent/claude_session_adopt_fallback_test.go rename to src/crux/agent/claude_session_adopt_fallback_test.go diff --git a/src/pkg/agent/claude_session_adopt_test.go b/src/crux/agent/claude_session_adopt_test.go similarity index 99% rename from src/pkg/agent/claude_session_adopt_test.go rename to src/crux/agent/claude_session_adopt_test.go index 816d337e21..1bd3d89481 100644 --- a/src/pkg/agent/claude_session_adopt_test.go +++ b/src/crux/agent/claude_session_adopt_test.go @@ -9,7 +9,7 @@ import ( "sync" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // fakeAgentUserRunner stands in for su-exec. Unit tests cannot create files diff --git a/src/pkg/agent/claude_session_json_helpers_test.go b/src/crux/agent/claude_session_json_helpers_test.go similarity index 100% rename from src/pkg/agent/claude_session_json_helpers_test.go rename to src/crux/agent/claude_session_json_helpers_test.go diff --git a/src/pkg/agent/claude_session_state.go b/src/crux/agent/claude_session_state.go similarity index 100% rename from src/pkg/agent/claude_session_state.go rename to src/crux/agent/claude_session_state.go diff --git a/src/pkg/agent/codex_home_nfs_safety_test.go b/src/crux/agent/codex_home_nfs_safety_test.go similarity index 100% rename from src/pkg/agent/codex_home_nfs_safety_test.go rename to src/crux/agent/codex_home_nfs_safety_test.go diff --git a/src/pkg/agent/codex_kick_regression_test.go b/src/crux/agent/codex_kick_regression_test.go similarity index 100% rename from src/pkg/agent/codex_kick_regression_test.go rename to src/crux/agent/codex_kick_regression_test.go diff --git a/src/pkg/agent/codex_launch_test.go b/src/crux/agent/codex_launch_test.go similarity index 98% rename from src/pkg/agent/codex_launch_test.go rename to src/crux/agent/codex_launch_test.go index b4101921fe..db33707db0 100644 --- a/src/pkg/agent/codex_launch_test.go +++ b/src/crux/agent/codex_launch_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // codexLaunchCmd builds the codex launch command the way Start does for an diff --git a/src/pkg/agent/codex_readiness_test.go b/src/crux/agent/codex_readiness_test.go similarity index 100% rename from src/pkg/agent/codex_readiness_test.go rename to src/crux/agent/codex_readiness_test.go diff --git a/src/crux/agent/conditions.go b/src/crux/agent/conditions.go new file mode 100644 index 0000000000..50cb056764 --- /dev/null +++ b/src/crux/agent/conditions.go @@ -0,0 +1,40 @@ +package agent + +import ( + "github.com/hivecommons/hive/crux" +) + +// SetConditionReporter installs the host's sink for the runtime's +// self-reported error conditions (restart storms, repeated kick failures, +// stalls, data-disk pressure, token-mint failures, CI-poll nudges). Hive +// wires pkg/fleetreport's runtime tracker; a standalone runtime may leave +// it unset, in which case observations are dropped. Safe to call +// concurrently with the runtime; the reporter is read on paths that hold +// m.mu, so it is stored atomically rather than under the lock. +func (m *Manager) SetConditionReporter(r crux.ConditionReporter) { + if r == nil { + m.conditionReporter.Store(nil) + return + } + m.conditionReporter.Store(&r) +} + +// observeCondition forwards an observation to the installed reporter with +// pane-derived text redacted first, so a host sink never sees a GitHub token +// or API key that scrolled through an agent pane. +func (m *Manager) observeCondition(class, key, message, diagnostics string) { + r := m.conditionReporter.Load() + if r == nil || *r == nil { + return + } + (*r).Observe(class, key, RedactPaneTokens(message), RedactPaneTokens(diagnostics)) +} + +// clearCondition forwards a recovery to the installed reporter. +func (m *Manager) clearCondition(class, key string) { + r := m.conditionReporter.Load() + if r == nil || *r == nil { + return + } + (*r).Clear(class, key) +} diff --git a/src/crux/agent/conditions_test.go b/src/crux/agent/conditions_test.go new file mode 100644 index 0000000000..952484a493 --- /dev/null +++ b/src/crux/agent/conditions_test.go @@ -0,0 +1,125 @@ +package agent + +import ( + "strings" + "sync" + "testing" +) + +// conditionObservation is one Observe call the runtime forwarded to its +// ConditionReporter. +type conditionObservation struct { + Class, Key, Message, Diagnostics string +} + +// conditionRecorder is the test ConditionReporter: it keeps every Observe +// and Clear in order so a test can assert exactly what the runtime emitted. +// Threshold and streak semantics belong to the host (pkg/fleetreport) and +// are tested there; here an observation is an observation. +type conditionRecorder struct { + mu sync.Mutex + observed []conditionObservation + cleared []conditionObservation +} + +func (r *conditionRecorder) Observe(class, key, message, diagnostics string) { + r.mu.Lock() + defer r.mu.Unlock() + r.observed = append(r.observed, conditionObservation{Class: class, Key: key, Message: message, Diagnostics: diagnostics}) +} + +func (r *conditionRecorder) Clear(class, key string) { + r.mu.Lock() + defer r.mu.Unlock() + r.cleared = append(r.cleared, conditionObservation{Class: class, Key: key}) +} + +// observations returns every observation of class, in order. +func (r *conditionRecorder) observations(class string) []conditionObservation { + r.mu.Lock() + defer r.mu.Unlock() + var out []conditionObservation + for _, o := range r.observed { + if o.Class == class { + out = append(out, o) + } + } + return out +} + +// active returns the observations of class whose (class, key) has not been +// cleared since — the recorder's view of what a host would still hold open. +func (r *conditionRecorder) active(class string) []conditionObservation { + r.mu.Lock() + defer r.mu.Unlock() + open := map[string]bool{} + var out []conditionObservation + for _, o := range r.observed { + if o.Class == class { + open[o.Key] = true + } + } + for _, c := range r.cleared { + if c.Class == class { + open[c.Key] = false + } + } + for _, o := range r.observed { + if o.Class == class && open[o.Key] { + out = append(out, o) + } + } + return out +} + +// clears returns every Clear of class, in order. +func (r *conditionRecorder) clears(class string) []conditionObservation { + r.mu.Lock() + defer r.mu.Unlock() + var out []conditionObservation + for _, c := range r.cleared { + if c.Class == class { + out = append(out, c) + } + } + return out +} + +// recordConditions installs a conditionRecorder on m for the test. +func recordConditions(t *testing.T, m *Manager) *conditionRecorder { + t.Helper() + rec := &conditionRecorder{} + m.SetConditionReporter(rec) + t.Cleanup(func() { m.SetConditionReporter(nil) }) + return rec +} + +func TestConditionReporterNilDropsAndRedacts(t *testing.T) { + m := &Manager{} + // Unset: observations and clears are dropped without panicking. + m.observeCondition("c", "k", "msg", "diag") + m.clearCondition("c", "k") + + rec := recordConditions(t, m) + m.observeCondition("c", "k", "token ghp_abcdefghijklmnopqrstuvwxyz123456 in message", "pane: ghp_abcdefghijklmnopqrstuvwxyz123456") + obs := rec.observations("c") + if len(obs) != 1 { + t.Fatalf("observations = %d, want 1", len(obs)) + } + for _, s := range []string{obs[0].Message, obs[0].Diagnostics} { + if containsToken := len(s) > 0 && strings.Contains(s, "ghp_abcdefghijklmnopqrstuvwxyz123456"); containsToken { + t.Fatalf("token reached the reporter: %q", s) + } + } + m.clearCondition("c", "k") + if len(rec.active("c")) != 0 || len(rec.clears("c")) != 1 { + t.Fatalf("clear not forwarded: active=%d clears=%d", len(rec.active("c")), len(rec.clears("c"))) + } + + // Setting nil uninstalls. + m.SetConditionReporter(nil) + m.observeCondition("c", "k", "again", "") + if len(rec.observations("c")) != 1 { + t.Fatal("observation reached an uninstalled reporter") + } +} diff --git a/src/pkg/agent/config_has_tokens_test.go b/src/crux/agent/config_has_tokens_test.go similarity index 100% rename from src/pkg/agent/config_has_tokens_test.go rename to src/crux/agent/config_has_tokens_test.go diff --git a/src/pkg/agent/consent_wedge.go b/src/crux/agent/consent_wedge.go similarity index 100% rename from src/pkg/agent/consent_wedge.go rename to src/crux/agent/consent_wedge.go diff --git a/src/pkg/agent/consent_wedge_test.go b/src/crux/agent/consent_wedge_test.go similarity index 100% rename from src/pkg/agent/consent_wedge_test.go rename to src/crux/agent/consent_wedge_test.go diff --git a/src/pkg/agent/converse_capability_test.go b/src/crux/agent/converse_capability_test.go similarity index 99% rename from src/pkg/agent/converse_capability_test.go rename to src/crux/agent/converse_capability_test.go index a704a3091c..062eefc7c4 100644 --- a/src/pkg/agent/converse_capability_test.go +++ b/src/crux/agent/converse_capability_test.go @@ -8,7 +8,7 @@ import ( "gopkg.in/yaml.v3" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestDefaultCapabilitiesAreEmptyEverywhere is the compatibility contract of diff --git a/src/pkg/agent/copilot_auth_error_test.go b/src/crux/agent/copilot_auth_error_test.go similarity index 100% rename from src/pkg/agent/copilot_auth_error_test.go rename to src/crux/agent/copilot_auth_error_test.go diff --git a/src/pkg/agent/copilot_cache_janitor.go b/src/crux/agent/copilot_cache_janitor.go similarity index 100% rename from src/pkg/agent/copilot_cache_janitor.go rename to src/crux/agent/copilot_cache_janitor.go diff --git a/src/pkg/agent/copilot_cache_janitor_test.go b/src/crux/agent/copilot_cache_janitor_test.go similarity index 100% rename from src/pkg/agent/copilot_cache_janitor_test.go rename to src/crux/agent/copilot_cache_janitor_test.go diff --git a/src/crux/agent/copilot_models.go b/src/crux/agent/copilot_models.go new file mode 100644 index 0000000000..395c0d3453 --- /dev/null +++ b/src/crux/agent/copilot_models.go @@ -0,0 +1,11 @@ +package agent + +import "github.com/hivecommons/hive/crux" + +// CanonicalizeCopilotModel and CopilotLaunchModel are defined in crux so the +// dashboard canonicalizes Copilot model ids without importing the runtime; +// the launch and routing paths in this package call them through these names. +var ( + CanonicalizeCopilotModel = crux.CanonicalizeCopilotModel + CopilotLaunchModel = crux.CopilotLaunchModel +) diff --git a/src/crux/agent/copilot_models_test.go b/src/crux/agent/copilot_models_test.go new file mode 100644 index 0000000000..0524262983 --- /dev/null +++ b/src/crux/agent/copilot_models_test.go @@ -0,0 +1,28 @@ +package agent + +import "testing" + +// TestNormalizeModelNameCopilotDrift verifies the launch-time path: the old +// blind trailing-digits dot-rewrite corrupted claude-fable-5 into the +// CLI-rejected claude-fable.5 (#4262); normalizeModelName must now emit +// CLI-accepted spellings for copilot, self-correcting stored bad ids. +func TestNormalizeModelNameCopilotDrift(t *testing.T) { + tests := []struct{ in, want string }{ + {"claude-fable-5", "claude-fable-5"}, // dashed family must NOT gain a dot + {"claude-fable.5", "claude-fable-5"}, // stored bad id self-corrects at launch + {"claude-sonnet-5", "claude-sonnet-5"}, + {"claude-opus-5", "claude-opus-5"}, + {"claude-sonnet-4-6", "claude-sonnet-4.6"}, // YAML-friendly dashed still maps to dotted + {"claude-opus-4.6", "claude-opus-4.6"}, // legitimate dot unchanged + {"gpt-5.5", "gpt-5.5"}, + {"gemini-2.5-pro", "gemini-2.5-pro"}, + {"auto", "auto"}, // auto-select sentinel flows through + {"gpt-next", "gpt-next"}, // unknown id passthrough + {"custom-7", "custom-7"}, // unknown id: no blind dot-rewrite anymore + } + for _, tt := range tests { + if got := normalizeModelNameForBackend(tt.in, "copilot", false); got != tt.want { + t.Errorf("normalizeModelNameForBackend(%q, copilot, false) = %q, want %q", tt.in, got, tt.want) + } + } +} diff --git a/src/pkg/agent/copilot_question_form.go b/src/crux/agent/copilot_question_form.go similarity index 100% rename from src/pkg/agent/copilot_question_form.go rename to src/crux/agent/copilot_question_form.go diff --git a/src/pkg/agent/copilot_question_form_test.go b/src/crux/agent/copilot_question_form_test.go similarity index 98% rename from src/pkg/agent/copilot_question_form_test.go rename to src/crux/agent/copilot_question_form_test.go index e6cecadba0..9ca0482c72 100644 --- a/src/pkg/agent/copilot_question_form_test.go +++ b/src/crux/agent/copilot_question_form_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) const copilotQuestionFormPane = ` ○ Asking user Waiting for response diff --git a/src/pkg/agent/copilot_session_refresh_test.go b/src/crux/agent/copilot_session_refresh_test.go similarity index 98% rename from src/pkg/agent/copilot_session_refresh_test.go rename to src/crux/agent/copilot_session_refresh_test.go index c65ec0773e..9558255619 100644 --- a/src/pkg/agent/copilot_session_refresh_test.go +++ b/src/crux/agent/copilot_session_refresh_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/internal/testutil" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" ) func TestCopilotSessionRefreshInterval(t *testing.T) { diff --git a/src/pkg/agent/copilot_session_startdelay_test.go b/src/crux/agent/copilot_session_startdelay_test.go similarity index 100% rename from src/pkg/agent/copilot_session_startdelay_test.go rename to src/crux/agent/copilot_session_startdelay_test.go diff --git a/src/pkg/agent/copilot_token_login_test.go b/src/crux/agent/copilot_token_login_test.go similarity index 100% rename from src/pkg/agent/copilot_token_login_test.go rename to src/crux/agent/copilot_token_login_test.go diff --git a/src/pkg/agent/copilot_token_promote_test.go b/src/crux/agent/copilot_token_promote_test.go similarity index 99% rename from src/pkg/agent/copilot_token_promote_test.go rename to src/crux/agent/copilot_token_promote_test.go index 8e8ebfebba..6824ce9306 100644 --- a/src/pkg/agent/copilot_token_promote_test.go +++ b/src/crux/agent/copilot_token_promote_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // extractCopilotToken must handle both value shapes the CLI writes, honor the diff --git a/src/pkg/agent/copilot_token_propagate_test.go b/src/crux/agent/copilot_token_propagate_test.go similarity index 99% rename from src/pkg/agent/copilot_token_propagate_test.go rename to src/crux/agent/copilot_token_propagate_test.go index 55b4b7ccf2..deae03ef90 100644 --- a/src/pkg/agent/copilot_token_propagate_test.go +++ b/src/crux/agent/copilot_token_propagate_test.go @@ -5,7 +5,7 @@ import ( "sort" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // copilotAgent is a running copilot agent with a tmux session, which is the diff --git a/src/pkg/agent/copilot_token_store.go b/src/crux/agent/copilot_token_store.go similarity index 95% rename from src/pkg/agent/copilot_token_store.go rename to src/crux/agent/copilot_token_store.go index 240dd00771..0fa66a4fa3 100644 --- a/src/pkg/agent/copilot_token_store.go +++ b/src/crux/agent/copilot_token_store.go @@ -2,12 +2,11 @@ package agent import ( "encoding/json" - "net/http" "os" "strings" - "time" - "github.com/hivecommons/hive/pkg/claude" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/claude" ) // configHasTokens returns true if either the Copilot config or Claude @@ -276,30 +275,7 @@ func GitHubTokenLogin(token string) string { // "" on any failure. Short-timeout, one call — used only on the rare seed path // where the config lacks a valid identity and on the model-discovery rejection // path. Overridable in tests. -var githubTokenLogin = func(token string) string { - req, err := http.NewRequest("GET", "https://api.github.com/user", nil) - if err != nil { - return "" - } - req.Header.Set("Authorization", "Bearer "+token) - req.Header.Set("Accept", "application/vnd.github+json") - client := &http.Client{Timeout: 5 * time.Second} - resp, err := client.Do(req) - if err != nil { - return "" - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - return "" - } - var body struct { - Login string `json:"login"` - } - if json.NewDecoder(resp.Body).Decode(&body) != nil { - return "" - } - return strings.TrimSpace(body.Login) -} +var githubTokenLogin = crux.GitHubTokenLogin // copilotIdentityKey renders a lastLoggedInUser value — string or // {"host","login"} object — as the ":" copilotTokens key the CLI diff --git a/src/pkg/agent/coverage_boost_test.go b/src/crux/agent/coverage_boost_test.go similarity index 98% rename from src/pkg/agent/coverage_boost_test.go rename to src/crux/agent/coverage_boost_test.go index 088b95f670..e3303e9947 100644 --- a/src/pkg/agent/coverage_boost_test.go +++ b/src/crux/agent/coverage_boost_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/internal/testutil" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" ) // --------------------------------------------------------------------------- @@ -297,6 +297,30 @@ func TestAgentEnvPairs_NonInference_NoAnthropicVars(t *testing.T) { } } +func TestAgentEnvPairs_NonInference_NoProxyLoopback(t *testing.T) { + m := NewManager(map[string]config.AgentConfig{ + "scanner": {Backend: "claude", Model: "sonnet"}, + }, discardLogger(), ProjectContext{}) + + ap := &AgentProcess{ + Name: "scanner", + Config: config.AgentConfig{Backend: "claude", Model: "sonnet"}, + } + + count := 0 + for _, p := range m.agentEnvPairs(ap) { + if p.Key == "NO_PROXY" { + count++ + if p.Value != "127.0.0.1,localhost,::1" { + t.Errorf("NO_PROXY = %q, want loopback list", p.Value) + } + } + } + if count != 1 { + t.Errorf("NO_PROXY emitted %d times, want 1", count) + } +} + func TestAgentEnvPairs_UID_NonInference_HOME(t *testing.T) { m := NewManager(map[string]config.AgentConfig{ "scanner": {Backend: "claude", Model: "sonnet"}, @@ -1388,19 +1412,20 @@ func TestNudgeIfKickStalled(t *testing.T) { func TestCreditStallRecoveryLocked(t *testing.T) { kicked := paneContentHash("pane at kick time") agent := &AgentProcess{lastInferKickPane: kicked, stallNudgeSent: true} + m := &Manager{} - creditStallRecoveryLocked(agent, kicked) + m.creditStallRecoveryLocked(agent, kicked) if agent.StallRecoveries != 0 || agent.stallNudgeSent { t.Fatalf("unchanged pane must not credit, got %d sent=%v", agent.StallRecoveries, agent.stallNudgeSent) } agent.stallNudgeSent = true - creditStallRecoveryLocked(agent, paneContentHash("moved on")) + m.creditStallRecoveryLocked(agent, paneContentHash("moved on")) if agent.StallRecoveries != 1 || agent.stallNudgeSent { t.Fatalf("nudge then pane change then new kick must credit, got %d sent=%v", agent.StallRecoveries, agent.stallNudgeSent) } - creditStallRecoveryLocked(agent, paneContentHash("moved again")) + m.creditStallRecoveryLocked(agent, paneContentHash("moved again")) if agent.StallRecoveries != 1 { t.Fatalf("no outstanding nudge must not credit, got %d", agent.StallRecoveries) } diff --git a/src/pkg/agent/crash_coverage_test.go b/src/crux/agent/crash_coverage_test.go similarity index 78% rename from src/pkg/agent/crash_coverage_test.go rename to src/crux/agent/crash_coverage_test.go index 4fa0e9b0c0..df587184e7 100644 --- a/src/pkg/agent/crash_coverage_test.go +++ b/src/crux/agent/crash_coverage_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/internal/testutil" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" ) // TestCheckAndRestartCrashedAgents_BarePaneRestarts: a running agent with a @@ -145,3 +145,37 @@ func TestCheckAndRestartCrashedAgents_ConsentStuck(t *testing.T) { t.Error("consent-stuck path should record when the screen was first observed") } } + +// TestCheckAndRestartCrashedAgents_ClaudeConsentStuck: a plain claude agent +// launches with --dangerously-skip-permissions too, so when it parks on the +// bypass consent screen it must go through the same consent-stuck dismissal +// as inference agents instead of waiting forever (#14307). +func TestCheckAndRestartCrashedAgents_ClaudeConsentStuck(t *testing.T) { + m := NewManager(map[string]config.AgentConfig{"cxc": {Backend: "claude"}}, discardLogger(), ProjectContext{}) + origExists := tmuxSessionExists + tmuxSessionExists = func(*Manager, *AgentProcess) bool { return true } + t.Cleanup(func() { tmuxSessionExists = origExists }) + + m.mu.RLock() + agent := m.agents["cxc"] + m.mu.RUnlock() + termSeams(m).captureVisiblePane = func(*AgentProcess) string { + return "WARNING: Claude Code running in Bypass Permissions mode\n❯ 1. No, exit\n 2. Yes, I accept\nEnter to confirm · Esc to exit\n" + } + + old := time.Now().Add(-2 * time.Minute) + m.mu.Lock() + agent.State = StateRunning + agent.StartedAt = &old + m.mu.Unlock() + + if restarted := m.CheckAndRestartCrashedAgents(context.Background()); len(restarted) != 0 { + t.Errorf("consent-stuck claude agent should not restart, got %v", restarted) + } + m.mu.RLock() + seen := agent.consentSeenAt + m.mu.RUnlock() + if seen.IsZero() { + t.Error("claude agent on the bypass consent screen should enter the consent-stuck path") + } +} diff --git a/src/pkg/agent/crash_pane_tail_test.go b/src/crux/agent/crash_pane_tail_test.go similarity index 100% rename from src/pkg/agent/crash_pane_tail_test.go rename to src/crux/agent/crash_pane_tail_test.go diff --git a/src/pkg/agent/credential_watchdog_test.go b/src/crux/agent/credential_watchdog_test.go similarity index 99% rename from src/pkg/agent/credential_watchdog_test.go rename to src/crux/agent/credential_watchdog_test.go index 0b9f7a50c7..5c6d0300a0 100644 --- a/src/pkg/agent/credential_watchdog_test.go +++ b/src/crux/agent/credential_watchdog_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestCredentialWatchdogInterval_Default(t *testing.T) { diff --git a/src/pkg/agent/credentials_watchdog.go b/src/crux/agent/credentials_watchdog.go similarity index 99% rename from src/pkg/agent/credentials_watchdog.go rename to src/crux/agent/credentials_watchdog.go index 31d507f026..f64c76eca0 100644 --- a/src/pkg/agent/credentials_watchdog.go +++ b/src/crux/agent/credentials_watchdog.go @@ -8,7 +8,7 @@ import ( "syscall" "time" - "github.com/hivecommons/hive/pkg/claude" + "github.com/hivecommons/hive/crux/claude" ) const ( diff --git a/src/crux/agent/crux_runtime.go b/src/crux/agent/crux_runtime.go new file mode 100644 index 0000000000..c5ba71118c --- /dev/null +++ b/src/crux/agent/crux_runtime.go @@ -0,0 +1,21 @@ +package agent + +import "github.com/hivecommons/hive/crux" + +// *Manager is the runtime crux.Runtime describes. The assertion lives here, +// not in the crux root package, because crux/agent imports the root and the +// root must not import it back. +var _ crux.Runtime = (*Manager)(nil) + +// *Manager is also the in-process crux.Host: the one implementation that +// can be handed Go closures and minters directly (hivecommons/hive#14138). +var _ crux.Host = (*Manager)(nil) + +// AsRuntime adapts a manager to crux.Runtime, returning a nil interface for a +// nil manager so consumers' `runtime != nil` checks keep their meaning. +func AsRuntime(m *Manager) crux.Runtime { + if m == nil { + return nil + } + return m +} diff --git a/src/crux/agent/crux_runtime_test.go b/src/crux/agent/crux_runtime_test.go new file mode 100644 index 0000000000..8ce3064d5c --- /dev/null +++ b/src/crux/agent/crux_runtime_test.go @@ -0,0 +1,97 @@ +package agent + +import ( + "log/slog" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" +) + +func TestAsRuntime_NilManagerIsNilRuntime(t *testing.T) { + var m *Manager + if rt := AsRuntime(m); rt != nil { + t.Fatalf("AsRuntime(nil) = %#v, want a nil interface so `rt != nil` guards keep working", rt) + } +} + +func TestAsRuntime_WrapsManagerUnchanged(t *testing.T) { + m := NewManager(map[string]config.AgentConfig{}, slog.Default(), ProjectContext{}) + rt := AsRuntime(m) + if rt == nil { + t.Fatal("AsRuntime(non-nil) returned nil") + } + if got, ok := rt.(*Manager); !ok || got != m { + t.Fatalf("AsRuntime returned %T, want the same *Manager", rt) + } + if _, err := rt.GetStatus("no-such-agent"); err == nil { + t.Fatal("GetStatus on an unknown agent should fail through the Runtime surface as it does on the manager") + } +} + +// TestSnapshotCopiesPaneCapture pins that the crux.Process a snapshot hands +// out still answers PaneLines from the capture it was taken with, and that +// the copy does not alias the live capture. +func TestSnapshotCopiesPaneCapture(t *testing.T) { + a := &AgentProcess{Name: "scanner", lastPaneCapture: []string{"hello", "world"}} + snap := a.snapshot() + if got := snap.PaneLines(10); len(got) != 2 || got[0] != "hello" || got[1] != "world" { + t.Fatalf("snapshot PaneLines = %q, want [hello world]", got) + } + a.lastPaneCapture[0] = "mutated" + if got := snap.PaneLines(10); len(got) != 2 || got[0] != "hello" { + t.Fatalf("snapshot PaneLines after live mutation = %q, want the captured copy", got) + } + if got, want := snap.FilteredPaneLines(1), a.FilteredPaneLines(1); len(got) != 1 || len(want) != 1 || got[0] != "world" { + t.Fatalf("snapshot FilteredPaneLines(1) = %q, live = %q, want [world]", got, want) + } +} + +// Status readers must preserve diagnostic and CI state from the live agent. +func TestStatusSnapshotsCarryDiagnosticAndCIState(t *testing.T) { + a := &AgentProcess{ + Name: "scanner", + AwaitingCI: true, + BootstrapOverride: "custom bootstrap", + CIPollNudges: 3, + KickRefused: true, + KickRefusalReason: "waiting for startup", + LastError: "provider unavailable", + } + m := NewManager(map[string]config.AgentConfig{}, slog.Default(), ProjectContext{}) + m.agents[a.Name] = a + readers := map[string]func() (*crux.Process, error){ + "GetStatus": func() (*crux.Process, error) { return m.GetStatus(a.Name) }, + "GetStatusFast": func() (*crux.Process, error) { return m.GetStatusFast(a.Name) }, + "AllStatuses": func() (*crux.Process, error) { return m.AllStatuses()[a.Name], nil }, + } + for name, read := range readers { + t.Run(name, func(t *testing.T) { + snap, err := read() + if err != nil { + t.Fatal(err) + } + if snap == nil { + t.Fatal("missing snapshot") + } + if snap.AwaitingCI != a.AwaitingCI { + t.Errorf("AwaitingCI = %v, want %v", snap.AwaitingCI, a.AwaitingCI) + } + if snap.BootstrapOverride != a.BootstrapOverride { + t.Errorf("BootstrapOverride = %q, want %q", snap.BootstrapOverride, a.BootstrapOverride) + } + if snap.CIPollNudges != a.CIPollNudges { + t.Errorf("CIPollNudges = %d, want %d", snap.CIPollNudges, a.CIPollNudges) + } + if snap.KickRefused != a.KickRefused { + t.Errorf("KickRefused = %v, want %v", snap.KickRefused, a.KickRefused) + } + if snap.KickRefusalReason != a.KickRefusalReason { + t.Errorf("KickRefusalReason = %q, want %q", snap.KickRefusalReason, a.KickRefusalReason) + } + if snap.LastError != a.LastError { + t.Errorf("LastError = %q, want %q", snap.LastError, a.LastError) + } + }) + } +} diff --git a/src/pkg/agent/dashboard_token_isolation_test.go b/src/crux/agent/dashboard_token_isolation_test.go similarity index 98% rename from src/pkg/agent/dashboard_token_isolation_test.go rename to src/crux/agent/dashboard_token_isolation_test.go index 830e321899..54f159e758 100644 --- a/src/pkg/agent/dashboard_token_isolation_test.go +++ b/src/crux/agent/dashboard_token_isolation_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // #8348: a hub-launched agent must never be handed the dashboard bearer, in diff --git a/src/pkg/agent/dedup_bench_test.go b/src/crux/agent/dedup_bench_test.go similarity index 100% rename from src/pkg/agent/dedup_bench_test.go rename to src/crux/agent/dedup_bench_test.go diff --git a/src/pkg/agent/diag_coverage_test.go b/src/crux/agent/diag_coverage_test.go similarity index 99% rename from src/pkg/agent/diag_coverage_test.go rename to src/crux/agent/diag_coverage_test.go index abf1676e6a..b7edb55cb6 100644 --- a/src/pkg/agent/diag_coverage_test.go +++ b/src/crux/agent/diag_coverage_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestRunCopilotDiagnostic_AuthError drives the diagnostic loop: a bare copilot diff --git a/src/pkg/agent/dismiss_coverage_test.go b/src/crux/agent/dismiss_coverage_test.go similarity index 100% rename from src/pkg/agent/dismiss_coverage_test.go rename to src/crux/agent/dismiss_coverage_test.go diff --git a/src/crux/agent/dismiss_prompts_fake_test.go b/src/crux/agent/dismiss_prompts_fake_test.go new file mode 100644 index 0000000000..0358fcdc49 --- /dev/null +++ b/src/crux/agent/dismiss_prompts_fake_test.go @@ -0,0 +1,130 @@ +package agent + +import ( + "runtime" + "slices" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" +) + +const readyPromptPane = ": esc to interrupt" + +type scriptedPromptPane struct { + mu sync.Mutex + initialPane string + afterDownPane string + readyAfterKeys int + keys []string +} + +func newDismissPromptHarness(t *testing.T, name, initialPane string, readyAfterKeys int) (*Manager, *AgentProcess, *scriptedPromptPane) { + t.Helper() + m := NewManager(map[string]config.AgentConfig{name: {Backend: "litellm"}}, discardLogger(), ProjectContext{}) + m.mu.RLock() + agent := m.agents[name] + m.mu.RUnlock() + + script := &scriptedPromptPane{initialPane: initialPane, readyAfterKeys: readyAfterKeys} + termSeams(m).captureVisiblePane = script.capture + termSeams(m).sendKeys = script.sendKeys + termSeams(m).sleepDuringPromptDismiss = func(time.Duration) { runtime.Gosched() } + m.promptDismissTimeout = 200 * time.Millisecond + return m, agent, script +} + +func (s *scriptedPromptPane) capture(*AgentProcess) string { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.keys) >= s.readyAfterKeys { + return readyPromptPane + } + if s.afterDownPane != "" && slices.Contains(s.keys, "Down") { + return s.afterDownPane + } + return s.initialPane +} + +func (s *scriptedPromptPane) sendKeys(_ *AgentProcess, keys ...string) { + s.mu.Lock() + defer s.mu.Unlock() + s.keys = append(s.keys, keys...) +} + +func (s *scriptedPromptPane) keySnapshot() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.keys...) +} + +func runDismissPromptScript(t *testing.T, m *Manager, agent *AgentProcess, script *scriptedPromptPane, wantKeys []string) { + t.Helper() + done := make(chan struct{}) + go func() { + m.dismissInferencePrompts(agent) + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("dismissInferencePrompts should return") + } + if got := script.keySnapshot(); !slices.Equal(got, wantKeys) { + t.Fatalf("dismissInferencePrompts keys = %v, want %v", got, wantKeys) + } +} + +const bypassConsentKickPane = "WARNING: Claude Code running in Bypass Permissions mode\n❯ 1. No, exit\n 2. Yes, I accept\nEnter to confirm · Esc to exit" + +// TestAcceptBypassConsentForKick_AnswersScreen is the #14307 guard: a kick that +// finds the bypass consent screen accepts it in place, instead of restarting +// the CLI into the same screen forever. +func TestAcceptBypassConsentForKick_AnswersScreen(t *testing.T) { + m, agent, script := newDismissPromptHarness(t, "kickconsent", bypassConsentKickPane, 2) + script.afterDownPane = "WARNING: Claude Code running in Bypass Permissions mode\n 1. No, exit\n❯ 2. Yes, I accept\nEnter to confirm · Esc to exit" + + if !m.acceptBypassConsentForKick(agent, bypassConsentKickPane) { + t.Fatal("acceptBypassConsentForKick should answer the bypass consent screen") + } + if got, want := script.keySnapshot(), []string{"Down", "Enter"}; !slices.Equal(got, want) { + t.Fatalf("keys = %v, want %v", got, want) + } +} + +// TestAcceptBypassConsentForKick_LeavesOtherScreensToRestart: only the bypass +// consent screen is answered here; a bare shell or another menu keeps the +// existing restart recovery and receives no keystrokes. +func TestAcceptBypassConsentForKick_LeavesOtherScreensToRestart(t *testing.T) { + for name, pane := range map[string]string{ + "bare shell": "dev@hive:~$ ", + "other menu": ": Enter to confirm\n❯ 1. No, exit", + "title only": "Bypass Permissions mode\n❯ No, exit", + "empty capture": "", + } { + t.Run(name, func(t *testing.T) { + m, agent, script := newDismissPromptHarness(t, "kickother", pane, 99) + if m.acceptBypassConsentForKick(agent, pane) { + t.Fatal("acceptBypassConsentForKick should decline a non-bypass screen") + } + if got := script.keySnapshot(); len(got) != 0 { + t.Fatalf("keys = %v, want none", got) + } + }) + } +} + +// TestAcceptBypassConsentForKick_FallsBackWhenNotAccepted: if the accept option +// is never reached the helper reports failure so the caller restarts as before. +func TestAcceptBypassConsentForKick_FallsBackWhenNotAccepted(t *testing.T) { + m, agent, script := newDismissPromptHarness(t, "kickstuck", bypassConsentKickPane, 99) + if m.acceptBypassConsentForKick(agent, bypassConsentKickPane) { + t.Fatal("acceptBypassConsentForKick should report failure when the screen never clears") + } + for _, k := range script.keySnapshot() { + if k == "Enter" { + t.Fatal("must never confirm while \"No, exit\" is still selected") + } + } +} diff --git a/src/pkg/agent/enters_prompt_guard_test.go b/src/crux/agent/enters_prompt_guard_test.go similarity index 98% rename from src/pkg/agent/enters_prompt_guard_test.go rename to src/crux/agent/enters_prompt_guard_test.go index e9c97bf98a..0316eee673 100644 --- a/src/pkg/agent/enters_prompt_guard_test.go +++ b/src/crux/agent/enters_prompt_guard_test.go @@ -15,7 +15,7 @@ import ( "log/slog" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // syncBuffer is a bytes.Buffer safe for a logger that may be written from diff --git a/src/pkg/agent/envpairs_coverage_test.go b/src/crux/agent/envpairs_coverage_test.go similarity index 98% rename from src/pkg/agent/envpairs_coverage_test.go rename to src/crux/agent/envpairs_coverage_test.go index 2a3d228e67..c3f9be4a61 100644 --- a/src/pkg/agent/envpairs_coverage_test.go +++ b/src/crux/agent/envpairs_coverage_test.go @@ -3,7 +3,7 @@ package agent import ( "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestAgentEnvPairs_ClaudeTokenAndExtras covers the claude-token, BeadsDir, diff --git a/src/pkg/agent/experiment_env_test.go b/src/crux/agent/experiment_env_test.go similarity index 98% rename from src/pkg/agent/experiment_env_test.go rename to src/crux/agent/experiment_env_test.go index d4983e9191..970cdb3492 100644 --- a/src/pkg/agent/experiment_env_test.go +++ b/src/crux/agent/experiment_env_test.go @@ -3,7 +3,7 @@ package agent import ( "testing" - "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/crux/config" ) // TestExperimentEnvPairsOnlyForExperimenter pins RFC 11585 part 5's diff --git a/src/pkg/agent/explain_mode_test.go b/src/crux/agent/explain_mode_test.go similarity index 99% rename from src/pkg/agent/explain_mode_test.go rename to src/crux/agent/explain_mode_test.go index 9155b9fc1d..ea30c8862f 100644 --- a/src/pkg/agent/explain_mode_test.go +++ b/src/crux/agent/explain_mode_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // Explain mode (#3887) lets an operator ask an agent to say WHY it did what it diff --git a/src/pkg/agent/fatal_network_producing_test.go b/src/crux/agent/fatal_network_producing_test.go similarity index 100% rename from src/pkg/agent/fatal_network_producing_test.go rename to src/crux/agent/fatal_network_producing_test.go diff --git a/src/pkg/agent/fleet_breaker_test.go b/src/crux/agent/fleet_breaker_test.go similarity index 99% rename from src/pkg/agent/fleet_breaker_test.go rename to src/crux/agent/fleet_breaker_test.go index 1d723433d8..b6a945b1f4 100644 --- a/src/pkg/agent/fleet_breaker_test.go +++ b/src/crux/agent/fleet_breaker_test.go @@ -5,7 +5,7 @@ import ( "log/slog" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // breakerTestManager builds a Manager and forces each named agent into a known diff --git a/src/pkg/agent/gitremotes_coverage_test.go b/src/crux/agent/gitremotes_coverage_test.go similarity index 97% rename from src/pkg/agent/gitremotes_coverage_test.go rename to src/crux/agent/gitremotes_coverage_test.go index da0ae93136..11433a27d1 100644 --- a/src/pkg/agent/gitremotes_coverage_test.go +++ b/src/crux/agent/gitremotes_coverage_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestSanitizeGitRemotes_StripsEmbeddedToken creates a real git repo under the diff --git a/src/pkg/agent/headless_credentials.go b/src/crux/agent/headless_credentials.go similarity index 96% rename from src/pkg/agent/headless_credentials.go rename to src/crux/agent/headless_credentials.go index e7c74a791f..affa0cc3f9 100644 --- a/src/pkg/agent/headless_credentials.go +++ b/src/crux/agent/headless_credentials.go @@ -6,8 +6,8 @@ import ( "path/filepath" "strings" - "github.com/hivecommons/hive/pkg/claude" - "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/config" ) // HeadlessCredentialSources carries the credential inputs a headless launch diff --git a/src/pkg/agent/headless_credentials_test.go b/src/crux/agent/headless_credentials_test.go similarity index 98% rename from src/pkg/agent/headless_credentials_test.go rename to src/crux/agent/headless_credentials_test.go index 7dca1b0b06..16e294d8d7 100644 --- a/src/pkg/agent/headless_credentials_test.go +++ b/src/crux/agent/headless_credentials_test.go @@ -6,7 +6,7 @@ import ( "slices" "testing" - "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/crux/config" ) func TestHeadlessCredentialEnvCopilotReadsDurableToken(t *testing.T) { diff --git a/src/pkg/agent/helpers_coverage_test.go b/src/crux/agent/helpers_coverage_test.go similarity index 99% rename from src/pkg/agent/helpers_coverage_test.go rename to src/crux/agent/helpers_coverage_test.go index 7824737ca3..23582b2483 100644 --- a/src/pkg/agent/helpers_coverage_test.go +++ b/src/crux/agent/helpers_coverage_test.go @@ -5,7 +5,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // --------------------------------------------------------------------------- diff --git a/src/pkg/agent/host_state_deny_test.go b/src/crux/agent/host_state_deny_test.go similarity index 98% rename from src/pkg/agent/host_state_deny_test.go rename to src/crux/agent/host_state_deny_test.go index c3316aacb8..ddfb526639 100644 --- a/src/pkg/agent/host_state_deny_test.go +++ b/src/crux/agent/host_state_deny_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/internal/testutil" + "github.com/hivecommons/hive/crux/internal/testutil" ) // backendsConfPath is the shell half of this policy. Both launch paths must diff --git a/src/pkg/agent/interactive_home.go b/src/crux/agent/interactive_home.go similarity index 100% rename from src/pkg/agent/interactive_home.go rename to src/crux/agent/interactive_home.go diff --git a/src/pkg/agent/interactive_home_errors_test.go b/src/crux/agent/interactive_home_errors_test.go similarity index 99% rename from src/pkg/agent/interactive_home_errors_test.go rename to src/crux/agent/interactive_home_errors_test.go index 83d4f27f7b..e4c0493310 100644 --- a/src/pkg/agent/interactive_home_errors_test.go +++ b/src/crux/agent/interactive_home_errors_test.go @@ -5,7 +5,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // mismatchUID is a UID the test process does not own, so os.Chown fails with diff --git a/src/pkg/agent/interactive_home_gitconfig_test.go b/src/crux/agent/interactive_home_gitconfig_test.go similarity index 99% rename from src/pkg/agent/interactive_home_gitconfig_test.go rename to src/crux/agent/interactive_home_gitconfig_test.go index e80af4e8e8..6140b0c4cd 100644 --- a/src/pkg/agent/interactive_home_gitconfig_test.go +++ b/src/crux/agent/interactive_home_gitconfig_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // gitConfigHomeManager builds a manager whose logger records into the returned diff --git a/src/pkg/agent/interactive_home_test.go b/src/crux/agent/interactive_home_test.go similarity index 99% rename from src/pkg/agent/interactive_home_test.go rename to src/crux/agent/interactive_home_test.go index 7f3d2f6ebe..79cfde95d6 100644 --- a/src/pkg/agent/interactive_home_test.go +++ b/src/crux/agent/interactive_home_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // withSharedAgentHome points the interactive-home machinery at a temp dir for diff --git a/src/pkg/agent/invocation_metadata_test.go b/src/crux/agent/invocation_metadata_test.go similarity index 99% rename from src/pkg/agent/invocation_metadata_test.go rename to src/crux/agent/invocation_metadata_test.go index 152f371d13..745b94ac1c 100644 --- a/src/pkg/agent/invocation_metadata_test.go +++ b/src/crux/agent/invocation_metadata_test.go @@ -3,7 +3,7 @@ package agent import ( "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestInvocationMetadata pins the contract the invocation-attribution trail diff --git a/src/pkg/agent/jev_install.go b/src/crux/agent/jev_install.go similarity index 99% rename from src/pkg/agent/jev_install.go rename to src/crux/agent/jev_install.go index 4f647d490e..614c0d56e8 100644 --- a/src/pkg/agent/jev_install.go +++ b/src/crux/agent/jev_install.go @@ -8,7 +8,7 @@ import ( "path/filepath" "strings" - "github.com/hivecommons/hive/pkg/jev" + "github.com/hivecommons/hive/crux/jev" ) // installJevForAgent seeds the jev-decide skill into the agent's CLI home diff --git a/src/pkg/agent/jev_install_test.go b/src/crux/agent/jev_install_test.go similarity index 98% rename from src/pkg/agent/jev_install_test.go rename to src/crux/agent/jev_install_test.go index 83e4beec84..73f62a3405 100644 --- a/src/pkg/agent/jev_install_test.go +++ b/src/crux/agent/jev_install_test.go @@ -6,8 +6,8 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/jev" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/jev" ) func jevEnv(t *testing.T, mode string) map[string]string { diff --git a/src/pkg/agent/kick_async.go b/src/crux/agent/kick_async.go similarity index 92% rename from src/pkg/agent/kick_async.go rename to src/crux/agent/kick_async.go index 42e4349ff3..f257bbaba2 100644 --- a/src/pkg/agent/kick_async.go +++ b/src/crux/agent/kick_async.go @@ -6,20 +6,20 @@ import ( "sync" "time" - "github.com/hivecommons/hive/pkg/crux/types" + "github.com/hivecommons/hive/crux" ) -// Kick dispatch phases and KickDispatch are defined in pkg/crux/types as part +// Kick dispatch phases and KickDispatch are defined in pkg/crux as part // of the Crux runtime surface (hivecommons/hive#13536); aliased here. const ( - KickPhasePending = types.KickPhasePending - KickPhaseDelivered = types.KickPhaseDelivered - KickPhaseFailed = types.KickPhaseFailed + KickPhasePending = crux.KickPhasePending + KickPhaseDelivered = crux.KickPhaseDelivered + KickPhaseFailed = crux.KickPhaseFailed ) // KickDispatch is the observable outcome of one asynchronous kick; see -// types.KickDispatch. -type KickDispatch = types.KickDispatch +// crux.KickDispatch. +type KickDispatch = crux.KickDispatch // kickDispatchRegistry holds the latest dispatch per agent plus the in-flight // guard. It is deliberately independent of m.mu: the background delivery @@ -273,18 +273,22 @@ func (m *Manager) deliverKickAsync(name, message string) error { pane := m.captureVisiblePaneForAgent(agent) if !paneHasCLIMarker(pane) || paneShowsConsentScreen(pane) { consentScreen := paneShowsConsentScreen(pane) - if consentScreen { - // Same wedge bookkeeping as SendKick — see noteConsentWedge. - m.noteConsentWedge(name) - } - m.logger.Warn("agent CLI crashed or stuck on consent screen, restarting before kick", - "name", name, "consent_screen", consentScreen) m.mu.Unlock() - if err := m.restartForKick(context.Background(), name); err != nil { - return fmt.Errorf("failed to restart crashed agent %s: %w", name, err) - } - if !m.waitForCLIReadyForAgent(agent) { - return fmt.Errorf("agent %s CLI did not become ready after restart", name) + // Answer the bypass consent screen in place: a restart relaunches + // into the same screen and the kick never lands (#14307). + if !consentScreen || !m.acceptBypassConsentForKick(agent, pane) { + if consentScreen { + // Same wedge bookkeeping as SendKick — see noteConsentWedge. + m.noteConsentWedge(name) + } + m.logger.Warn("agent CLI crashed or stuck on consent screen, restarting before kick", + "name", name, "consent_screen", consentScreen) + if err := m.restartForKick(context.Background(), name); err != nil { + return fmt.Errorf("failed to restart crashed agent %s: %w", name, err) + } + if !m.waitForCLIReadyForAgent(agent) { + return fmt.Errorf("agent %s CLI did not become ready after restart", name) + } } m.mu.Lock() agent, ok = m.agents[name] diff --git a/src/pkg/agent/kick_async_deliver_test.go b/src/crux/agent/kick_async_deliver_test.go similarity index 99% rename from src/pkg/agent/kick_async_deliver_test.go rename to src/crux/agent/kick_async_deliver_test.go index 988f2bae0f..060a60d9f1 100644 --- a/src/pkg/agent/kick_async_deliver_test.go +++ b/src/crux/agent/kick_async_deliver_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // Tests for the slow half of the asynchronous kick path (#5325): diff --git a/src/pkg/agent/kick_async_test.go b/src/crux/agent/kick_async_test.go similarity index 99% rename from src/pkg/agent/kick_async_test.go rename to src/crux/agent/kick_async_test.go index cd2037a9bd..d6e99031e8 100644 --- a/src/pkg/agent/kick_async_test.go +++ b/src/crux/agent/kick_async_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // Tests for the asynchronous kick path (#5325). diff --git a/src/pkg/agent/kick_cancel.go b/src/crux/agent/kick_cancel.go similarity index 100% rename from src/pkg/agent/kick_cancel.go rename to src/crux/agent/kick_cancel.go diff --git a/src/pkg/agent/kick_cancel_test.go b/src/crux/agent/kick_cancel_test.go similarity index 99% rename from src/pkg/agent/kick_cancel_test.go rename to src/crux/agent/kick_cancel_test.go index 93ab21412e..f42953ec15 100644 --- a/src/pkg/agent/kick_cancel_test.go +++ b/src/crux/agent/kick_cancel_test.go @@ -11,8 +11,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/internal/testutil" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" ) // TestKickDispatchRegistry_CancelFailsPending: cancel fails only a PENDING diff --git a/src/pkg/agent/kick_chunk_pause_test.go b/src/crux/agent/kick_chunk_pause_test.go similarity index 100% rename from src/pkg/agent/kick_chunk_pause_test.go rename to src/crux/agent/kick_chunk_pause_test.go diff --git a/src/pkg/agent/kick_delivery_failure_test.go b/src/crux/agent/kick_delivery_failure_test.go similarity index 92% rename from src/pkg/agent/kick_delivery_failure_test.go rename to src/crux/agent/kick_delivery_failure_test.go index 17d3a99dab..f6cd3aa112 100644 --- a/src/pkg/agent/kick_delivery_failure_test.go +++ b/src/crux/agent/kick_delivery_failure_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/internal/testutil" - "github.com/hivecommons/hive/pkg/fleetreport" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/internal/testutil" ) // A ready pane is not proof that tmux accepted the kick. Failed writes must @@ -147,9 +147,8 @@ func TestSendKickPropagatesFinalDeliveryFailure(t *testing.T) { } func TestKickWriteFailuresReportAndClearOnRecovery(t *testing.T) { - tr := fleetreport.NewRuntimeTracker(time.Now) - fleetreport.SetDefaultTrackerForTest(t, tr) m, a, _ := kickLogTestManager(t, "") + tr := recordConditions(t, m) termSeams(m).sendLiteralErr = func(*AgentProcess, string) error { return errors.New("send failed") } m.mu.Lock() defer m.mu.Unlock() @@ -162,22 +161,20 @@ func TestKickWriteFailuresReportAndClearOnRecovery(t *testing.T) { } } found := false - for _, ev := range tr.Evidence() { - if ev.Class == fleetreport.ClassKickFailedRepeatedly && strings.Contains(ev.Message, "send text") { + for _, ev := range tr.active(crux.ClassKickFailedRepeatedly) { + if strings.Contains(ev.Message, "send text") { found = true } } if !found || a.KicksUndeliverable != 3 { - t.Fatalf("missing repeated delivery failure: count=%d, evidence=%+v", a.KicksUndeliverable, tr.Evidence()) + t.Fatalf("missing repeated delivery failure: count=%d, evidence=%+v", a.KicksUndeliverable, tr.observed) } termSeams(m).sendLiteralErr = nil if err := m.deliverKickLocked(a, "task", "send-kick"); err != nil { t.Fatal(err) } - for _, ev := range tr.Evidence() { - if ev.Class == fleetreport.ClassKickFailedRepeatedly { - t.Fatal("successful retry left repeated-failure evidence active") - } + if ev := tr.active(crux.ClassKickFailedRepeatedly); len(ev) != 0 { + t.Fatalf("successful retry left repeated-failure evidence active: %+v", ev) } if a.KicksUndeliverable != 0 || !a.busyNoActivityRestartAt.IsZero() || a.LastKick == nil || len(a.KickHistory) != 1 { t.Fatal("successful retry did not clear failures and the stall-restart marker and record exactly one kick") diff --git a/src/pkg/agent/kick_log_alt_screen_test.go b/src/crux/agent/kick_log_alt_screen_test.go similarity index 100% rename from src/pkg/agent/kick_log_alt_screen_test.go rename to src/crux/agent/kick_log_alt_screen_test.go diff --git a/src/pkg/agent/kick_logs.go b/src/crux/agent/kick_logs.go similarity index 98% rename from src/pkg/agent/kick_logs.go rename to src/crux/agent/kick_logs.go index d6623b7c10..f5071ef72a 100644 --- a/src/pkg/agent/kick_logs.go +++ b/src/crux/agent/kick_logs.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/crux/types" + "github.com/hivecommons/hive/crux" ) const ( @@ -71,8 +71,8 @@ const ( kickLogFileMode = os.FileMode(0o600) ) -// KickLogInfo describes one archived kick log; see types.KickLogInfo. -type KickLogInfo = types.KickLogInfo +// KickLogInfo describes one archived kick log; see crux.KickLogInfo. +type KickLogInfo = crux.KickLogInfo // kickLogSettingsFromEnv resolves the archive root, per-agent retention // count, and per-agent byte cap, applying env overrides over the defaults. @@ -260,6 +260,12 @@ func (m *Manager) kickLogHeaderLocked(agent *AgentProcess, stampKey string, now // kick log headers. It is a func so pkg/agent stays free of pkg/dashboard, // which owns the build info the version endpoint serves. func (m *Manager) SetBuildInfo(fn func() (version, commit string)) { + if fn != nil { + // The binary's own commit also defaults HIVE_SHA for launched agents + // (#11220); the kick-log header and the agent env agree on it. + _, commit := fn() + SetBuildCommit(commit) + } m.mu.Lock() defer m.mu.Unlock() m.buildInfo = fn diff --git a/src/pkg/agent/kick_logs_dir_test.go b/src/crux/agent/kick_logs_dir_test.go similarity index 100% rename from src/pkg/agent/kick_logs_dir_test.go rename to src/crux/agent/kick_logs_dir_test.go diff --git a/src/pkg/agent/kick_logs_test.go b/src/crux/agent/kick_logs_test.go similarity index 99% rename from src/pkg/agent/kick_logs_test.go rename to src/crux/agent/kick_logs_test.go index 01c2b033a0..161e4d91e0 100644 --- a/src/pkg/agent/kick_logs_test.go +++ b/src/crux/agent/kick_logs_test.go @@ -16,7 +16,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // kickLogTestManager builds a Manager with one agent whose tmux capture and diff --git a/src/pkg/agent/kick_observer.go b/src/crux/agent/kick_observer.go similarity index 100% rename from src/pkg/agent/kick_observer.go rename to src/crux/agent/kick_observer.go diff --git a/src/pkg/agent/kick_observer_test.go b/src/crux/agent/kick_observer_test.go similarity index 100% rename from src/pkg/agent/kick_observer_test.go rename to src/crux/agent/kick_observer_test.go diff --git a/src/pkg/agent/kick_outcome.go b/src/crux/agent/kick_outcome.go similarity index 95% rename from src/pkg/agent/kick_outcome.go rename to src/crux/agent/kick_outcome.go index 32f3ba09d7..3a60327be6 100644 --- a/src/pkg/agent/kick_outcome.go +++ b/src/crux/agent/kick_outcome.go @@ -5,22 +5,22 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/crux/types" + "github.com/hivecommons/hive/crux" ) -// Kick outcome kinds and KickOutcome are defined in pkg/crux/types as part of +// Kick outcome kinds and KickOutcome are defined in pkg/crux as part of // the Crux runtime surface (hivecommons/hive#13536); aliased here. pkg/governor // mirrors the strings (governor.KickOutcome*) so it needs no import edge. const ( - KickOutcomeQuestion = types.KickOutcomeQuestion - KickOutcomeStandDown = types.KickOutcomeStandDown - KickOutcomeNoOp = types.KickOutcomeNoOp - KickOutcomeEnded = types.KickOutcomeEnded + KickOutcomeQuestion = crux.KickOutcomeQuestion + KickOutcomeStandDown = crux.KickOutcomeStandDown + KickOutcomeNoOp = crux.KickOutcomeNoOp + KickOutcomeEnded = crux.KickOutcomeEnded ) // KickOutcome is how an agent's most recent kicked turn ended; see -// types.KickOutcome. -type KickOutcome = types.KickOutcome +// crux.KickOutcome. +type KickOutcome = crux.KickOutcome // kickOutcomeGrace is how long after delivery a pane at the input prompt is // still assumed to be the CLI accepting the kick rather than a finished turn. diff --git a/src/pkg/agent/kick_outcome_test.go b/src/crux/agent/kick_outcome_test.go similarity index 100% rename from src/pkg/agent/kick_outcome_test.go rename to src/crux/agent/kick_outcome_test.go diff --git a/src/pkg/agent/kick_refusal_echo_test.go b/src/crux/agent/kick_refusal_echo_test.go similarity index 100% rename from src/pkg/agent/kick_refusal_echo_test.go rename to src/crux/agent/kick_refusal_echo_test.go diff --git a/src/pkg/agent/kick_restart_recovery_test.go b/src/crux/agent/kick_restart_recovery_test.go similarity index 93% rename from src/pkg/agent/kick_restart_recovery_test.go rename to src/crux/agent/kick_restart_recovery_test.go index 72c2929ba0..c5542f7856 100644 --- a/src/pkg/agent/kick_restart_recovery_test.go +++ b/src/crux/agent/kick_restart_recovery_test.go @@ -17,7 +17,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // forceFastPaneShell pins the test tmux server's default-shell to /bin/sh so @@ -61,6 +61,15 @@ func forceFastPaneShell(t *testing.T) { // so the goroutine is drained before the test's env restores run. func quiesceThenMarkReady(t *testing.T, m *Manager, name string) (wait func()) { t.Helper() + // TestMain's 5s readiness budget leaves less than 1s after the 4s + // quiesce below for tmux to render the marker on a loaded runner. + // These tests cover restart/kick delivery, not readiness timeouts. + // No tests in this package run in parallel; restore the budget so + // negative-path tests keep TestMain's shorter timeout. + origReadyTimeout := cliReadyTimeout + cliReadyTimeout = 15 * time.Second + t.Cleanup(func() { cliReadyTimeout = origReadyTimeout }) + stop := make(chan struct{}) var wg sync.WaitGroup wg.Add(1) diff --git a/src/pkg/agent/kick_transcript.go b/src/crux/agent/kick_transcript.go similarity index 99% rename from src/pkg/agent/kick_transcript.go rename to src/crux/agent/kick_transcript.go index 7326e88afe..e5c88cefa5 100644 --- a/src/pkg/agent/kick_transcript.go +++ b/src/crux/agent/kick_transcript.go @@ -14,7 +14,7 @@ import ( "time" "unicode/utf8" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // Kick log transcript source (hivecommons/hive#11762, #9579). diff --git a/src/pkg/agent/kick_transcript_claude.go b/src/crux/agent/kick_transcript_claude.go similarity index 100% rename from src/pkg/agent/kick_transcript_claude.go rename to src/crux/agent/kick_transcript_claude.go diff --git a/src/pkg/agent/kick_transcript_copilot.go b/src/crux/agent/kick_transcript_copilot.go similarity index 100% rename from src/pkg/agent/kick_transcript_copilot.go rename to src/crux/agent/kick_transcript_copilot.go diff --git a/src/pkg/agent/kick_transcript_test.go b/src/crux/agent/kick_transcript_test.go similarity index 99% rename from src/pkg/agent/kick_transcript_test.go rename to src/crux/agent/kick_transcript_test.go index 1097ee2fe1..9db3b510e5 100644 --- a/src/pkg/agent/kick_transcript_test.go +++ b/src/crux/agent/kick_transcript_test.go @@ -16,7 +16,7 @@ import ( "time" "unicode/utf8" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) const ( diff --git a/src/pkg/agent/kill_floor_guard_test.go b/src/crux/agent/kill_floor_guard_test.go similarity index 100% rename from src/pkg/agent/kill_floor_guard_test.go rename to src/crux/agent/kill_floor_guard_test.go diff --git a/src/pkg/agent/launch_coverage_test.go b/src/crux/agent/launch_coverage_test.go similarity index 98% rename from src/pkg/agent/launch_coverage_test.go rename to src/crux/agent/launch_coverage_test.go index 6f3bd4017d..38c9982cc3 100644 --- a/src/pkg/agent/launch_coverage_test.go +++ b/src/crux/agent/launch_coverage_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/internal/testutil" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" ) // TestStart_CLIAlreadyRunning exercises launchInTmux's early-return branch: the diff --git a/src/pkg/agent/launch_failure_banner_test.go b/src/crux/agent/launch_failure_banner_test.go similarity index 99% rename from src/pkg/agent/launch_failure_banner_test.go rename to src/crux/agent/launch_failure_banner_test.go index 0d4e489789..c33caa8d9a 100644 --- a/src/pkg/agent/launch_failure_banner_test.go +++ b/src/crux/agent/launch_failure_banner_test.go @@ -19,7 +19,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestLaunchFailureBanner(t *testing.T) { diff --git a/src/pkg/agent/launch_scrub_test.go b/src/crux/agent/launch_scrub_test.go similarity index 98% rename from src/pkg/agent/launch_scrub_test.go rename to src/crux/agent/launch_scrub_test.go index 172a5d0c10..6b7e22ee41 100644 --- a/src/pkg/agent/launch_scrub_test.go +++ b/src/crux/agent/launch_scrub_test.go @@ -17,7 +17,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/sandbox" + "github.com/hivecommons/hive/crux/sandbox" ) const ( @@ -115,7 +115,7 @@ func TestSandboxExecutorScrubsLaunchOutput(t *testing.T) { } // agentLaunchScriptPath locates bin/agent-launch.sh relative to this package -// (src/pkg/agent → repo root → bin). +// (src/crux/agent → repo root → bin). func agentLaunchScriptPath(t *testing.T) string { t.Helper() p, err := filepath.Abs(filepath.Join("..", "..", "..", "bin", "agent-launch.sh")) diff --git a/src/pkg/agent/linear_credential_test.go b/src/crux/agent/linear_credential_test.go similarity index 99% rename from src/pkg/agent/linear_credential_test.go rename to src/crux/agent/linear_credential_test.go index d8bb266a46..1ad77ddcdc 100644 --- a/src/pkg/agent/linear_credential_test.go +++ b/src/crux/agent/linear_credential_test.go @@ -3,7 +3,7 @@ package agent import ( "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func linearCredManager(t *testing.T, mode string, cred LinearCredential) (*Manager, *AgentProcess) { diff --git a/src/pkg/agent/login_code.go b/src/crux/agent/login_code.go similarity index 100% rename from src/pkg/agent/login_code.go rename to src/crux/agent/login_code.go diff --git a/src/pkg/agent/login_code_test.go b/src/crux/agent/login_code_test.go similarity index 99% rename from src/pkg/agent/login_code_test.go rename to src/crux/agent/login_code_test.go index 348ee1cd2d..655175d317 100644 --- a/src/pkg/agent/login_code_test.go +++ b/src/crux/agent/login_code_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestValidateLoginCodeRejectsCommandInjection is the security property of the diff --git a/src/pkg/agent/login_expiry_test.go b/src/crux/agent/login_expiry_test.go similarity index 100% rename from src/pkg/agent/login_expiry_test.go rename to src/crux/agent/login_expiry_test.go diff --git a/src/pkg/agent/login_model_403_test.go b/src/crux/agent/login_model_403_test.go similarity index 100% rename from src/pkg/agent/login_model_403_test.go rename to src/crux/agent/login_model_403_test.go diff --git a/src/pkg/agent/manager.go b/src/crux/agent/manager.go similarity index 89% rename from src/pkg/agent/manager.go rename to src/crux/agent/manager.go index b2f81d4136..408f6a7401 100644 --- a/src/pkg/agent/manager.go +++ b/src/crux/agent/manager.go @@ -10,33 +10,31 @@ import ( "sync/atomic" "time" - "github.com/hivecommons/hive/pkg/crux/types" - - "github.com/hivecommons/hive/pkg/claude" - hiveconfig "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/effects" - "github.com/hivecommons/hive/pkg/fleetreport" - "github.com/hivecommons/hive/pkg/pushbroker" - "github.com/hivecommons/hive/pkg/sandbox" - "github.com/hivecommons/hive/pkg/taskmcp" - "github.com/hivecommons/hive/pkg/watchdog" + "github.com/hivecommons/hive/crux" + + "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/effects" + "github.com/hivecommons/hive/crux/pushbroker" + "github.com/hivecommons/hive/crux/sandbox" + "github.com/hivecommons/hive/crux/taskmcp" + "github.com/hivecommons/hive/crux/watchdog" ) -// ProcessState and the State* values are defined in pkg/crux/types as part +// ProcessState and the State* values are defined in pkg/crux as part // of the Crux runtime surface (hivecommons/hive#13536); aliased here. -type ProcessState = types.ProcessState +type ProcessState = crux.ProcessState const ( - StateIdle = types.StateIdle - StateRunning = types.StateRunning - StateStopped = types.StateStopped - StateFailed = types.StateFailed - StatePaused = types.StatePaused + StateIdle = crux.StateIdle + StateRunning = crux.StateRunning + StateStopped = crux.StateStopped + StateFailed = crux.StateFailed + StatePaused = crux.StatePaused ) -// KickRecord is one entry of KickHistory; see types.KickRecord. -type KickRecord = types.KickRecord +// KickRecord is one entry of KickHistory; see crux.KickRecord. +type KickRecord = crux.KickRecord const ( outputBufferCapacity = 500 @@ -200,13 +198,10 @@ const ( tmuxWheelBindingElse = "copy-mode -eH" ) -// BreakerTrigger is the distinct PausedTrigger stamped on every pause the fleet -// breaker performs. It serves two jobs: the audit log attributes the pause to -// the breaker, and ReleaseBreaker uses it as the guard that distinguishes a -// pause the breaker still owns (safe to auto-resume) from one an operator -// re-applied during the breaker window (must stay paused). Anything whose -// current PausedTrigger != BreakerTrigger was last paused by something else. -const BreakerTrigger = "fleet-breaker" +// BreakerTrigger is crux.BreakerTrigger, the PausedTrigger stamped on every +// pause the fleet breaker performs; ReleaseBreaker uses it as the guard that +// distinguishes a pause the breaker still owns from one an operator re-applied. +const BreakerTrigger = crux.BreakerTrigger type AgentProcess struct { Name string @@ -409,11 +404,15 @@ type AgentProcess struct { // the cooldown reset on the next kick. lastTransientNudge time.Time transientNudgesThisKick int - // CI-poll guard (#9673): baseline is the CI-poll command count already in - // scrollback when the kick's first poll tick ran (-1 = not yet captured). - ciPollBaseline int - ciPollNudgeSent bool - CIPollNudges int // total CI-polling nudges sent + // CI-poll guard (#9673): ciPollBaseline is the CI-poll command count already + // in scrollback when the kick's first poll tick ran (-1 = not yet captured); + // ciPollCommandBaseline holds those commands in order. The capture keeps + // only the newest lines, so later polls match the baseline as a prefix of + // the kick's commands instead of subtracting counts (#14065). + ciPollBaseline int + ciPollCommandBaseline []string + ciPollNudgeSent bool + CIPollNudges int // total CI-polling nudges sent // AwaitingCI (#9673 item 3): true while the agent has crossed the // CI-poll nudge threshold this kick AND is currently idle at the prompt // (the same signal nudgeIfPollingCI uses to decide whether to nudge). @@ -516,88 +515,10 @@ type AgentProcess struct { } // ProjectContext holds project-level config injected into agent boot prompts. -type ProjectContext struct { - Org string - Repos []string - PrimaryRepoName string - ACMMLevel int - PRsAllowed bool - PolicyDir string - // GHHost is the bare hostname of the source forge when it is NOT public - // github.com (e.g. "github.ibm.com" for a GHE spoke), derived from the - // configured github.api_url. Exported to agents as GH_HOST so the gh CLI - // targets the right host — without it every agent gh call went to - // api.github.com where the project's repos do not exist (root-caused live - // 2026-08-20: the security agent's issue/PR creation failed silently on - // every GHE-hosted hive). Empty ⇒ public github.com, nothing exported. - GHHost string - // AppAuthoredPRs mirrors config github.app_authored_prs: when true, push- - // capable agents get the App installation token as GITHUB_TOKEN so the GitHub - // MCP server authors PRs/commits as the App bot. Default false → no token is - // injected and behavior is unchanged (opt-in per hive). - AppAuthoredPRs bool - // TaskMCPURL is the hub-served, read-only task context endpoint for - // hub-launched agents. Empty keeps launches byte-for-byte unchanged. It - // carries NO credential: the dashboard bearer must never reach an agent - // (#8348), so the per-launch token comes from TaskMCPLaunchToken instead. - TaskMCPURL string - // TaskMCPLaunchToken mints the credential one launch presents to the task - // MCP endpoint, bound to that launch's scope (task id, repo, agent, - // generation) and revoked when the launch ends. Nil means the URL is - // handed over without a token; the hub then refuses the agent's calls - // rather than falling back to a wider credential. - TaskMCPLaunchToken func(scope taskmcp.LaunchScope) string - // AppBotLogin and AppBotUserID identify the installed GitHub App's bot - // account. When AppAuthoredPRs is on, push-capable agents use this bot's - // noreply address ("+@users.noreply.github.com") for git - // author/committer email so GitHub links their fixer commits to the App - // while the commit name and sign-off still name the lane. AppBotUserID - // resolves the bot's GitHub USER id (not the App id, which GitHub never - // links) and is called at each launch so a failed lookup (0) only costs - // that launch the linkage; nil means unknown. - AppBotLogin string - AppBotUserID func() int64 - // BobSessionPrefix mirrors governor.bob.session_prefix. Empty is - // default-off; bob launch commands remain byte-identical unless it or a - // per-agent bob.session_label is configured. - BobSessionPrefix string - // RepoPaused reports whether a repo carries an operator pause (#6203). It is - // a live predicate rather than a snapshot list because ProjectContext is - // built once at boot and a pause is taken mid-run — a repo frozen for a - // release must drop out of the next agent's scope without a hive restart. - // Nil means nothing is paused, which is every hive that does not use the - // feature and every test that constructs a ProjectContext literal. - RepoPaused func(repo string) bool - // AgentRepos returns the repositories one agent serves (#6204), and - // AgentPrimaryRepo the repo its prompts should treat as "the" repo. Both - // are live functions rather than a snapshot map because ProjectContext is - // built once at boot while a scope is edited mid-run — a specialist - // re-scoped in the dashboard must reach the next agent launch without a - // hive restart. - // - // Nil means no scoping: every agent gets Repos and PrimaryRepo, which is - // what every hive did before this field and what every test that builds a - // ProjectContext literal still gets. - AgentRepos func(agent string) []string - AgentPrimaryRepo func(agent string) string - // ExperimentEnv is the hive-wide experimentation environment (RFC - // #11585 part 5, pkg/nousengine.HiveEnv): the Nous dispatcher and data - // root as the hive resolved them (env > yaml > default). It is exported - // ONLY to agents whose role is config.RoleExperimenter — alongside that - // agent's own engine name — so the strategist and every other role - // launch byte-for-byte as before. Nil exports only the engine name. - ExperimentEnv map[string]string -} - -func (p ProjectContext) PrimaryRepo() string { - if strings.TrimSpace(p.PrimaryRepoName) != "" { - return strings.TrimPrefix(p.PrimaryRepoName, p.Org+"/") - } - if len(p.Repos) > 0 { - return strings.TrimPrefix(p.Repos[0], p.Org+"/") - } - return "" -} +// It is defined in the crux root package (crux.ProjectContext) so a host can +// build one, and crux.Process can be its own type, without importing this +// package. +type ProjectContext = crux.ProjectContext type Manager struct { agents map[string]*AgentProcess @@ -618,7 +539,7 @@ type Manager struct { // statusSnapshots backs GetStatusFast. Own mutex, NEVER m.mu — its whole // purpose is to be readable while m.mu is held by a restart (#7417). statusSnapMu sync.RWMutex - statusSnaps map[string]*AgentProcess + statusSnaps map[string]*crux.Process // consentWedges records consent-screen restarts for the heartbeat's // ConsentWedged signal (#5577). Own mutex, NEVER m.mu — the recording // sites can run with m.mu held. Zero value ready. @@ -652,6 +573,7 @@ type Manager struct { uidMap *UIDMap uidMapPath string appAuth AppTokenMinter + appNotInstalledWarned sync.Map // agent name -> struct{}: not-installed mint failure already logged agentMint AgentMintIssuer // optional, opt-in mint credential (nil ⇒ off) // bobAPIKeyResolver resolves the IBM bobshell API key at LAUNCH time (not @@ -697,7 +619,7 @@ type Manager struct { // advisorRolesResolver returns the live model_roles map the OMP adapter // projects as a --config overlay at launch (#9723). Same discipline as // advisorEnabledResolver. - advisorRolesResolver atomic.Pointer[func() map[string]hiveconfig.ModelRole] + advisorRolesResolver atomic.Pointer[func() map[string]config.ModelRole] // repoAutoMergeEnabled resolves the live per-repo auto-merge switch used by // the merge relay. Nil preserves the historic default-on behavior. repoAutoMergeEnabled atomic.Pointer[func(repo string) bool] @@ -801,6 +723,10 @@ type Manager struct { // recordPromptCallback: read on paths that hold m.mu. Nil falls back to // the OpenTelemetry global tracer; see SetSpanStarter. spanStarter atomic.Pointer[SpanStarter] + // conditionReporter is the host sink for self-reported error conditions; + // same atomic.Pointer discipline. Nil drops observations. See + // SetConditionReporter. + conditionReporter atomic.Pointer[crux.ConditionReporter] // deadSessionRecoveryOwnedElsewhere is set when the watchdog reconciler // owns restarting missing-session / bare-pane agents (RFC #4665), so this // manager's crash loop observes those two conditions without restarting @@ -1201,13 +1127,13 @@ func (m *Manager) Start(ctx context.Context, name string) error { if backend == "copilot" { m.runCopilotCacheJanitor() if reason, full := copilotDiskFullCondition(); full { - fleetreport.Observe(fleetreport.ClassDataDiskPressure, DataDiskPressureKey, "data disk pressure: "+reason, "") + m.observeCondition(crux.ClassDataDiskPressure, DataDiskPressureKey, "data disk pressure: "+reason, "") if err := m.Pause(name, DiskFullTrigger, reason); err != nil { return err } return fmt.Errorf("%s", reason) } - fleetreport.Clear(fleetreport.ClassDataDiskPressure, DataDiskPressureKey) + m.clearCondition(crux.ClassDataDiskPressure, DataDiskPressureKey) } // Runs with m.mu RELEASED — see mintAgentTokenUnlocked for why holding @@ -1650,7 +1576,7 @@ func (m *Manager) ProviderErrorBackoffRemaining(name string) (time.Duration, str // This does NOT fix the lock hold itself — restartWithReason still needs to be // phased so the relaunch happens outside m.mu. It stops that hold from being // visible to readers who never needed to be serialized against it. -func (m *Manager) GetStatusFast(name string) (*AgentProcess, error) { +func (m *Manager) GetStatusFast(name string) (*crux.Process, error) { if m.mu.TryRLock() { agent, ok := m.agents[name] if !ok { @@ -1662,12 +1588,11 @@ func (m *Manager) GetStatusFast(name string) (*AgentProcess, error) { m.statusSnapMu.Lock() if m.statusSnaps == nil { - m.statusSnaps = make(map[string]*AgentProcess) + m.statusSnaps = make(map[string]*crux.Process) } - // Cache the same pointer we return: AgentProcess contains locks - // (paneMu), so copying the value trips govet copylocks. Snapshots - // are read-only display values by contract; a later refresh - // replaces the map entry rather than mutating this one. + // Cache the same pointer we return. Snapshots are read-only + // display values by contract; a later refresh replaces the map + // entry rather than mutating this one. m.statusSnaps[name] = &snap m.statusSnapMu.Unlock() @@ -1678,14 +1603,13 @@ func (m *Manager) GetStatusFast(name string) (*AgentProcess, error) { cached, ok := m.statusSnaps[name] m.statusSnapMu.RUnlock() if ok && cached != nil { - // Returned as-is (no copy) for the same copylocks reason; stale - // snapshots are read-only. + // Returned as-is (no copy); stale snapshots are read-only. return cached, nil } return nil, fmt.Errorf("agent %s status unavailable: manager busy", name) } -func (m *Manager) GetStatus(name string) (*AgentProcess, error) { +func (m *Manager) GetStatus(name string) (*crux.Process, error) { m.mu.RLock() defer m.mu.RUnlock() @@ -1738,12 +1662,12 @@ func (m *Manager) CountAgentsWithModel() int { // StartTranscriptActivityRefresh on its own goroutine, never walked here, // so callers on a deadline (the hub heartbeat collect, the dashboard status // build) cannot be stalled by a slow /data. -func (m *Manager) AllStatuses() map[string]*AgentProcess { +func (m *Manager) AllStatuses() map[string]*crux.Process { now := time.Now() m.mu.Lock() defer m.mu.Unlock() - result := make(map[string]*AgentProcess, len(m.agents)) + result := make(map[string]*crux.Process, len(m.agents)) for k, v := range m.agents { m.refreshBusyVisibilityLocked(v, now) snap := v.snapshot() @@ -1886,73 +1810,6 @@ func (m *Manager) TmuxSession(name string) string { return agent.tmuxSession } -// ActiveRepos is Repos minus the operator-paused entries: the repos an agent -// should be told to work. -// -// PrimaryRepo deliberately does NOT filter. The primary repo is the hive's -// identity — what $HIVE_REPO names and what an agent's workdir is a checkout -// of — and re-pointing it at a different repository because the operator -// froze the first one would be a far larger surprise than the pause itself. -// A paused primary simply stops appearing in the work scope. -func (p ProjectContext) ActiveRepos() []string { - return p.unpaused(p.Repos) -} - -// ReposFor returns the repositories the named agent serves, falling back to the -// hive-wide list when the agent is unscoped or no scope function is wired. -func (p ProjectContext) ReposFor(agent string) []string { - if p.AgentRepos == nil { - return p.Repos - } - if repos := p.AgentRepos(agent); repos != nil { - return repos - } - return p.Repos -} - -// ActiveReposFor is ReposFor minus the operator-paused entries: the repos the -// named agent should actually be handed work on. -// -// The two narrowings compose rather than override. A scope says which repos an -// agent is FOR (#6204); a pause says which repos are open for work at all -// (#6203). An agent scoped to a repo the operator has since paused is handed -// nothing for it, and unpausing restores it without touching the scope. -func (p ProjectContext) ActiveReposFor(agent string) []string { - return p.unpaused(p.ReposFor(agent)) -} - -// unpaused drops the operator-paused entries from repos, preserving order. -func (p ProjectContext) unpaused(repos []string) []string { - if p.RepoPaused == nil { - return repos - } - out := make([]string, 0, len(repos)) - for _, r := range repos { - if p.RepoPaused(r) { - continue - } - out = append(out, r) - } - return out -} - -// PrimaryRepoFor returns the repo the named agent's prompts should treat as -// "the" repo — what $HIVE_REPO names. -// -// Unlike PrimaryRepo() this is per-agent, and it has to be: the shipped -// templates say `gh issue create --repo "$HIVE_REPO"`, so handing a specialist -// scoped away from the hive primary the hive primary would aim every example it -// has at a repository the proxy then refuses. An unscoped agent is unaffected. -// Like PrimaryRepo it does not filter paused repos — see ActiveRepos. -func (p ProjectContext) PrimaryRepoFor(agent string) string { - if p.AgentPrimaryRepo != nil { - if repo := strings.TrimPrefix(strings.TrimSpace(p.AgentPrimaryRepo(agent)), p.Org+"/"); repo != "" { - return repo - } - } - return p.PrimaryRepo() -} - func (m *Manager) bobPreflightUnlocked(agent *AgentProcess) { if agent == nil { return diff --git a/src/pkg/agent/manager3_coverage_test.go b/src/crux/agent/manager3_coverage_test.go similarity index 92% rename from src/pkg/agent/manager3_coverage_test.go rename to src/crux/agent/manager3_coverage_test.go index 2a5b01419e..5c84861f83 100644 --- a/src/pkg/agent/manager3_coverage_test.go +++ b/src/crux/agent/manager3_coverage_test.go @@ -1,15 +1,18 @@ package agent import ( + "bytes" "context" "fmt" + "log/slog" "os" "path/filepath" "strings" "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" ) // --------------------------------------------------------------------------- @@ -19,12 +22,16 @@ import ( type fakeMinter struct { calls int fail bool + err error lastTier string } func (f *fakeMinter) WriteAgentToken(ctx context.Context, agentName, tier string, agentUID int) error { f.calls++ f.lastTier = tier + if f.err != nil { + return f.err + } if f.fail { return context.Canceled } @@ -523,3 +530,36 @@ func TestAgentEnvPairs_DirectBackendBypassesProxyForLoopback(t *testing.T) { } } } + +func TestMintAgentTokenUnlocked_AppNotInstalledWarnsOnce(t *testing.T) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + m := NewManager(map[string]config.AgentConfig{"cxa": {Backend: "claude"}}, logger, ProjectContext{}) + fm := &fakeMinter{err: fmt.Errorf("minting scoped token for cxa: %w", crux.ErrGitHubAppNotInstalled)} + m.SetAppAuth(fm) + rec := recordConditions(t, m) + m.mu.Lock() + agent := m.agents["cxa"] + agent.UID = 2001 + m.mu.Unlock() + + m.mintAgentTokenUnlocked(context.Background(), agent) + m.mintAgentTokenUnlocked(context.Background(), agent) + + if got := strings.Count(buf.String(), "github app installation id is unset"); got != 1 { + t.Errorf("unset-installation warning logged %d times, want 1", got) + } + if got := len(rec.observations(crux.ClassGitHubAppNotInstalled)); got != 2 { + t.Errorf("not-installed observations = %d, want 2", got) + } + if got := len(rec.observations(crux.ClassGitHubAppTokenMintFailed)); got != 0 { + t.Errorf("generic mint-failed observations = %d, want 0", got) + } + + // A successful mint re-arms the warning and clears the condition. + fm.err = nil + m.mintAgentTokenUnlocked(context.Background(), agent) + if len(rec.clears(crux.ClassGitHubAppNotInstalled)) != 1 { + t.Error("successful mint did not clear the not-installed condition") + } +} diff --git a/src/pkg/agent/manager_authz_count_test.go b/src/crux/agent/manager_authz_count_test.go similarity index 99% rename from src/pkg/agent/manager_authz_count_test.go rename to src/crux/agent/manager_authz_count_test.go index ee685e33ff..ca27ce9d9d 100644 --- a/src/pkg/agent/manager_authz_count_test.go +++ b/src/crux/agent/manager_authz_count_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // acmmLevelPushCapable is an ACMM level at which the "quality" agent defaults diff --git a/src/pkg/agent/manager_consent.go b/src/crux/agent/manager_consent.go similarity index 89% rename from src/pkg/agent/manager_consent.go rename to src/crux/agent/manager_consent.go index 7434d8344c..eed12c8b98 100644 --- a/src/pkg/agent/manager_consent.go +++ b/src/crux/agent/manager_consent.go @@ -240,6 +240,30 @@ func (m *Manager) confirmMenuOption(agent *AgentProcess, title, want, navKey str return false } +// acceptBypassConsentForKick answers a "Bypass Permissions mode" consent +// screen that a kick found in front of the CLI. Restarting instead lands on +// the same screen every time, so kicks never arrive (#14307). Returns true +// once the screen was accepted and the pane no longer shows a consent screen; +// false leaves recovery to the caller's restart. The caller must not hold +// m.mu: menu navigation pauses between keystrokes. +func (m *Manager) acceptBypassConsentForKick(agent *AgentProcess, pane string) bool { + if !strings.Contains(pane, bypassConsentTitle) || !strings.Contains(pane, bypassConsentAcceptOption) { + return false + } + m.logger.Info("accepting bypass-permissions consent before kick", "agent", agent.Name) + if !m.confirmMenuOption(agent, bypassConsentTitle, bypassConsentAcceptOption, "Down") { + return false + } + return !paneShowsConsentScreen(m.captureVisiblePaneForAgent(agent)) +} + +// claudeConsentBackend reports whether an agent's backend runs Claude Code +// with --dangerously-skip-permissions and so can land on the bypass consent +// screen: the claude CLI backend itself and every inference backend. +func claudeConsentBackend(backend string) bool { + return backend == "claude" || IsInferenceBackend(backend) +} + const ( // consentStuckGracePeriod is how long a consent screen must stay visible // across watcher passes before the agent counts as stuck. The launch-time @@ -262,7 +286,8 @@ func (m *Manager) clearConsentTracking(name string) { } } -// dismissConsentIfStuck re-runs dismissInferencePrompts for an inference agent +// dismissConsentIfStuck re-runs dismissInferencePrompts for a Claude Code agent +// (claude or inference backend) // whose pane has shown a consent screen for longer than the grace period, // subject to a per-agent cooldown. Called from the watcher loop // (CheckAndRestartCrashedAgents) so an agent that lands on a consent screen @@ -289,7 +314,7 @@ func (m *Manager) dismissConsentIfStuck(name string) { agent.lastConsentDismiss = now m.mu.Unlock() - m.logger.Warn("inference agent stuck on consent screen, re-running prompt dismissal", + m.logger.Warn("agent stuck on consent screen, re-running prompt dismissal", "name", name, "stuck_seconds", int(stuckFor.Seconds())) go m.dismissInferencePrompts(agent) } diff --git a/src/pkg/agent/manager_copilot_auth.go b/src/crux/agent/manager_copilot_auth.go similarity index 98% rename from src/pkg/agent/manager_copilot_auth.go rename to src/crux/agent/manager_copilot_auth.go index dff4162db6..075033bf4c 100644 --- a/src/pkg/agent/manager_copilot_auth.go +++ b/src/crux/agent/manager_copilot_auth.go @@ -2,6 +2,8 @@ package agent import ( "time" + + "github.com/hivecommons/hive/crux/config" ) const ( @@ -12,7 +14,7 @@ const ( // otherwise ignores the model entirely; "low" is the effort agy defaults // to on its own, so this makes the configured model take effect without // changing behaviour. - agyDefaultEffort = "low" + agyDefaultEffort = config.AgyDefaultEffort tokenRestartCooldownSec = 60 // minimum seconds between token-triggered restarts per agent // rateLimitRestartCooldownSec paces relaunches of an agent parked on the diff --git a/src/pkg/agent/manager_copilot_transient_test.go b/src/crux/agent/manager_copilot_transient_test.go similarity index 100% rename from src/pkg/agent/manager_copilot_transient_test.go rename to src/crux/agent/manager_copilot_transient_test.go diff --git a/src/pkg/agent/manager_coverage2_test.go b/src/crux/agent/manager_coverage2_test.go similarity index 99% rename from src/pkg/agent/manager_coverage2_test.go rename to src/crux/agent/manager_coverage2_test.go index 387b5d7d59..f803e38ea3 100644 --- a/src/pkg/agent/manager_coverage2_test.go +++ b/src/crux/agent/manager_coverage2_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // --------------------------------------------------------------------------- diff --git a/src/pkg/agent/manager_coverage_test.go b/src/crux/agent/manager_coverage_test.go similarity index 99% rename from src/pkg/agent/manager_coverage_test.go rename to src/crux/agent/manager_coverage_test.go index 5bcf87c31b..f1f47babba 100644 --- a/src/pkg/agent/manager_coverage_test.go +++ b/src/crux/agent/manager_coverage_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // --------------------------------------------------------------------------- diff --git a/src/pkg/agent/manager_credentials.go b/src/crux/agent/manager_credentials.go similarity index 98% rename from src/pkg/agent/manager_credentials.go rename to src/crux/agent/manager_credentials.go index 001acab572..5fb1a2d898 100644 --- a/src/pkg/agent/manager_credentials.go +++ b/src/crux/agent/manager_credentials.go @@ -7,8 +7,9 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/claude" - "github.com/hivecommons/hive/pkg/logscrub" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/logscrub" ) // sharedCopilotConfigPath and sharedClaudeCredentialPath are vars (not consts) @@ -23,7 +24,7 @@ var ( // CopilotUserTokenPath is where the dashboard's device-flow login persists // the Copilot OAuth token; injected into agents as COPILOT_GITHUB_TOKEN. -const CopilotUserTokenPath = "/data/copilot-user-token" +const CopilotUserTokenPath = crux.CopilotUserTokenPath var copilotUserTokenWatchPath = CopilotUserTokenPath diff --git a/src/pkg/agent/manager_env.go b/src/crux/agent/manager_env.go similarity index 96% rename from src/pkg/agent/manager_env.go rename to src/crux/agent/manager_env.go index 1f3a944319..4b59488db9 100644 --- a/src/pkg/agent/manager_env.go +++ b/src/crux/agent/manager_env.go @@ -11,12 +11,11 @@ import ( "strconv" "strings" - advisorpkg "github.com/hivecommons/hive/pkg/advisor" - hiveconfig "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/crux/types" - "github.com/hivecommons/hive/pkg/gitidentity" - "github.com/hivecommons/hive/pkg/jev" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/advisorhook" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/gitidentity" + "github.com/hivecommons/hive/crux/jev" ) // buildCommit is the running binary's ldflags-stamped short commit, recorded @@ -291,8 +290,10 @@ func (m *Manager) agentEnvPairs(agent *AgentProcess) []agentEnvPair { proxyURL := ProxyURL() vars = append(vars, agentEnvPair{"HTTPS_PROXY", proxyURL, false}) vars = append(vars, agentEnvPair{"HTTP_PROXY", proxyURL, false}) - // Loopback must bypass the proxy for every backend so a repo's own tests - // see real connection errors instead of the proxy's 502. + // Loopback must never traverse the hive proxy for any backend: an agent + // polling a local dev server would otherwise get proxy-side dial errors + // (#14076), and a repo's own tests must see real connection errors + // instead of the proxy's 502. vars = append(vars, agentEnvPair{"NO_PROXY", loopbackNoProxy, false}) vars = append(vars, agentEnvPair{"no_proxy", loopbackNoProxy, false}) vars = append(vars, agentEnvPair{"HIVE_PROXY_AGENT", agent.Name, false}) @@ -391,7 +392,7 @@ func (m *Manager) agentEnvPairs(agent *AgentProcess) []agentEnvPair { // per-agent tier-SCOPED App token, and refreshAgentTokens re-pushes it hourly // so it never goes stale. if m.project.AppAuthoredPRs && m.appAuth != nil && agent.UID > 0 && m.agentMode(agent).CanPush() { - if data, err := os.ReadFile(types.AgentTokenCachePath(types.DefaultAgentTokenCacheDir, agent.Name)); err == nil { + if data, err := os.ReadFile(crux.AgentTokenCachePath(crux.DefaultAgentTokenCacheDir, agent.Name)); err == nil { if tok := strings.TrimSpace(string(data)); tok != "" { vars = append(vars, agentEnvPair{"GITHUB_TOKEN", tok, true}) } @@ -480,7 +481,7 @@ func (m *Manager) agentEnvPairs(agent *AgentProcess) []agentEnvPair { // the advisor is on for the agent; it carries no credential — the hive // names the caller from the socket UID, exactly like the Jev endpoint. if m.advisorEnabledFor(agent.Name) { - vars = append(vars, agentEnvPair{advisorpkg.EndpointEnvVar, advisorpkg.DefaultEndpoint, false}) + vars = append(vars, agentEnvPair{advisorhook.EndpointEnvVar, advisorhook.DefaultEndpoint, false}) } // Repeat-failure check (#14550): BASH_ENV names the Hive-owned record // script and SHELL is forced to bash, only while the check is on and only @@ -497,7 +498,7 @@ func (m *Manager) agentEnvPairs(agent *AgentProcess) []agentEnvPair { // GIT_SSL_CAINFO only — NOT SSL_CERT_FILE (that breaks Copilot API TLS) vars = append(vars, agentEnvPair{"GIT_SSL_CAINFO", proxyCACertPath, false}) if agent.UID > 0 { - vars = append(vars, agentEnvPair{"HIVE_AGENT_TOKEN_CACHE", types.AgentTokenCachePath(types.DefaultAgentTokenCacheDir, agent.Name), false}) + vars = append(vars, agentEnvPair{"HIVE_AGENT_TOKEN_CACHE", crux.AgentTokenCachePath(crux.DefaultAgentTokenCacheDir, agent.Name), false}) } if agent.UID > 0 { // Per-UID agents get a per-agent HOME (#4596) — AgentHome is the single @@ -569,7 +570,7 @@ func (m *Manager) agentEnvPairs(agent *AgentProcess) []agentEnvPair { // Advisor lane (#9724): Copilot CLI has no hooks launch flag, so hive points // COPILOT_HOME at the agent's own hive-provisioned .copilot directory, whose // hooks directory carries the advisor's hook file. - if backend == hiveconfig.AdvisorCopilotBackend && m.advisorHooksActive(agent, backend) { + if backend == config.AdvisorCopilotBackend && m.advisorHooksActive(agent, backend) { vars = append(vars, agentEnvPair{"COPILOT_HOME", advisorCopilotHome(agent), false}) } @@ -594,17 +595,17 @@ func (m *Manager) agentEnvPairs(agent *AgentProcess) []agentEnvPair { // experimentEnvPairs exports the experimentation-engine settings to an agent // carrying the experimenter role, and nothing to any other agent: the -// agent's own engine (hiveconfig.NousEngineEnv, from its config) and the hive's +// agent's own engine (config.NousEngineEnv, from its config) and the hive's // resolved dispatcher and data root (ProjectContext.ExperimentEnv). Keys are // sorted so the launch command is deterministic. func (m *Manager) experimentEnvPairs(agent *AgentProcess) []agentEnvPair { if agent == nil || !agent.Config.IsExperimenter() { return nil } - pairs := []agentEnvPair{{hiveconfig.NousEngineEnv, agent.Config.ExperimentEngine(), false}} + pairs := []agentEnvPair{{config.NousEngineEnv, agent.Config.ExperimentEngine(), false}} keys := make([]string, 0, len(m.project.ExperimentEnv)) for k := range m.project.ExperimentEnv { - if k == hiveconfig.NousEngineEnv { + if k == config.NousEngineEnv { continue } keys = append(keys, k) diff --git a/src/pkg/agent/manager_env_git_identity_test.go b/src/crux/agent/manager_env_git_identity_test.go similarity index 99% rename from src/pkg/agent/manager_env_git_identity_test.go rename to src/crux/agent/manager_env_git_identity_test.go index 3311ab7bba..b87a0f37d6 100644 --- a/src/pkg/agent/manager_env_git_identity_test.go +++ b/src/crux/agent/manager_env_git_identity_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func gitIdentityPairs(pairs []agentEnvPair) map[string]string { diff --git a/src/pkg/agent/manager_extra_test.go b/src/crux/agent/manager_extra_test.go similarity index 99% rename from src/pkg/agent/manager_extra_test.go rename to src/crux/agent/manager_extra_test.go index 465895c067..cc5c1d1fdd 100644 --- a/src/pkg/agent/manager_extra_test.go +++ b/src/crux/agent/manager_extra_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestAddAgent(t *testing.T) { diff --git a/src/pkg/agent/manager_helper_paths_test.go b/src/crux/agent/manager_helper_paths_test.go similarity index 99% rename from src/pkg/agent/manager_helper_paths_test.go rename to src/crux/agent/manager_helper_paths_test.go index b8d78bcaea..d4a91b5836 100644 --- a/src/pkg/agent/manager_helper_paths_test.go +++ b/src/crux/agent/manager_helper_paths_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // Path/branch coverage for small deterministic manager helpers whose error diff --git a/src/pkg/agent/manager_homes.go b/src/crux/agent/manager_homes.go similarity index 100% rename from src/pkg/agent/manager_homes.go rename to src/crux/agent/manager_homes.go diff --git a/src/pkg/agent/manager_inference_home_anchor_test.go b/src/crux/agent/manager_inference_home_anchor_test.go similarity index 100% rename from src/pkg/agent/manager_inference_home_anchor_test.go rename to src/crux/agent/manager_inference_home_anchor_test.go diff --git a/src/pkg/agent/manager_kick.go b/src/crux/agent/manager_kick.go similarity index 88% rename from src/pkg/agent/manager_kick.go rename to src/crux/agent/manager_kick.go index 3a35f37504..54089300cb 100644 --- a/src/pkg/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -7,11 +7,12 @@ import ( "os" "path/filepath" "regexp" + "slices" "strings" "time" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/fleetreport" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" "go.opentelemetry.io/otel/attribute" ) @@ -104,7 +105,7 @@ func notRunningReason(agent *AgentProcess) string { // already audits. Only kicks carrying it may be superseded by a queued // mention: the governor re-sends a cadence kick on its next tick, whereas any // other caller's message is one-shot (#12982). -const GovernorCadenceKickSource = "governor-eval" +const GovernorCadenceKickSource = crux.GovernorCadenceKickSource func (m *Manager) SendKick(name string, message string) error { return m.SendKickWithSource(name, message, "") @@ -164,16 +165,21 @@ func (m *Manager) SendKickWithSource(name string, message string, source string) // (observed live: "-bash: NEVER: command not found"). pane := m.captureVisiblePaneForAgent(agent) if !paneHasCLIMarker(pane) || paneShowsConsentScreen(pane) { - m.logger.Warn("agent CLI crashed or stuck on consent screen, restarting before kick", - "name", name, "consent_screen", paneShowsConsentScreen(pane)) + consentScreen := paneShowsConsentScreen(pane) m.mu.Unlock() - if err := m.restartForKick(context.Background(), name); err != nil { - m.mu.Lock() - return fmt.Errorf("failed to restart crashed agent %s: %w", name, err) - } - if !m.waitForCLIReadyForAgent(agent) { - m.mu.Lock() - return fmt.Errorf("agent %s CLI did not become ready after restart", name) + // Answer the bypass consent screen in place: a restart relaunches + // into the same screen and the kick never lands (#14307). + if !consentScreen || !m.acceptBypassConsentForKick(agent, pane) { + m.logger.Warn("agent CLI crashed or stuck on consent screen, restarting before kick", + "name", name, "consent_screen", consentScreen) + if err := m.restartForKick(context.Background(), name); err != nil { + m.mu.Lock() + return fmt.Errorf("failed to restart crashed agent %s: %w", name, err) + } + if !m.waitForCLIReadyForAgent(agent) { + m.mu.Lock() + return fmt.Errorf("agent %s CLI did not become ready after restart", name) + } } m.mu.Lock() agent, ok = m.agents[name] @@ -470,6 +476,7 @@ func (m *Manager) recordDeliveredKickLocked(agent *AgentProcess, message, trigge agent.transientNudgesThisKick = 0 agent.lastTransientNudge = time.Time{} agent.ciPollBaseline = -1 + agent.ciPollCommandBaseline = nil agent.ciPollNudgeSent = false agent.AwaitingCI = false @@ -942,7 +949,7 @@ func paneAfterKickBaseline(pane, baseline string) string { // right after delivery. Caller must hold m.mu. func (m *Manager) recordInferenceKick(agent *AgentProcess, at time.Time) { visible := m.captureVisiblePaneForAgent(agent) - creditStallRecoveryLocked(agent, paneContentHash(visible)) + m.creditStallRecoveryLocked(agent, paneContentHash(visible)) agent.lastInferKickAt = at agent.lastInferKickPane = paneContentHash(visible) agent.lastInferKickVisible = visible @@ -957,13 +964,13 @@ func (m *Manager) recordInferenceKick(agent *AgentProcess, at time.Time) { // outstanding and the pane has moved off the hash recorded at the previous // kick, then clears the armed state. Called before a new kick or a // provider-error disarm would reset it unseen. Caller must hold m.mu. -func creditStallRecoveryLocked(agent *AgentProcess, paneHash string) { +func (m *Manager) creditStallRecoveryLocked(agent *AgentProcess, paneHash string) { if !agent.stallNudgeSent { return } if agent.lastInferKickPane != "" && paneHash != agent.lastInferKickPane { agent.StallRecoveries++ - fleetreport.Clear(fleetreport.ClassAgentStall, agent.Name) + m.clearCondition(crux.ClassAgentStall, agent.Name) } agent.stallNudgeSent = false } @@ -973,19 +980,19 @@ func creditStallRecoveryLocked(agent *AgentProcess, paneHash string) { const stallNudgeTailLines = 15 // observeStallNudgeLocked self-reports a swallowed "continue" nudge -// (#13470): the tracker applies the AgentStallNudgeStreak threshold and a +// (#13470): the host reporter applies its nudge-streak threshold and a // recovery (creditStallRecoveryLocked / the pane moving after a nudge) // clears the condition, so the count is consecutive unrecovered nudges. // Numbers and timestamps in the message are normalised out of the // fingerprint, so one issue per hive version collects every agent. Caller // holds m.mu. -func observeStallNudgeLocked(agent *AgentProcess, pane string, now time.Time) { +func (m *Manager) observeStallNudgeLocked(agent *AgentProcess, pane string, now time.Time) { sinceKick := now.Sub(agent.lastInferKickAt).Round(time.Minute) msg := fmt.Sprintf("agent stalled after kick: %d continue nudge(s) sent without the pane moving (last kick %s, %s ago; pane frozen at hash %s)", agent.StallNudges, agent.lastInferKickAt.UTC().Format(time.RFC3339), roundDurationForHumans(sinceKick), agent.lastInferKickPane) diag := fmt.Sprintf("agent=%s lane=%s backend=%s stall_nudges=%d stall_recoveries=%d kick_stall_timeout=%s\n--- pane tail (%d lines) ---\n%s", agent.Name, agent.Config.Role, effectiveBackend(agent), agent.StallNudges, agent.StallRecoveries, inferenceKickStallTimeout, stallNudgeTailLines, paneTail(pane, stallNudgeTailLines)) - fleetreport.Observe(fleetreport.ClassAgentStall, agent.Name, msg, diag) + m.observeCondition(crux.ClassAgentStall, agent.Name, msg, diag) } // nudgeIfKickStalled watches an inference agent after a kick and corrects @@ -1017,7 +1024,7 @@ func (m *Manager) nudgeIfKickStalled(name, pane string) { } sinceKick := now.Sub(agent.lastInferKickAt) if match, ok := classifyProviderError(paneAfterKickBaseline(pane, agent.lastInferKickVisible)); ok { - creditStallRecoveryLocked(agent, paneContentHash(pane)) + m.creditStallRecoveryLocked(agent, paneContentHash(pane)) backoff := m.markProviderErrorLocked(agent, match, now) attempt := agent.providerErrorBackoffAttempt m.mu.Unlock() @@ -1041,7 +1048,7 @@ func (m *Manager) nudgeIfKickStalled(name, pane string) { agent.stallNudgeSent = true agent.StallNudges++ totalNudges := agent.StallNudges - observeStallNudgeLocked(agent, pane, now) + m.observeStallNudgeLocked(agent, pane, now) m.mu.Unlock() m.logger.Warn("inference agent stalled after kick, sending continue nudge", @@ -1058,7 +1065,7 @@ func (m *Manager) nudgeIfKickStalled(name, pane string) { // The pane moved after a stall nudge: the nudge recovered the stall. agent.StallRecoveries++ agent.stallNudgeSent = false - fleetreport.Clear(fleetreport.ClassAgentStall, agent.Name) + m.clearCondition(crux.ClassAgentStall, agent.Name) } // The pane moved since the kick — the CLI consumed it and the response @@ -1199,8 +1206,10 @@ const ( // pkg/agent TestMain can shrink them for the whole suite: they accumulate to // well over the default 10m `go test` timeout when 440 tests pay production // pacing (1-3s sleeps, 60-120s poll deadlines) against stub CLIs that render -// instantly. Production values are unchanged — nothing outside TestMain may -// mutate them. Relationships the values encode (bobInputHandlerSettleDelay +// instantly. Production values are unchanged. Outside TestMain, a test may +// override one for its own body only if it restores the original via +// t.Cleanup (or defer); this is safe because no test in the package uses +// t.Parallel. Relationships the values encode (bobInputHandlerSettleDelay // distinct from every tmux-pacing delay, and far below inputPromptTimeout) are // pinned by TestBobSettleDelay_* and must be preserved by any test override. var ( @@ -1487,8 +1496,11 @@ const ( // have run before the nudge is also self-reported to the fleet (#13863). // A lone poll past the wall clock is a normal post-push check: the agent // is still nudged in-session, but the fleet only hears about repeated - // polling. - ciPollReportMinPolls = 2 + // polling. The kick text allows up to two status checks on the same run + // (e.g. diagnosing a failed check), so two polls followed by stopping is + // permitted behaviour, not a loop; report from 3, matching + // ciPollNudgeThreshold (#14327). + ciPollReportMinPolls = 3 ciPollNudgeMessage = "Stop polling CI. Do not run gh run watch/view, gh pr checks, or gh pr view --json statusCheckRollup again this turn: leave the PR as is and move to the next work-list item. The automerge sweep handles green PRs." ) @@ -1521,22 +1533,100 @@ var ciPollCommandRe = regexp.MustCompile(`\bgh (?:run (?:watch|view|list)\b|pr c // polling commands exactly this way. var ciPollProhibitionRe = regexp.MustCompile("(?i)\\b(?:no|not|never|don't|avoid|without)\\s+(?:\\w+\\s+)?[`'\"]*$") -// countCIPollCommands counts CI-status commands in pane, skipping mentions +// ciLogReadRe identifies diagnostic log reads, rather than CI-status polls. +var ciLogReadRe = regexp.MustCompile(`^gh run view\b[^\n;|&]*\s--log(?:-failed)?(?:=true)?(?:\s|$)`) + +// ciPollCommands returns each CI-status command in pane as its normalized +// command line (from the command name to end of line), skipping mentions // that are prohibitions quoted from the kick prompt or policy: those landed // in the pane right after the kick and read as poll commands issued 0s in. -func countCIPollCommands(pane string) int { +func ciPollCommands(pane string) []string { pane = stripExplainLines(pane) - n := 0 + var cmds []string for _, loc := range ciPollCommandRe.FindAllStringIndex(pane, -1) { lineStart := strings.LastIndex(pane[:loc[0]], "\n") + 1 if ciPollProhibitionRe.MatchString(pane[lineStart:loc[0]]) { continue } - n++ + line, _, _ := strings.Cut(pane[loc[0]:], "\n") + // Failure diagnosis can revisit the same log while comparing jobs or + // extracting evidence. It does not wait for a check to finish (#14382). + if ciLogReadRe.MatchString(line) { + continue + } + cmds = append(cmds, strings.Join(strings.Fields(strings.TrimRight(line, " \t┃│")), " ")) + } + return cmds +} + +// countCIPollCommands counts CI-status commands in pane. +func countCIPollCommands(pane string) int { + return len(ciPollCommands(pane)) +} + +// ciPollCommandsSinceBaseline returns the commands in current that were run +// after baseline was taken. The pane capture holds only the newest +// tmuxCaptureLines lines and evicts the oldest first, so baseline is a prefix +// of the kick's full command sequence and current is a suffix of it. The +// longest baseline tail that opens current is what is still visible; the rest +// of current is new. Subtracting per-command counts instead let reads that had +// scrolled out cancel genuine new ones (#14065). +func ciPollCommandsSinceBaseline(current, baseline []string) []string { + for k := min(len(current), len(baseline)); k > 0; k-- { + if slices.Equal(baseline[len(baseline)-k:], current[:k]) { + return current[k:] + } + } + return current +} + +// countRepeatedCIPollCommands counts CI-status commands whose command line +// appears more than once since baseline. Waiting on one PR re-reads the same +// target; a merge sweep or CI-repair pass reads each of many PRs once, and +// that is the lane's work, not polling (#14016). Reads already in baseline are +// dropped BEFORE deciding whether a command repeats: one read in each of two +// kicks is not repeated polling in either kick (#14061). +func countRepeatedCIPollCommands(pane string, baseline []string) int { + counts := map[string]int{} + for _, cmd := range ciPollCommandsSinceBaseline(ciPollCommands(pane), baseline) { + counts[cmd]++ + } + n := 0 + for _, c := range counts { + if c > 1 { + n += c + } } return n } +// repeatedCIPollCommandLines lists each command line run more than once since +// baseline as "× ", most-repeated first then by command, so a +// ci_poll_nudge report shows which reads repeated (#14289). +func repeatedCIPollCommandLines(pane string, baseline []string) []string { + counts := map[string]int{} + for _, cmd := range ciPollCommandsSinceBaseline(ciPollCommands(pane), baseline) { + counts[cmd]++ + } + var cmds []string + for cmd, c := range counts { + if c > 1 { + cmds = append(cmds, cmd) + } + } + slices.SortFunc(cmds, func(a, b string) int { + if counts[a] != counts[b] { + return counts[b] - counts[a] + } + return strings.Compare(a, b) + }) + lines := make([]string, 0, len(cmds)) + for _, cmd := range cmds { + lines = append(lines, fmt.Sprintf("%d× %s", counts[cmd], RedactPaneTokens(cmd))) + } + return lines +} + // reviewLaneAgent mirrors the review swarm's own admission rule // (review.hasReviewCapability): a name or role carrying "review" is a // reviewer. pkg/review is not imported here (cycle risk); the token is tiny. @@ -1549,36 +1639,60 @@ func reviewLaneAgent(agent *AgentProcess) bool { return false } +// ciPollNudgeExemptAgent identifies lanes whose work inherently reads CI +// status across repositories. The guard cannot distinguish those operational +// reads from an agent polling its own PR, so nudging these lanes interrupts the +// work they were kicked to do. +func ciPollNudgeExemptAgent(agent *AgentProcess) bool { + if reviewLaneAgent(agent) { + return true + } + for _, f := range []string{agent.Name, agent.Config.Role} { + lane := strings.TrimSpace(f) + if strings.EqualFold(lane, "ci-maintainer") || strings.EqualFold(lane, "quality") { + return true + } + } + return false +} + // nudgeIfPollingCI tells an agent that keeps polling CI within one kick to // move on. Policy templates already forbid it; this is the harness backstop. // Sends at most one nudge per kick, and only at an idle prompt so it never // splices into a running response. // // Two independent triggers count as "polling CI" (#9673's ask, item 2): more -// than ciPollNudgeThreshold poll commands this kick, OR at least one poll +// than ciPollNudgeThreshold repeated poll commands this kick (the same +// command line run more than once, #14016), OR at least one poll // command with the kick already running longer than ciPollNudgeWallClock — a // single `gh run watch` blocks for the whole wait and would otherwise never -// cross the count threshold. +// cross the count threshold. The wall-clock trigger skips a sweep: a scanner +// reading each of several PRs once late in a long kick is doing its work, not +// waiting on one PR (#14066). func (m *Manager) nudgeIfPollingCI(agent *AgentProcess, scrollback, visible string) { if agent.kickDelivering.Load() || agent.LastKick == nil { return } - // Reviewers read CI on the PRs under review as evidence and own no PR to - // wait on, so polling checks is not the loop this guard targets (#13860). - if reviewLaneAgent(agent) { + // Reviewers, CI maintainers and quality agents read check status as evidence + // for the PRs and repositories they inspect. Those reads are not an own-PR + // wait loop, which this guard cannot identify from pane text alone + // (#13860, #13898, #14188). + if ciPollNudgeExemptAgent(agent) { return } - count := countCIPollCommands(scrollback) + cmds := ciPollCommands(scrollback) m.mu.Lock() if agent.ciPollBaseline < 0 { - agent.ciPollBaseline = count + agent.ciPollBaseline = len(cmds) + agent.ciPollCommandBaseline = cmds m.mu.Unlock() return } - newPolls := count - agent.ciPollBaseline - overThreshold := newPolls > ciPollNudgeThreshold || - (newPolls > 0 && time.Since(*agent.LastKick) > ciPollNudgeWallClock) + newPolls := len(ciPollCommandsSinceBaseline(cmds, agent.ciPollCommandBaseline)) + newRepeats := countRepeatedCIPollCommands(scrollback, agent.ciPollCommandBaseline) + overThreshold := newRepeats > ciPollNudgeThreshold || + (newPolls > 0 && !ciPollSweep(newPolls, newRepeats) && time.Since(*agent.LastKick) > ciPollNudgeWallClock) alreadyNudged := agent.ciPollNudgeSent m.mu.Unlock() @@ -1611,25 +1725,38 @@ func (m *Manager) nudgeIfPollingCI(agent *AgentProcess, scrollback, visible stri // Self-report (#13470, was the hive-code-defect "CI-poll nudge" counter): // the harness caps the nudge at one per kick, so every observation is a - // distinct kick the agent spent polling its own CI. - // A lone poll past the wall clock is a normal post-push check, so only - // repeated polling reaches the fleet; the in-session nudge is unchanged. - if ciPollReportWorthy(newPolls) { - fleetreport.Observe(fleetreport.ClassCIPollNudge, agent.Name, - fmt.Sprintf("agent kept polling CI on its own PR after the kick: stop-polling nudge sent (%d poll command(s) since the kick, %s after it; %d nudge(s) since boot)", newPolls, roundDurationForHumans(sinceKick), totalCIPollNudges), - fmt.Sprintf("agent=%s lane=%s polls_total=%d polls_since_kick=%d threshold=%d wall_clock=%s report_min_polls=%d\n--- pane tail (%d lines) ---\n%s", - agent.Name, lane, count, newPolls, ciPollNudgeThreshold, ciPollNudgeWallClock, ciPollReportMinPolls, stallNudgeTailLines, paneTail(visible, stallNudgeTailLines))) + // distinct kick in which the agent repeated CI-status commands; whether + // the target was its own PR cannot be inferred from pane text (#14384). + // A lone poll past the wall clock is a normal post-push check, and one + // read each of several PRs is a sweep, so only repeated polling of the + // same target reaches the fleet; the in-session nudge is unchanged. + if ciPollReportWorthy(newRepeats) { + m.mu.Lock() + repeated := repeatedCIPollCommandLines(scrollback, agent.ciPollCommandBaseline) + m.mu.Unlock() + m.observeCondition(crux.ClassCIPollNudge, agent.Name, + fmt.Sprintf("repeated CI-status commands observed after the kick: stop-polling nudge sent (%d repeated of %d poll command(s) since the kick, %s after it; %d nudge(s) since boot)", newRepeats, newPolls, roundDurationForHumans(sinceKick), totalCIPollNudges), + fmt.Sprintf("agent=%s lane=%s polls_total=%d polls_since_kick=%d repeated_since_kick=%d threshold=%d wall_clock=%s report_min_polls=%d\n--- pane tail (%d lines) ---\n%s\n--- repeated poll commands ---\n%s", + agent.Name, lane, len(cmds), newPolls, newRepeats, ciPollNudgeThreshold, ciPollNudgeWallClock, ciPollReportMinPolls, stallNudgeTailLines, paneTail(visible, stallNudgeTailLines), strings.Join(repeated, "\n"))) } - m.logger.Warn("agent is polling CI, sending stop-polling nudge", "name", agent.Name, "polls", count) + m.logger.Warn("agent is polling CI, sending stop-polling nudge", "name", agent.Name, "polls", len(cmds)) m.tmuxSendLiteralForAgent(agent, ciPollNudgeMessage) time.Sleep(textToEnterDelay) m.tmuxSendEntersForAgent(agent) } +// ciPollSweep reports whether the poll commands since the kick read several +// targets once each: more than one command and none of them repeated. A lone +// poll is not a sweep; it may be the blocking `gh run watch` the wall-clock +// trigger exists for. +func ciPollSweep(newPolls, newRepeats int) bool { + return newPolls > 1 && newRepeats == 0 +} + // ciPollReportWorthy reports whether a CI-poll nudge is repeated polling -// (at least ciPollReportMinPolls commands since the kick) rather than a -// single check. -func ciPollReportWorthy(newPolls int) bool { - return newPolls >= ciPollReportMinPolls +// (at least ciPollReportMinPolls repeated commands since the kick) rather +// than a single check or a sweep. +func ciPollReportWorthy(newRepeats int) bool { + return newRepeats >= ciPollReportMinPolls } diff --git a/src/pkg/agent/manager_launch.go b/src/crux/agent/manager_launch.go similarity index 98% rename from src/pkg/agent/manager_launch.go rename to src/crux/agent/manager_launch.go index 01034625d3..7c39e19d09 100644 --- a/src/pkg/agent/manager_launch.go +++ b/src/crux/agent/manager_launch.go @@ -14,8 +14,8 @@ import ( "syscall" "time" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/taskmcp" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/taskmcp" ) // backendDefersStartupKick reports whether a backend's bootstrap prompt is @@ -1266,28 +1266,10 @@ func claudeEffortFlag(effort string) string { return fmt.Sprintf(" --effort %s", effort) } -// agyLaunchEffort returns the --effort agy is launched with: the configured -// reasoning effort when it is one agy accepts (low/medium/high), else the -// effort encoded in agy's model id suffix, else agyDefaultEffort. The rejection -// mirrors the contributor relay's rule — agy refuses efforts from codex's wider -// vocabulary (e.g. xhigh), and launching with one would make agy ignore the -// model outright, the exact failure --effort exists to prevent. +// agyLaunchEffort is config.AgyLaunchEffort: the --effort agy is launched +// with for a model and configured effort. func agyLaunchEffort(model, effort string) string { - for _, v := range config.ReasoningEffortsByBackend["agy"] { - if v == effort { - return effort - } - } - return agyEffortFromModel(model) -} - -func agyEffortFromModel(model string) string { - for _, v := range config.ReasoningEffortsByBackend["agy"] { - if strings.HasSuffix(model, "-"+v) { - return v - } - } - return agyDefaultEffort + return config.AgyLaunchEffort(model, effort) } // connectionMCPFlags builds MCP-related launch flags from connection configs. diff --git a/src/pkg/agent/manager_modes.go b/src/crux/agent/manager_modes.go similarity index 85% rename from src/pkg/agent/manager_modes.go rename to src/crux/agent/manager_modes.go index 0beabbc017..2f5f42d3e4 100644 --- a/src/pkg/agent/manager_modes.go +++ b/src/crux/agent/manager_modes.go @@ -5,7 +5,8 @@ import ( "path/filepath" "strings" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" ) // ClearModeOverrides clears Config.Mode for the NAMED agents so that @@ -131,39 +132,9 @@ func effectiveAgentMode(name, modeStr string, level int) AgentMode { // DefaultAgentMode returns the default mode for a given agent name and ACMM level, // ignoring any hive.yaml override. Used by the dashboard to show "(default)" indicators. +// The table lives in the crux root package (crux.DefaultAgentMode). func DefaultAgentMode(agentName string, level int) AgentMode { - if agentName == "supervisor" { - return ModeAdvisory - } - switch level { - case 1: - return ModeAdvisory - case 2: - return ModeAdvisory - case 3: - if agentName == "quality" { - return ModeIssuesAndPRs - } - return ModeAdvisory - case 4: - switch agentName { - case "quality", "sec-check", "ci-maintainer": - return ModeIssuesAndPRs - case "scanner", "guide": - return ModeIssuesOnly - default: - return ModeAdvisory - } - case 5: - return ModeIssuesAndPRs - case 6: - if agentName == "scanner" { - return ModeIssuesPRsMerge - } - return ModeIssuesAndPRs - default: - return ModeAdvisory - } + return crux.DefaultAgentMode(agentName, level) } // AuthorizePROpen enforces the policy for the hive-opens-PR watcher: an agent @@ -391,46 +362,10 @@ func (m *Manager) InvocationMetadata(agentName string) (backend, model, effort s return backend, model, ResolveReasoningEffort(backend, model, agent.Config.ReasoningEffort), true } -// ResolveReasoningEffort reports the reasoning effort the hive actually launches -// a given backend/model pair with, given the agent's configured reasoning_effort. -// Exported because the attribution trail is -// resolved in TWO places — Manager.InvocationMetadata above for a running agent, -// and cmd/hive's fallback that reads straight from config when the Manager does -// not know the agent — and both must give the same answer. -// -// Before this existed the fallback carried its own hardcoded "low", so changing -// agyDefaultEffort here would have left cmd/hive silently stamping PRs with an -// effort agy was no longer being launched with. An attribution trail that -// misreports is worse than one that says nothing. -// -// The rules mirror the launch path exactly: -// - agy REQUIRES --effort whenever --model is given, so with a model it runs -// at agyLaunchEffort(model, configured) and with no model at no effort at all. -// - codex is launched with `-c model_reasoning_effort` only when an effort -// is configured; unset means codex's own default, which the hive does not -// resolve, so the honest answer is the configured value verbatim. -// - claude is launched with `--effort` only when the configured effort is -// one Claude Code accepts (claudeEffortFlag drops anything else), so the -// answer is the configured value when valid and "" otherwise — never a -// value the CLI was not actually given (hivecommons/hive#8377). -// - omp is launched with `--thinking` only when an effort is configured; -// unset means omp's per-model default, so report the configured value. -// - every other backend takes its effort from its own config, which the -// hive does not resolve here, so the honest answer is "". +// ResolveReasoningEffort is config.ResolveReasoningEffort: the effort the +// launch path actually passes for a backend/model pair, shared with +// InvocationMetadata above and cmd/hive's config-only fallback so the +// attribution trail never disagrees with the launch. func ResolveReasoningEffort(backend, model, configured string) string { - switch backend { - case "agy": - if model != "" { - return agyLaunchEffort(model, configured) - } - return "" - case codexBackend, "omp": - return configured - case config.ClaudeBackend: - if config.ValidEffort(config.ClaudeBackend, configured) { - return configured - } - return "" - } - return "" + return config.ResolveReasoningEffort(backend, model, configured) } diff --git a/src/pkg/agent/manager_operator_stop_test.go b/src/crux/agent/manager_operator_stop_test.go similarity index 97% rename from src/pkg/agent/manager_operator_stop_test.go rename to src/crux/agent/manager_operator_stop_test.go index 71582fe165..9d0e4e148a 100644 --- a/src/pkg/agent/manager_operator_stop_test.go +++ b/src/crux/agent/manager_operator_stop_test.go @@ -3,7 +3,7 @@ package agent import ( "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestGovernorMayStart_OperatorStop(t *testing.T) { diff --git a/src/pkg/agent/manager_pane_sanitizer_test.go b/src/crux/agent/manager_pane_sanitizer_test.go similarity index 100% rename from src/pkg/agent/manager_pane_sanitizer_test.go rename to src/crux/agent/manager_pane_sanitizer_test.go diff --git a/src/pkg/agent/manager_pane_signals.go b/src/crux/agent/manager_pane_signals.go similarity index 98% rename from src/pkg/agent/manager_pane_signals.go rename to src/crux/agent/manager_pane_signals.go index 0cb47c0829..aedf201989 100644 --- a/src/pkg/agent/manager_pane_signals.go +++ b/src/crux/agent/manager_pane_signals.go @@ -4,7 +4,8 @@ import ( "regexp" "strings" - "github.com/hivecommons/hive/pkg/watchdog" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/watchdog" ) // cliPaneMarkers are strings that appear in a tmux pane when a CLI (claude, @@ -516,7 +517,9 @@ func paneShowsCopilotQuestionForm(output string) bool { strings.Contains(output, "enter accept · tab next · ctrl+d decline · esc cancel") } -func (a *AgentProcess) snapshot() AgentProcess { +// snapshot copies the agent's observable state into the crux.Process the +// manager hands out from GetStatus, GetStatusFast and AllStatuses. +func (a *AgentProcess) snapshot() crux.Process { history := make([]KickRecord, len(a.KickHistory)) copy(history, a.KickHistory) a.paneMu.RLock() @@ -534,7 +537,7 @@ func (a *AgentProcess) snapshot() AgentProcess { conds := make([]watchdog.Condition, len(a.WatchdogConditions)) copy(conds, a.WatchdogConditions) a.paneMu.RUnlock() - return AgentProcess{ + snap := crux.Process{ Name: a.Name, ID: a.ID, Config: a.Config, @@ -564,6 +567,10 @@ func (a *AgentProcess) snapshot() AgentProcess { TurnLoss: cloneTurnLoss(a.TurnLoss), KickHistory: history, LastKickMessage: a.LastKickMessage, + KickRefused: a.KickRefused, + KickRefusalReason: a.KickRefusalReason, + BootstrapOverride: a.BootstrapOverride, + LastError: a.LastError, KickOutcome: a.KickOutcome, NeedsLogin: needsLogin, LoginURL: loginURL, @@ -573,6 +580,8 @@ func (a *AgentProcess) snapshot() AgentProcess { WatchdogConditions: conds, StallNudges: a.StallNudges, StallRecoveries: a.StallRecoveries, + CIPollNudges: a.CIPollNudges, + AwaitingCI: a.AwaitingCI, ActionNudges: a.ActionNudges, TransientNudges: a.TransientNudges, ProviderErrorClass: a.ProviderErrorClass, @@ -589,12 +598,11 @@ func (a *AgentProcess) snapshot() AgentProcess { StartBackoffUntil: a.StartBackoffUntil, HasLaunched: a.HasLaunched, LaunchedMode: a.LaunchedMode, - tmuxSession: a.tmuxSession, - tmuxSocket: a.tmuxSocket, OutputBuffer: a.OutputBuffer, - lastPaneCapture: pane, SubAgentModels: subAgentModels, } + snap.SetPaneCapture(pane, filterPaneOutput) + return snap } // PaneLines returns the last n lines from the most recent tmux pane capture, diff --git a/src/pkg/agent/manager_pause.go b/src/crux/agent/manager_pause.go similarity index 96% rename from src/pkg/agent/manager_pause.go rename to src/crux/agent/manager_pause.go index 42f18afdd5..c118f2a959 100644 --- a/src/pkg/agent/manager_pause.go +++ b/src/crux/agent/manager_pause.go @@ -5,26 +5,16 @@ import ( "fmt" "sort" "time" -) -// PauseCausation is hook causation metadata carried in agent pause/resume -// events without making pkg/agent import pkg/hooks. -type PauseCausation struct { - Depth int - HookName string - OriginTransition string -} + "github.com/hivecommons/hive/crux" +) -// PauseTransitionEvent describes a durable agent pause/resume transition. -type PauseTransitionEvent struct { - Agent string - Paused bool - Trigger string - Reason string - By string - Causation PauseCausation - At time.Time -} +// PauseCausation and PauseTransitionEvent are defined in crux so the host +// observes pause transitions without importing the runtime. +type ( + PauseCausation = crux.PauseCausation + PauseTransitionEvent = crux.PauseTransitionEvent +) // SetPersistPauseCallback wires a function that persists an agent's paused // state to config (called on pause/resume). Idempotent saves are the diff --git a/src/pkg/agent/manager_poll.go b/src/crux/agent/manager_poll.go similarity index 100% rename from src/pkg/agent/manager_poll.go rename to src/crux/agent/manager_poll.go diff --git a/src/pkg/agent/manager_poll_kick_delivery_test.go b/src/crux/agent/manager_poll_kick_delivery_test.go similarity index 99% rename from src/pkg/agent/manager_poll_kick_delivery_test.go rename to src/crux/agent/manager_poll_kick_delivery_test.go index 3052174412..b01351e142 100644 --- a/src/pkg/agent/manager_poll_kick_delivery_test.go +++ b/src/crux/agent/manager_poll_kick_delivery_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // Delivering a kick is not instantaneous: deliverKickLocked types the message diff --git a/src/pkg/agent/manager_poll_model_fallback_test.go b/src/crux/agent/manager_poll_model_fallback_test.go similarity index 100% rename from src/pkg/agent/manager_poll_model_fallback_test.go rename to src/crux/agent/manager_poll_model_fallback_test.go diff --git a/src/pkg/agent/manager_replicas_test.go b/src/crux/agent/manager_replicas_test.go similarity index 95% rename from src/pkg/agent/manager_replicas_test.go rename to src/crux/agent/manager_replicas_test.go index 94e15d1be9..606f940ea6 100644 --- a/src/pkg/agent/manager_replicas_test.go +++ b/src/crux/agent/manager_replicas_test.go @@ -4,7 +4,7 @@ import ( "log/slog" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestReconcileAgentsAddsReplicaWithUIDAndRemovesExtra(t *testing.T) { diff --git a/src/pkg/agent/manager_restart.go b/src/crux/agent/manager_restart.go similarity index 99% rename from src/pkg/agent/manager_restart.go rename to src/crux/agent/manager_restart.go index 8aae8786fe..bac580547d 100644 --- a/src/pkg/agent/manager_restart.go +++ b/src/crux/agent/manager_restart.go @@ -230,14 +230,18 @@ func (m *Manager) CheckAndRestartCrashedAgents(ctx context.Context) []string { // idle, and the storm breaker bounds repeat restarts. hungSessions = append(hungSessions, sessionRestart{name: name, reason: reason, busyNoActivity: true}) } - if IsInferenceBackend(effectiveBackend(agent)) { + // Plain claude agents launch with --dangerously-skip-permissions too, + // so they can park on the same bypass consent screen (#14307). + if backend := effectiveBackend(agent); claudeConsentBackend(backend) { if paneShowsConsentScreen(pane) { consentStuck = append(consentStuck, name) } else { if !agent.consentSeenAt.IsZero() { consentCleared = append(consentCleared, name) } - stallChecks = append(stallChecks, inferencePaneCheck{name: name, pane: pane}) + if IsInferenceBackend(backend) { + stallChecks = append(stallChecks, inferencePaneCheck{name: name, pane: pane}) + } } } } diff --git a/src/pkg/agent/manager_restart_telemetry.go b/src/crux/agent/manager_restart_telemetry.go similarity index 92% rename from src/pkg/agent/manager_restart_telemetry.go rename to src/crux/agent/manager_restart_telemetry.go index 22fed5d1fe..df08fac723 100644 --- a/src/pkg/agent/manager_restart_telemetry.go +++ b/src/crux/agent/manager_restart_telemetry.go @@ -4,11 +4,11 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/crux/types" + "github.com/hivecommons/hive/crux" ) -// RestartEvent is one restart and its reason; see types.RestartEvent. -type RestartEvent = types.RestartEvent +// RestartEvent is one restart and its reason; see crux.RestartEvent. +type RestartEvent = crux.RestartEvent func cloneRestartEvents(in []RestartEvent) []RestartEvent { if len(in) == 0 { diff --git a/src/pkg/agent/manager_routing.go b/src/crux/agent/manager_routing.go similarity index 99% rename from src/pkg/agent/manager_routing.go rename to src/crux/agent/manager_routing.go index 4657d939d6..0818792859 100644 --- a/src/pkg/agent/manager_routing.go +++ b/src/crux/agent/manager_routing.go @@ -5,7 +5,7 @@ import ( "os/exec" "strings" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // effectiveBackend returns the agent's backend accounting for any override. diff --git a/src/pkg/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go similarity index 98% rename from src/pkg/agent/manager_sandbox.go rename to src/crux/agent/manager_sandbox.go index 3778a03be6..2418c9ff04 100644 --- a/src/pkg/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -7,11 +7,11 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/effects" - "github.com/hivecommons/hive/pkg/kubejob" - "github.com/hivecommons/hive/pkg/pushbroker" - "github.com/hivecommons/hive/pkg/sandbox" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/effects" + "github.com/hivecommons/hive/crux/kubejob" + "github.com/hivecommons/hive/crux/pushbroker" + "github.com/hivecommons/hive/crux/sandbox" ) // SetSandboxConfig wires the disabled-by-default sandbox kick executor gate. diff --git a/src/pkg/agent/manager_sandbox_wiring_test.go b/src/crux/agent/manager_sandbox_wiring_test.go similarity index 88% rename from src/pkg/agent/manager_sandbox_wiring_test.go rename to src/crux/agent/manager_sandbox_wiring_test.go index fe1d4ebf22..05483d260f 100644 --- a/src/pkg/agent/manager_sandbox_wiring_test.go +++ b/src/crux/agent/manager_sandbox_wiring_test.go @@ -5,8 +5,8 @@ import ( "log/slog" "testing" - "github.com/hivecommons/hive/pkg/crux/types" - "github.com/hivecommons/hive/pkg/pushbroker" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/pushbroker" ) type wiringPRCreator struct{ id int } @@ -17,8 +17,8 @@ type wiringMinter struct{ id int } func (*wiringMinter) MintPushToken(context.Context, string) (string, error) { return "", nil } -func (*wiringPRCreator) CreatePR(context.Context, string, string, string, string, string) (types.PRResult, error) { - return types.PRResult{}, nil +func (*wiringPRCreator) CreatePR(context.Context, string, string, string, string, string) (crux.PRResult, error) { + return crux.PRResult{}, nil } // #9621: SandboxGitHubWiring reports what the sandbox executor will use, so diff --git a/src/pkg/agent/manager_symlink_test.go b/src/crux/agent/manager_symlink_test.go similarity index 100% rename from src/pkg/agent/manager_symlink_test.go rename to src/crux/agent/manager_symlink_test.go diff --git a/src/pkg/agent/manager_terminal.go b/src/crux/agent/manager_terminal.go similarity index 100% rename from src/pkg/agent/manager_terminal.go rename to src/crux/agent/manager_terminal.go diff --git a/src/pkg/agent/manager_test.go b/src/crux/agent/manager_test.go similarity index 99% rename from src/pkg/agent/manager_test.go rename to src/crux/agent/manager_test.go index ef792a23c1..6c046c6268 100644 --- a/src/pkg/agent/manager_test.go +++ b/src/crux/agent/manager_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) var stubBinDir string diff --git a/src/pkg/agent/manager_thrash.go b/src/crux/agent/manager_thrash.go similarity index 100% rename from src/pkg/agent/manager_thrash.go rename to src/crux/agent/manager_thrash.go diff --git a/src/pkg/agent/manager_tmux.go b/src/crux/agent/manager_tmux.go similarity index 99% rename from src/pkg/agent/manager_tmux.go rename to src/crux/agent/manager_tmux.go index f264d47713..3a44f5f4b9 100644 --- a/src/pkg/agent/manager_tmux.go +++ b/src/crux/agent/manager_tmux.go @@ -9,8 +9,8 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/agent/tmux" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/agent/tmux" + "github.com/hivecommons/hive/crux/config" ) // paneCaptureSleep is a pacing var, not a const, for the same reason as the diff --git a/src/pkg/agent/manager_tokens.go b/src/crux/agent/manager_tokens.go similarity index 93% rename from src/pkg/agent/manager_tokens.go rename to src/crux/agent/manager_tokens.go index c0155f5560..7a039f39b5 100644 --- a/src/pkg/agent/manager_tokens.go +++ b/src/crux/agent/manager_tokens.go @@ -2,33 +2,23 @@ package agent import ( "context" + "errors" "fmt" "os" "strings" "time" - "github.com/hivecommons/hive/pkg/claude" - "github.com/hivecommons/hive/pkg/crux/types" - "github.com/hivecommons/hive/pkg/fleetreport" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/claude" ) -// AppTokenMinter is implemented by the host's GitHub App auth (pkg/github.AppAuth) -// to mint per-agent scoped tokens. -type AppTokenMinter interface { - WriteAgentToken(ctx context.Context, agentName, tier string, agentUID int) error -} - -// AgentMintIssuer is implemented by mint.AgentMinter. It is the OPT-IN, -// ADDITIONAL short-lived-credential path: when the mint is enabled it issues a -// scoped OIDC/JWT for an agent (subject=agent, scopes from its tier) that the -// agent can present ALONGSIDE its GitHub App token to a WIF broker. It never -// replaces WriteAgentToken. Enabled() reports false (and MintAgentToken returns -// "" with no error) when the mint is off, so wiring stays a strict no-op by -// default. -type AgentMintIssuer interface { - Enabled() bool - MintAgentToken(agentName, tier string) (string, error) -} +// AppTokenMinter and AgentMintIssuer are the host-provided credential +// capabilities, defined in the crux root (crux.AppTokenMinter, +// crux.AgentMintIssuer) so a host names them without importing this package. +type ( + AppTokenMinter = crux.AppTokenMinter + AgentMintIssuer = crux.AgentMintIssuer +) // ReloadClaudeToken re-reads the Claude credentials file and updates the // cached token. Called by the dashboard after a successful OAuth login. @@ -43,18 +33,11 @@ func (m *Manager) ReloadClaudeToken() { // "rejected by upstream" log line, so each one has to read as an answer to // "which credential did GitHub just reject?". const ( - // CopilotTokenSourceEnv: the COPILOT_GITHUB_TOKEN the hive process was - // started with (a provisioned/machine token, typically). - CopilotTokenSourceEnv = "the COPILOT_GITHUB_TOKEN environment variable" - // CopilotTokenSourceDashboardLogin: a device-flow login completed on the - // dashboard during this process's lifetime. - CopilotTokenSourceDashboardLogin = "the dashboard Copilot login" - // CopilotTokenSourceDurableFile: a dashboard login persisted by an earlier - // hive process and read back at startup. - CopilotTokenSourceDurableFile = "a dashboard Copilot login persisted at " + CopilotUserTokenPath - // CopilotTokenSourceCLIConfig: an in-agent `/login` the reconciler promoted - // out of the shared Copilot CLI config. - CopilotTokenSourceCLIConfig = "an in-agent /login promoted from the shared Copilot CLI config" + // Defined in the crux root package; see crux.CopilotTokenSourceEnv. + CopilotTokenSourceEnv = crux.CopilotTokenSourceEnv + CopilotTokenSourceDashboardLogin = crux.CopilotTokenSourceDashboardLogin + CopilotTokenSourceDurableFile = crux.CopilotTokenSourceDurableFile + CopilotTokenSourceCLIConfig = crux.CopilotTokenSourceCLIConfig ) // SetCopilotToken updates the cached Copilot token injected into agent @@ -316,19 +299,9 @@ func (m *Manager) bobAPIKey() string { return (*fnp)() } -// LinearCredential is what an agent authenticates to api.linear.app with. -// Exactly one of the two is expected to be set: AccessToken is the installed -// Linear agent app's OAuth token (sent as `Authorization: Bearer …`, so writes -// are authored by the app identity — the parity of GitHub's App bot); APIKey -// is the work-source personal API key (sent bare as `Authorization: …`), the -// fallback when no workspace is connected. Empty means "inject nothing". -type LinearCredential struct { - AccessToken string - APIKey string -} - -// Empty reports whether no credential is available. -func (c LinearCredential) Empty() bool { return c.AccessToken == "" && c.APIKey == "" } +// LinearCredential is crux.LinearCredential, the host-provided Linear +// write credential. +type LinearCredential = crux.LinearCredential // SetLinearCredentialResolver injects the resolver for the Linear write // credential. Called from main.go with a closure over the live Linear agent @@ -910,7 +883,7 @@ func (m *Manager) refreshAgentTokens(ctx context.Context) { // match the launch injection — advisory agents stay GITHUB_TOKEN-less, and // hives that have not opted in get nothing injected. if m.project.AppAuthoredPRs && a.tmuxSession != "" && m.agentMode(a).CanPush() { - if data, err := os.ReadFile(types.AgentTokenCachePath(types.DefaultAgentTokenCacheDir, a.Name)); err == nil { + if data, err := os.ReadFile(crux.AgentTokenCachePath(crux.DefaultAgentTokenCacheDir, a.Name)); err == nil { if tok := strings.TrimSpace(string(data)); tok != "" { _ = m.tmuxCmd(a, "set-environment", "-t", a.tmuxSession, "GITHUB_TOKEN", tok).Run() } @@ -949,7 +922,15 @@ func (m *Manager) mintAgentTokenUnlocked(ctx context.Context, agent *AgentProces return } tier := TokenTierForRole(m.agentMode(agent), agent.Config.Role) - if err := m.appAuth.WriteAgentToken(ctx, agent.Name, tier, agent.UID); err != nil { + if err := m.appAuth.WriteAgentToken(ctx, agent.Name, tier, agent.UID); err != nil && errors.Is(err, crux.ErrGitHubAppNotInstalled) { + // Static misconfiguration: every tick would fail identically, so log + // once per agent until a mint succeeds instead of every supervisor tick. + if _, seen := m.appNotInstalledWarned.LoadOrStore(agent.Name, struct{}{}); !seen { + m.logger.Warn("github app installation id is unset — per-agent scoped token NOT delivered, this agent's GitHub writes (gh + git push) will FAIL; set github.installation_id or install the App on the hive's org so discovery resolves it", + "agent", agent.Name, "tier", tier, "error", err) + } + m.observeCondition(crux.ClassGitHubAppNotInstalled, agent.Name, "github app installation id is unset: "+err.Error(), "") + } else if err != nil { // Be precise about the blast radius. Since audit H3 the shared-cache // fallback is GONE: gh-wrapper.sh and git-credential-hive.sh no // longer fall back to /var/run/hive-metrics/gh-app-token.cache (the @@ -960,9 +941,11 @@ func (m *Manager) mintAgentTokenUnlocked(ctx context.Context, agent *AgentProces // token delivery is repaired — not a silent privilege escalation. m.logger.Warn("per-agent scoped token NOT delivered — this agent's GitHub writes (gh + git push) will FAIL (no shared-cache fallback by design; see audit H3)", "agent", agent.Name, "tier", tier, "error", err) - fleetreport.Observe(fleetreport.ClassGitHubAppTokenMintFailed, agent.Name, "github app token mint failed: "+err.Error(), "") + m.observeCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name, "github app token mint failed: "+err.Error(), "") } else { - fleetreport.Clear(fleetreport.ClassGitHubAppTokenMintFailed, agent.Name) + m.appNotInstalledWarned.Delete(agent.Name) + m.clearCondition(crux.ClassGitHubAppNotInstalled, agent.Name) + m.clearCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name) } // Additionally issue an opt-in short-lived mint token (no-op when the mint // is disabled). This is additive and fail-safe — it never blocks launch. diff --git a/src/pkg/agent/manager_uid_isolation.go b/src/crux/agent/manager_uid_isolation.go similarity index 100% rename from src/pkg/agent/manager_uid_isolation.go rename to src/crux/agent/manager_uid_isolation.go diff --git a/src/pkg/agent/mint_deadlock_test.go b/src/crux/agent/mint_deadlock_test.go similarity index 100% rename from src/pkg/agent/mint_deadlock_test.go rename to src/crux/agent/mint_deadlock_test.go diff --git a/src/pkg/agent/mint_token_cache_test.go b/src/crux/agent/mint_token_cache_test.go similarity index 99% rename from src/pkg/agent/mint_token_cache_test.go rename to src/crux/agent/mint_token_cache_test.go index faf3d1756e..3e3db3090d 100644 --- a/src/pkg/agent/mint_token_cache_test.go +++ b/src/crux/agent/mint_token_cache_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // These tests cover the per-agent mint-token cache plumbing end to end: diff --git a/src/pkg/agent/mode.go b/src/crux/agent/mode.go similarity index 96% rename from src/pkg/agent/mode.go rename to src/crux/agent/mode.go index e345a7db4f..74b4c9746d 100644 --- a/src/pkg/agent/mode.go +++ b/src/crux/agent/mode.go @@ -1,7 +1,7 @@ package agent import ( - "github.com/hivecommons/hive/pkg/agentmode" + "github.com/hivecommons/hive/crux/agentmode" ) // AgentMode and its constants live in the stdlib-only leaf pkg/agentmode so diff --git a/src/pkg/agent/mode_test.go b/src/crux/agent/mode_test.go similarity index 99% rename from src/pkg/agent/mode_test.go rename to src/crux/agent/mode_test.go index f78deaa671..7052ea53bb 100644 --- a/src/pkg/agent/mode_test.go +++ b/src/crux/agent/mode_test.go @@ -3,7 +3,7 @@ package agent import ( "testing" - "github.com/hivecommons/hive/pkg/agentmode" + "github.com/hivecommons/hive/crux/agentmode" ) func TestAgentModeString(t *testing.T) { diff --git a/src/pkg/agent/not_running_reason_test.go b/src/crux/agent/not_running_reason_test.go similarity index 100% rename from src/pkg/agent/not_running_reason_test.go rename to src/crux/agent/not_running_reason_test.go diff --git a/src/pkg/agent/oomkill.go b/src/crux/agent/oomkill.go similarity index 100% rename from src/pkg/agent/oomkill.go rename to src/crux/agent/oomkill.go diff --git a/src/pkg/agent/oomkill_test.go b/src/crux/agent/oomkill_test.go similarity index 100% rename from src/pkg/agent/oomkill_test.go rename to src/crux/agent/oomkill_test.go diff --git a/src/pkg/agent/pane_classify.go b/src/crux/agent/pane_classify.go similarity index 99% rename from src/pkg/agent/pane_classify.go rename to src/crux/agent/pane_classify.go index ee64527c51..f48b0a3b53 100644 --- a/src/pkg/agent/pane_classify.go +++ b/src/crux/agent/pane_classify.go @@ -5,6 +5,8 @@ import ( "regexp" "strings" "time" + + "github.com/hivecommons/hive/crux" ) // loginPromptPatterns are substrings that indicate an agent is stuck on a @@ -62,11 +64,8 @@ var loginPromptPatterns = []string{ // FatalNetworkErrorPatterns is exported so prompt builders can assert their // text never contains one: a prompt echoed into the pane that quotes one of // these restarts the agent every time it pauses to think. -var FatalNetworkErrorPatterns = []string{ - "invalid peer certificate", - "BadSignature", - "fetch failed", -} +// The list itself is defined in the crux root package. +var FatalNetworkErrorPatterns = crux.FatalNetworkErrorPatterns() // paneShowsFatalNetworkError returns true if any line contains a fatal // TLS/network error pattern that requires an agent restart. @@ -89,7 +88,7 @@ func paneShowsFatalNetworkError(lines []string) bool { // The CLI then silently runs the substituted model for the whole session // with no further indication (#9927). copilotPinnedCLIModels/ // copilotCLIAcceptedModels (pkg/dashboard/cli_models.go, -// pkg/agent/copilot_models.go) keep the picker from offering an id this CLI +// crux/copilot_models.go) keep the picker from offering an id this CLI // rejects, but this is the defense-in-depth catch for a model that reaches // --model some other way — a stale stored config from before #9933/#9943, a // hand-edited hive.yaml, or a future CLI pin that drops support for a diff --git a/src/pkg/agent/pane_fixtures_test.go b/src/crux/agent/pane_fixtures_test.go similarity index 98% rename from src/pkg/agent/pane_fixtures_test.go rename to src/crux/agent/pane_fixtures_test.go index 0290bc8e41..2aa9c979c3 100644 --- a/src/pkg/agent/pane_fixtures_test.go +++ b/src/crux/agent/pane_fixtures_test.go @@ -17,7 +17,7 @@ import ( // second `paneTailNonBlank` variant, while Go's paneTail always filtered // blanks — the direct root cause of kubestellar/hive#6413. // -// The path is relative to this package directory (src/pkg/agent), three +// The path is relative to this package directory (src/crux/agent), three // levels up to the repo root, then into bin/testdata/pane-fixtures. const paneFixturesDir = "../../../bin/testdata/pane-fixtures" diff --git a/src/pkg/agent/pane_rate_limit_test.go b/src/crux/agent/pane_rate_limit_test.go similarity index 100% rename from src/pkg/agent/pane_rate_limit_test.go rename to src/crux/agent/pane_rate_limit_test.go diff --git a/src/pkg/agent/pane_redaction_test.go b/src/crux/agent/pane_redaction_test.go similarity index 100% rename from src/pkg/agent/pane_redaction_test.go rename to src/crux/agent/pane_redaction_test.go diff --git a/src/pkg/agent/pause_observer_clear_test.go b/src/crux/agent/pause_observer_clear_test.go similarity index 100% rename from src/pkg/agent/pause_observer_clear_test.go rename to src/crux/agent/pause_observer_clear_test.go diff --git a/src/pkg/agent/pause_provenance_test.go b/src/crux/agent/pause_provenance_test.go similarity index 99% rename from src/pkg/agent/pause_provenance_test.go rename to src/crux/agent/pause_provenance_test.go index 79f9bd3a24..8c915caac8 100644 --- a/src/pkg/agent/pause_provenance_test.go +++ b/src/crux/agent/pause_provenance_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // #4041: pause provenance — WHO paused WHAT WHEN — must live on the agent diff --git a/src/pkg/agent/permissions_agylog_test.go b/src/crux/agent/permissions_agylog_test.go similarity index 100% rename from src/pkg/agent/permissions_agylog_test.go rename to src/crux/agent/permissions_agylog_test.go diff --git a/src/pkg/agent/permissions_bobstate_test.go b/src/crux/agent/permissions_bobstate_test.go similarity index 100% rename from src/pkg/agent/permissions_bobstate_test.go rename to src/crux/agent/permissions_bobstate_test.go diff --git a/src/pkg/agent/permissions_dedupe_test.go b/src/crux/agent/permissions_dedupe_test.go similarity index 100% rename from src/pkg/agent/permissions_dedupe_test.go rename to src/crux/agent/permissions_dedupe_test.go diff --git a/src/pkg/agent/permissions_fixentry_test.go b/src/crux/agent/permissions_fixentry_test.go similarity index 100% rename from src/pkg/agent/permissions_fixentry_test.go rename to src/crux/agent/permissions_fixentry_test.go diff --git a/src/pkg/agent/permissions_modefile_test.go b/src/crux/agent/permissions_modefile_test.go similarity index 100% rename from src/pkg/agent/permissions_modefile_test.go rename to src/crux/agent/permissions_modefile_test.go diff --git a/src/pkg/agent/permissions_sharedrepo_test.go b/src/crux/agent/permissions_sharedrepo_test.go similarity index 100% rename from src/pkg/agent/permissions_sharedrepo_test.go rename to src/crux/agent/permissions_sharedrepo_test.go diff --git a/src/pkg/agent/permissions_symlink_test.go b/src/crux/agent/permissions_symlink_test.go similarity index 100% rename from src/pkg/agent/permissions_symlink_test.go rename to src/crux/agent/permissions_symlink_test.go diff --git a/src/pkg/agent/permissions_watcher.go b/src/crux/agent/permissions_watcher.go similarity index 100% rename from src/pkg/agent/permissions_watcher.go rename to src/crux/agent/permissions_watcher.go diff --git a/src/pkg/agent/permissions_watcher_start_test.go b/src/crux/agent/permissions_watcher_start_test.go similarity index 100% rename from src/pkg/agent/permissions_watcher_start_test.go rename to src/crux/agent/permissions_watcher_start_test.go diff --git a/src/pkg/agent/persist_pause_deadlock_test.go b/src/crux/agent/persist_pause_deadlock_test.go similarity index 98% rename from src/pkg/agent/persist_pause_deadlock_test.go rename to src/crux/agent/persist_pause_deadlock_test.go index 2ab48a8cd4..73853e090f 100644 --- a/src/pkg/agent/persist_pause_deadlock_test.go +++ b/src/crux/agent/persist_pause_deadlock_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // These are the v4-idiom equivalents of v2's diff --git a/src/pkg/agent/pi_backend_test.go b/src/crux/agent/pi_backend_test.go similarity index 84% rename from src/pkg/agent/pi_backend_test.go rename to src/crux/agent/pi_backend_test.go index c95e1af88f..2ae2af430d 100644 --- a/src/pkg/agent/pi_backend_test.go +++ b/src/crux/agent/pi_backend_test.go @@ -1,11 +1,15 @@ package agent import ( + "errors" + "io/fs" "os" "path/filepath" "runtime" "strings" "testing" + + "github.com/hivecommons/hive/crux/internal/testutil" ) const piIdlePane = `╭──────────────────────────────────────────────╮ @@ -34,6 +38,11 @@ func TestDockerfileInstallsPiCLI(t *testing.T) { } dockerfile := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "Dockerfile")) body, err := os.ReadFile(dockerfile) + if errors.Is(err, fs.ErrNotExist) { + // src/Dockerfile belongs to the hive root module; absent when the crux + // module is built standalone (hivecommons/crux). + testutil.SkipUnlessRequired(t, "src/Dockerfile is not part of the crux module (standalone build)") + } if err != nil { t.Fatalf("read Dockerfile: %v", err) } diff --git a/src/pkg/agent/pluk_dirs.go b/src/crux/agent/pluk_dirs.go similarity index 100% rename from src/pkg/agent/pluk_dirs.go rename to src/crux/agent/pluk_dirs.go diff --git a/src/pkg/agent/pluk_dirs_branches_test.go b/src/crux/agent/pluk_dirs_branches_test.go similarity index 100% rename from src/pkg/agent/pluk_dirs_branches_test.go rename to src/crux/agent/pluk_dirs_branches_test.go diff --git a/src/pkg/agent/pluk_dirs_test.go b/src/crux/agent/pluk_dirs_test.go similarity index 100% rename from src/pkg/agent/pluk_dirs_test.go rename to src/crux/agent/pluk_dirs_test.go diff --git a/src/pkg/agent/pluk_publisher_test.go b/src/crux/agent/pluk_publisher_test.go similarity index 100% rename from src/pkg/agent/pluk_publisher_test.go rename to src/crux/agent/pluk_publisher_test.go diff --git a/src/pkg/agent/poll_coverage_test.go b/src/crux/agent/poll_coverage_test.go similarity index 98% rename from src/pkg/agent/poll_coverage_test.go rename to src/crux/agent/poll_coverage_test.go index fbdf62c253..0941bf0589 100644 --- a/src/pkg/agent/poll_coverage_test.go +++ b/src/crux/agent/poll_coverage_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestPollTmuxOutputForAgent_TLSError: a copilot agent whose pane shows a fatal diff --git a/src/pkg/agent/pr_session_resume.go b/src/crux/agent/pr_session_resume.go similarity index 87% rename from src/pkg/agent/pr_session_resume.go rename to src/crux/agent/pr_session_resume.go index 1d4ded7f39..cd9da5d53a 100644 --- a/src/pkg/agent/pr_session_resume.go +++ b/src/crux/agent/pr_session_resume.go @@ -3,10 +3,11 @@ package agent import ( "crypto/rand" "encoding/hex" - "errors" "fmt" "strconv" "time" + + "github.com/hivecommons/hive/crux" ) // PR follow-up session resume (hivecommons/hive#9583). @@ -45,20 +46,13 @@ func newSessionBootNonce() string { return hex.EncodeToString(b) } -// ErrResumeBusy means the authoring session still exists but cannot take a -// follow-up right now (mid-turn, provider backoff, restart hold). It is -// transient: the caller keeps the follow-up queued and retries. -var ErrResumeBusy = errors.New("agent session busy; follow-up deferred") - -// ErrResumeSessionGone means the session that authored the PR no longer -// exists, or the agent has moved on to other work since (a newer kick, a -// restart, a relaunch). The caller must fall back to a fresh dispatch. -var ErrResumeSessionGone = errors.New("authoring session no longer live") - -// ErrResumeUnsupported means this agent's execution mode has no resumable -// conversation to type into (sandbox jobs, the headless agy runner) or the -// operator paused it. The caller falls back to a fresh dispatch. -var ErrResumeUnsupported = errors.New("agent cannot resume a session") +// The resume sentinels are defined in the crux root so hosts can match them +// without importing this package; see crux.ErrResumeBusy and friends. +var ( + ErrResumeBusy = crux.ErrResumeBusy + ErrResumeSessionGone = crux.ErrResumeSessionGone + ErrResumeUnsupported = crux.ErrResumeUnsupported +) // sessionIDLocked renders the identity of the conversation the agent's CLI // currently holds. It changes whenever that conversation could have been diff --git a/src/pkg/agent/pr_session_resume_test.go b/src/crux/agent/pr_session_resume_test.go similarity index 100% rename from src/pkg/agent/pr_session_resume_test.go rename to src/crux/agent/pr_session_resume_test.go diff --git a/src/pkg/agent/proc_coverage_test.go b/src/crux/agent/proc_coverage_test.go similarity index 99% rename from src/pkg/agent/proc_coverage_test.go rename to src/crux/agent/proc_coverage_test.go index a5a1d39fb5..b76c2cd892 100644 --- a/src/pkg/agent/proc_coverage_test.go +++ b/src/crux/agent/proc_coverage_test.go @@ -8,7 +8,7 @@ import ( "strconv" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // withFakeProc redirects procRoot to a temp dir for the duration of a test. diff --git a/src/pkg/agent/process_leak_guard_test.go b/src/crux/agent/process_leak_guard_test.go similarity index 100% rename from src/pkg/agent/process_leak_guard_test.go rename to src/crux/agent/process_leak_guard_test.go diff --git a/src/pkg/agent/provider_error_helpers_test.go b/src/crux/agent/provider_error_helpers_test.go similarity index 100% rename from src/pkg/agent/provider_error_helpers_test.go rename to src/crux/agent/provider_error_helpers_test.go diff --git a/src/pkg/agent/provider_error_state_test.go b/src/crux/agent/provider_error_state_test.go similarity index 99% rename from src/pkg/agent/provider_error_state_test.go rename to src/crux/agent/provider_error_state_test.go index f7e4c4813d..766743ce42 100644 --- a/src/pkg/agent/provider_error_state_test.go +++ b/src/crux/agent/provider_error_state_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // Tests for the provider-error backoff state lifecycle in manager.go: diff --git a/src/pkg/agent/provider_error_test.go b/src/crux/agent/provider_error_test.go similarity index 99% rename from src/pkg/agent/provider_error_test.go rename to src/crux/agent/provider_error_test.go index 8d2fd4c14b..3580045667 100644 --- a/src/pkg/agent/provider_error_test.go +++ b/src/crux/agent/provider_error_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestClassifyProviderError(t *testing.T) { diff --git a/src/crux/agent/provider_status.go b/src/crux/agent/provider_status.go new file mode 100644 index 0000000000..54995afad0 --- /dev/null +++ b/src/crux/agent/provider_status.go @@ -0,0 +1,33 @@ +package agent + +import "github.com/hivecommons/hive/crux" + +// Provider status vocabulary and roll-up (ADR-0024, RFC #12611 §4). The +// definitions live in pkg/crux as part of the Crux runtime surface +// (hivecommons/hive#13536); these aliases keep pkg/agent's names valid. +const ( + ProviderAuthUsable = crux.ProviderAuthUsable + ProviderAuthRefreshable = crux.ProviderAuthRefreshable + ProviderAuthNeedsLogin = crux.ProviderAuthNeedsLogin + ProviderAuthStoreInaccessible = crux.ProviderAuthStoreInaccessible + ProviderAuthUnknown = crux.ProviderAuthUnknown + + ProviderAvailable = crux.ProviderAvailable + ProviderRateLimited = crux.ProviderRateLimited + ProviderQuotaExhausted = crux.ProviderQuotaExhausted + ProviderUnavailable = crux.ProviderUnavailable + ProviderAvailUnknown = crux.ProviderAvailUnknown +) + +// ProviderStatus is one provider's redacted auth/availability roll-up; see +// crux.ProviderStatus. +type ProviderStatus = crux.ProviderStatus + +// ProviderAgentEvidence is the per-agent input to RollUpProviderStatus; see +// crux.ProviderAgentEvidence. +type ProviderAgentEvidence = crux.ProviderAgentEvidence + +// RollUpProviderStatus is crux.RollUpProviderStatus. +func RollUpProviderStatus(agents []ProviderAgentEvidence) []ProviderStatus { + return crux.RollUpProviderStatus(agents) +} diff --git a/src/pkg/agent/quota_exhaustion_test.go b/src/crux/agent/quota_exhaustion_test.go similarity index 100% rename from src/pkg/agent/quota_exhaustion_test.go rename to src/crux/agent/quota_exhaustion_test.go diff --git a/src/pkg/agent/rate_limit_relaunch_test.go b/src/crux/agent/rate_limit_relaunch_test.go similarity index 100% rename from src/pkg/agent/rate_limit_relaunch_test.go rename to src/crux/agent/rate_limit_relaunch_test.go diff --git a/src/pkg/agent/reaper_test.go b/src/crux/agent/reaper_test.go similarity index 98% rename from src/pkg/agent/reaper_test.go rename to src/crux/agent/reaper_test.go index ab05dadbb4..06fc4ce27e 100644 --- a/src/pkg/agent/reaper_test.go +++ b/src/crux/agent/reaper_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestContainsCLIMarker(t *testing.T) { diff --git a/src/pkg/agent/record_prompt_test.go b/src/crux/agent/record_prompt_test.go similarity index 100% rename from src/pkg/agent/record_prompt_test.go rename to src/crux/agent/record_prompt_test.go diff --git a/src/pkg/agent/relaunch_mint_test.go b/src/crux/agent/relaunch_mint_test.go similarity index 99% rename from src/pkg/agent/relaunch_mint_test.go rename to src/crux/agent/relaunch_mint_test.go index f382012652..7001134fdf 100644 --- a/src/pkg/agent/relaunch_mint_test.go +++ b/src/crux/agent/relaunch_mint_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // These tests pin the #3962 fix: EVERY path that (re)launches an agent — diff --git a/src/pkg/agent/relaunch_observer_test.go b/src/crux/agent/relaunch_observer_test.go similarity index 100% rename from src/pkg/agent/relaunch_observer_test.go rename to src/crux/agent/relaunch_observer_test.go diff --git a/src/pkg/agent/relaunch_race_guard_test.go b/src/crux/agent/relaunch_race_guard_test.go similarity index 99% rename from src/pkg/agent/relaunch_race_guard_test.go rename to src/crux/agent/relaunch_race_guard_test.go index bc5568a1c8..93a7881e60 100644 --- a/src/pkg/agent/relaunch_race_guard_test.go +++ b/src/crux/agent/relaunch_race_guard_test.go @@ -18,7 +18,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // blockingMinter parks WriteAgentToken until released, signalling entry so the diff --git a/src/pkg/agent/repeat_failure_bash_env.go b/src/crux/agent/repeat_failure_bash_env.go similarity index 100% rename from src/pkg/agent/repeat_failure_bash_env.go rename to src/crux/agent/repeat_failure_bash_env.go diff --git a/src/pkg/agent/repeat_failure_bash_env_test.go b/src/crux/agent/repeat_failure_bash_env_test.go similarity index 99% rename from src/pkg/agent/repeat_failure_bash_env_test.go rename to src/crux/agent/repeat_failure_bash_env_test.go index c760ca6660..52ba1c6cab 100644 --- a/src/pkg/agent/repeat_failure_bash_env_test.go +++ b/src/crux/agent/repeat_failure_bash_env_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/crux/config" ) // requireBash skips when the host has no /bin/bash to run the route with. diff --git a/src/pkg/agent/repeat_failure_detect.go b/src/crux/agent/repeat_failure_detect.go similarity index 100% rename from src/pkg/agent/repeat_failure_detect.go rename to src/crux/agent/repeat_failure_detect.go diff --git a/src/pkg/agent/repeat_failure_detect_test.go b/src/crux/agent/repeat_failure_detect_test.go similarity index 100% rename from src/pkg/agent/repeat_failure_detect_test.go rename to src/crux/agent/repeat_failure_detect_test.go diff --git a/src/pkg/agent/repeat_failure_normalize.go b/src/crux/agent/repeat_failure_normalize.go similarity index 100% rename from src/pkg/agent/repeat_failure_normalize.go rename to src/crux/agent/repeat_failure_normalize.go diff --git a/src/pkg/agent/repeat_failure_normalize_test.go b/src/crux/agent/repeat_failure_normalize_test.go similarity index 100% rename from src/pkg/agent/repeat_failure_normalize_test.go rename to src/crux/agent/repeat_failure_normalize_test.go diff --git a/src/pkg/agent/repeat_failure_record.go b/src/crux/agent/repeat_failure_record.go similarity index 99% rename from src/pkg/agent/repeat_failure_record.go rename to src/crux/agent/repeat_failure_record.go index b6e62d9e18..0cf6852e6a 100644 --- a/src/pkg/agent/repeat_failure_record.go +++ b/src/crux/agent/repeat_failure_record.go @@ -16,8 +16,8 @@ import ( "time" "unicode/utf8" - "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/logscrub" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/logscrub" ) // Command records for the repeat-failure check (hivecommons/hive#14550, C1 diff --git a/src/pkg/agent/repeat_failure_record_test.go b/src/crux/agent/repeat_failure_record_test.go similarity index 99% rename from src/pkg/agent/repeat_failure_record_test.go rename to src/crux/agent/repeat_failure_record_test.go index 594b04c8b9..913aa4ae29 100644 --- a/src/pkg/agent/repeat_failure_record_test.go +++ b/src/crux/agent/repeat_failure_record_test.go @@ -15,7 +15,7 @@ import ( "time" "unicode/utf8" - "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/crux/config" ) const repeatFailureTestToken = "ghp_abcdefghijklmnopqrstuvwxyz0123456789" diff --git a/src/pkg/agent/repo_pause_test.go b/src/crux/agent/repo_pause_test.go similarity index 98% rename from src/pkg/agent/repo_pause_test.go rename to src/crux/agent/repo_pause_test.go index d9afc9f653..7ba35ae38b 100644 --- a/src/pkg/agent/repo_pause_test.go +++ b/src/crux/agent/repo_pause_test.go @@ -3,7 +3,7 @@ package agent import ( "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func pausedProject(paused ...string) ProjectContext { diff --git a/src/pkg/agent/restart_storm_breaker.go b/src/crux/agent/restart_storm_breaker.go similarity index 94% rename from src/pkg/agent/restart_storm_breaker.go rename to src/crux/agent/restart_storm_breaker.go index 5eaa011a53..563dd50b29 100644 --- a/src/pkg/agent/restart_storm_breaker.go +++ b/src/crux/agent/restart_storm_breaker.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/fleetreport" + "github.com/hivecommons/hive/crux" ) const ( @@ -64,9 +64,9 @@ func (m *Manager) applyRestartStormPlan(name, reason string, plan restartStormPl "agent", name, "reason", reason, "trigger", plan.pauseTrigger, "pause_reason", plan.pauseReason) } if plan.pauseTrigger == DiskFullTrigger { - fleetreport.Observe(fleetreport.ClassDataDiskPressure, DataDiskPressureKey, "data disk pressure: "+plan.pauseReason, "") + m.observeCondition(crux.ClassDataDiskPressure, DataDiskPressureKey, "data disk pressure: "+plan.pauseReason, "") } else { - fleetreport.Observe(fleetreport.ClassRestartStorm, name, "restart storm: "+reason, plan.pauseReason) + m.observeCondition(crux.ClassRestartStorm, name, "restart storm: "+reason, plan.pauseReason) } return true, false, m.Pause(name, plan.pauseTrigger, plan.pauseReason) case plan.backoff > 0: diff --git a/src/pkg/agent/restart_telemetry_test.go b/src/crux/agent/restart_telemetry_test.go similarity index 99% rename from src/pkg/agent/restart_telemetry_test.go rename to src/crux/agent/restart_telemetry_test.go index fdb5f43a5a..094586ec3b 100644 --- a/src/pkg/agent/restart_telemetry_test.go +++ b/src/crux/agent/restart_telemetry_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestRestartTelemetryPrunesRollingWindowAndReset(t *testing.T) { diff --git a/src/crux/agent/ringbuffer.go b/src/crux/agent/ringbuffer.go new file mode 100644 index 0000000000..5d2130c8fe --- /dev/null +++ b/src/crux/agent/ringbuffer.go @@ -0,0 +1,13 @@ +package agent + +import "github.com/hivecommons/hive/crux" + +// RingBuffer is the bounded output buffer behind Process.OutputBuffer. It is +// defined in the crux root package so crux.Process can carry it without the +// root importing this package. +type RingBuffer = crux.RingBuffer + +// NewRingBuffer returns a ring buffer holding the last capacity lines. +func NewRingBuffer(capacity int) *RingBuffer { + return crux.NewRingBuffer(capacity) +} diff --git a/src/pkg/agent/routing_coverage_test.go b/src/crux/agent/routing_coverage_test.go similarity index 99% rename from src/pkg/agent/routing_coverage_test.go rename to src/crux/agent/routing_coverage_test.go index 0c4ebb54fb..ecfc5b8388 100644 --- a/src/pkg/agent/routing_coverage_test.go +++ b/src/crux/agent/routing_coverage_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // --------------------------------------------------------------------------- diff --git a/src/pkg/agent/runtime_uid_isolation_marker_test.go b/src/crux/agent/runtime_uid_isolation_marker_test.go similarity index 99% rename from src/pkg/agent/runtime_uid_isolation_marker_test.go rename to src/crux/agent/runtime_uid_isolation_marker_test.go index aebfcda464..3e25111129 100644 --- a/src/pkg/agent/runtime_uid_isolation_marker_test.go +++ b/src/crux/agent/runtime_uid_isolation_marker_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // These tests pin the fix for the runtime-added-agent launch wedge: the diff --git a/src/pkg/agent/sandbox_executor.go b/src/crux/agent/sandbox_executor.go similarity index 97% rename from src/pkg/agent/sandbox_executor.go rename to src/crux/agent/sandbox_executor.go index d9e3160c5b..a60712fac3 100644 --- a/src/pkg/agent/sandbox_executor.go +++ b/src/crux/agent/sandbox_executor.go @@ -12,12 +12,12 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/crux/types" - "github.com/hivecommons/hive/pkg/effects" - "github.com/hivecommons/hive/pkg/logscrub" - "github.com/hivecommons/hive/pkg/outputschema" - "github.com/hivecommons/hive/pkg/pushbroker" - "github.com/hivecommons/hive/pkg/sandbox" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/effects" + "github.com/hivecommons/hive/crux/logscrub" + "github.com/hivecommons/hive/crux/outputschema" + "github.com/hivecommons/hive/crux/pushbroker" + "github.com/hivecommons/hive/crux/sandbox" ) const ( @@ -40,9 +40,8 @@ func (sandboxExecRunner) Run(ctx context.Context, dir string, env []string, name return cmd.CombinedOutput() } -type PRCreator interface { - CreatePR(ctx context.Context, repo, head, base, title, body string) (types.PRResult, error) -} +// PRCreator is crux.PRCreator, the host-provided pull-request capability. +type PRCreator = crux.PRCreator type SandboxKickSpec struct { Agent string @@ -80,7 +79,7 @@ type SandboxKickResult struct { TranscriptPath string Sandbox sandbox.Result Broker *pushbroker.Result - PR *types.PRResult + PR *crux.PRResult TimedOut bool Error string } diff --git a/src/pkg/agent/sandbox_executor_baseref_test.go b/src/crux/agent/sandbox_executor_baseref_test.go similarity index 97% rename from src/pkg/agent/sandbox_executor_baseref_test.go rename to src/crux/agent/sandbox_executor_baseref_test.go index 1cb470c41b..674f0c2c8d 100644 --- a/src/pkg/agent/sandbox_executor_baseref_test.go +++ b/src/crux/agent/sandbox_executor_baseref_test.go @@ -9,7 +9,7 @@ import ( "sync" "testing" - "github.com/hivecommons/hive/pkg/crux/types" + "github.com/hivecommons/hive/crux" ) // Regression coverage for kubestellar/hive#4928 on the sandbox path: the @@ -83,9 +83,9 @@ func (r *baseRefRunner) Run(ctx context.Context, dir string, env []string, name // baseRefPRClient records the base the executor asked the hive to open against. type baseRefPRClient struct{ base string } -func (p *baseRefPRClient) CreatePR(_ context.Context, _, _, base, _, _ string) (types.PRResult, error) { +func (p *baseRefPRClient) CreatePR(_ context.Context, _, _, base, _, _ string) (crux.PRResult, error) { p.base = base - return types.PRResult{Number: 1, URL: "https://example.test/pr/1"}, nil + return crux.PRResult{Number: 1, URL: "https://example.test/pr/1"}, nil } func runSandboxForBaseRef(t *testing.T, runner *baseRefRunner, specBase string) (*baseRefPRClient, SandboxKickResult, error) { diff --git a/src/pkg/agent/sandbox_executor_paths_test.go b/src/crux/agent/sandbox_executor_paths_test.go similarity index 98% rename from src/pkg/agent/sandbox_executor_paths_test.go rename to src/crux/agent/sandbox_executor_paths_test.go index 87688dd6ad..59dfb9d883 100644 --- a/src/pkg/agent/sandbox_executor_paths_test.go +++ b/src/crux/agent/sandbox_executor_paths_test.go @@ -9,10 +9,10 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/crux/types" - "github.com/hivecommons/hive/pkg/pushbroker" - "github.com/hivecommons/hive/pkg/sandbox" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/pushbroker" + "github.com/hivecommons/hive/crux/sandbox" ) // These tests pin the SandboxExecutor's error handling and the pure command/ @@ -57,9 +57,9 @@ type fakePRClient struct { called bool } -func (c *fakePRClient) CreatePR(ctx context.Context, repo, head, base, title, body string) (types.PRResult, error) { +func (c *fakePRClient) CreatePR(ctx context.Context, repo, head, base, title, body string) (crux.PRResult, error) { c.called = true - return types.PRResult{URL: "https://example.invalid/pr/1"}, c.err + return crux.PRResult{URL: "https://example.invalid/pr/1"}, c.err } func sandboxPathsSpec(workdir string) SandboxKickSpec { diff --git a/src/pkg/agent/sandbox_executor_test.go b/src/crux/agent/sandbox_executor_test.go similarity index 98% rename from src/pkg/agent/sandbox_executor_test.go rename to src/crux/agent/sandbox_executor_test.go index 48ed2d9f18..aa271f4496 100644 --- a/src/pkg/agent/sandbox_executor_test.go +++ b/src/crux/agent/sandbox_executor_test.go @@ -11,10 +11,10 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/crux/types" - "github.com/hivecommons/hive/pkg/pushbroker" - "github.com/hivecommons/hive/pkg/sandbox" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/pushbroker" + "github.com/hivecommons/hive/crux/sandbox" ) const sandboxStateTransitionTimeout = 30 * time.Second @@ -132,9 +132,9 @@ func (sandboxFakeMinter) MintPushToken(context.Context, string) (string, error) type sandboxFakePR struct{ called bool } -func (p *sandboxFakePR) CreatePR(context.Context, string, string, string, string, string) (types.PRResult, error) { +func (p *sandboxFakePR) CreatePR(context.Context, string, string, string, string, string) (crux.PRResult, error) { p.called = true - return types.PRResult{Number: 1, URL: "https://example.test/pr/1"}, nil + return crux.PRResult{Number: 1, URL: "https://example.test/pr/1"}, nil } func TestSandboxExecutorWorkspacePrepSanitizesCredentials(t *testing.T) { diff --git a/src/pkg/agent/sandbox_job_runtime_test.go b/src/crux/agent/sandbox_job_runtime_test.go similarity index 98% rename from src/pkg/agent/sandbox_job_runtime_test.go rename to src/crux/agent/sandbox_job_runtime_test.go index 6cbaf66510..911546990a 100644 --- a/src/pkg/agent/sandbox_job_runtime_test.go +++ b/src/crux/agent/sandbox_job_runtime_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/sandbox" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" ) // The Job runtime (#6311) must route a sandboxed kick through the launcher the diff --git a/src/pkg/agent/seams_coverage_test.go b/src/crux/agent/seams_coverage_test.go similarity index 99% rename from src/pkg/agent/seams_coverage_test.go rename to src/crux/agent/seams_coverage_test.go index d48ae3e16c..b4b28669d6 100644 --- a/src/pkg/agent/seams_coverage_test.go +++ b/src/crux/agent/seams_coverage_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // These tests redirect the package-level path vars (a minimal testability seam: diff --git a/src/pkg/agent/sendkick_coverage_test.go b/src/crux/agent/sendkick_coverage_test.go similarity index 96% rename from src/pkg/agent/sendkick_coverage_test.go rename to src/crux/agent/sendkick_coverage_test.go index 1dbdc29688..a7cd87d4b8 100644 --- a/src/pkg/agent/sendkick_coverage_test.go +++ b/src/crux/agent/sendkick_coverage_test.go @@ -3,7 +3,7 @@ package agent import ( "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestSendKick_DeliversToReadyPane covers SendKick's full success path diff --git a/src/pkg/agent/session_liveness.go b/src/crux/agent/session_liveness.go similarity index 100% rename from src/pkg/agent/session_liveness.go rename to src/crux/agent/session_liveness.go diff --git a/src/pkg/agent/session_liveness_test.go b/src/crux/agent/session_liveness_test.go similarity index 100% rename from src/pkg/agent/session_liveness_test.go rename to src/crux/agent/session_liveness_test.go diff --git a/src/pkg/agent/session_missing_test.go b/src/crux/agent/session_missing_test.go similarity index 100% rename from src/pkg/agent/session_missing_test.go rename to src/crux/agent/session_missing_test.go diff --git a/src/pkg/agent/shared_credential_test.go b/src/crux/agent/shared_credential_test.go similarity index 99% rename from src/pkg/agent/shared_credential_test.go rename to src/crux/agent/shared_credential_test.go index d5dc5ef7e6..c872547e7f 100644 --- a/src/pkg/agent/shared_credential_test.go +++ b/src/crux/agent/shared_credential_test.go @@ -11,8 +11,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/claude" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/claude" + "github.com/hivecommons/hive/crux/config" ) // Regression coverage for kubestellar/hive#5730. diff --git a/src/pkg/agent/shared_dev_uid_test.go b/src/crux/agent/shared_dev_uid_test.go similarity index 97% rename from src/pkg/agent/shared_dev_uid_test.go rename to src/crux/agent/shared_dev_uid_test.go index 33c9198162..1855766ea5 100644 --- a/src/pkg/agent/shared_dev_uid_test.go +++ b/src/crux/agent/shared_dev_uid_test.go @@ -4,7 +4,7 @@ import ( "log/slog" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // SharedDevUID feeds the uid_isolation health check (#11713): it must report diff --git a/src/pkg/agent/stale_launch_cmd_test.go b/src/crux/agent/stale_launch_cmd_test.go similarity index 98% rename from src/pkg/agent/stale_launch_cmd_test.go rename to src/crux/agent/stale_launch_cmd_test.go index b8693d99e4..1f34a940cb 100644 --- a/src/pkg/agent/stale_launch_cmd_test.go +++ b/src/crux/agent/stale_launch_cmd_test.go @@ -5,7 +5,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // A launch_cmd pinned to a backend binary path that does not exist (the diff --git a/src/crux/agent/stall_selfreport_test.go b/src/crux/agent/stall_selfreport_test.go new file mode 100644 index 0000000000..899ec2d819 --- /dev/null +++ b/src/crux/agent/stall_selfreport_test.go @@ -0,0 +1,53 @@ +package agent + +import ( + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" +) + +// A swallowed "continue" nudge feeds the agent_stall self-report with the +// kick/pane context the retired dashboard counter never had (#13470). Every +// nudge is observed (the host's tracker applies its streak before filing) +// and a credited recovery clears the condition. +func TestObserveStallNudgeFeedsSelfReportAndRecoveryClears(t *testing.T) { + m := &Manager{} + rec := recordConditions(t, m) + kickAt := time.Date(2026, 10, 21, 11, 0, 0, 0, time.UTC) + agent := &AgentProcess{Name: "scanner", Config: config.AgentConfig{Role: "scanner"}, lastInferKickAt: kickAt, lastInferKickPane: paneContentHash("frozen pane"), stallNudgeSent: true} + pane := "line one\ntoken ghp_abcdefghijklmnopqrstuvwxyz123456\nfrozen pane" + const nudges = 3 + for i := 0; i < nudges; i++ { + agent.StallNudges++ + m.observeStallNudgeLocked(agent, pane, kickAt.Add(5*time.Minute)) + } + ev := rec.active(crux.ClassAgentStall) + if len(ev) != nudges || ev[0].Key != "scanner" { + t.Fatalf("evidence = %#v", ev) + } + last := ev[nudges-1] + for _, want := range []string{"continue nudge(s) sent without the pane moving", "last kick 2026-10-21T11:00:00Z", "3 continue nudge(s)"} { + if !strings.Contains(last.Message, want) { + t.Errorf("message %q missing %q", last.Message, want) + } + } + for _, want := range []string{"agent=scanner", "lane=scanner", "stall_nudges=3", "pane tail", "frozen pane"} { + if !strings.Contains(last.Diagnostics, want) { + t.Errorf("diagnostics missing %q:\n%s", want, last.Diagnostics) + } + } + if strings.Contains(last.Diagnostics, "ghp_abcdefghijklmnopqrstuvwxyz123456") { + t.Fatalf("pane tail leaked a token:\n%s", last.Diagnostics) + } + // The pane moving before the next kick is the recovery that clears it. + m.creditStallRecoveryLocked(agent, paneContentHash("moved on")) + if ev := rec.active(crux.ClassAgentStall); len(ev) != 0 { + t.Fatalf("recovery did not clear the stall: %#v", ev) + } + if agent.StallRecoveries != 1 { + t.Fatalf("recoveries = %d", agent.StallRecoveries) + } +} diff --git a/src/pkg/agent/start_failure.go b/src/crux/agent/start_failure.go similarity index 92% rename from src/pkg/agent/start_failure.go rename to src/crux/agent/start_failure.go index 9cd4b9d8cd..0530e1c921 100644 --- a/src/pkg/agent/start_failure.go +++ b/src/crux/agent/start_failure.go @@ -7,22 +7,22 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/crux/types" + "github.com/hivecommons/hive/crux" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) -// StartFailureClass and its values are defined in pkg/crux/types as part of +// StartFailureClass and its values are defined in pkg/crux as part of // the Crux runtime surface (hivecommons/hive#13536); aliased here. -type StartFailureClass = types.StartFailureClass +type StartFailureClass = crux.StartFailureClass const ( - StartFailureLoginRequired = types.StartFailureLoginRequired - StartFailureCredentialRejected = types.StartFailureCredentialRejected - StartFailureCredentialMissing = types.StartFailureCredentialMissing - StartFailureBackendMismatch = types.StartFailureBackendMismatch - StartFailureBinaryMissing = types.StartFailureBinaryMissing - StartFailureNoOutput = types.StartFailureNoOutput + StartFailureLoginRequired = crux.StartFailureLoginRequired + StartFailureCredentialRejected = crux.StartFailureCredentialRejected + StartFailureCredentialMissing = crux.StartFailureCredentialMissing + StartFailureBackendMismatch = crux.StartFailureBackendMismatch + StartFailureBinaryMissing = crux.StartFailureBinaryMissing + StartFailureNoOutput = crux.StartFailureNoOutput ) // startFailureBlockThresholdDefault is how many CONSECUTIVE failures of the @@ -309,19 +309,12 @@ func (m *Manager) startFailureBackoffRemainingLocked(agent *AgentProcess, now ti return agent.StartBackoffUntil.Sub(now) } -// StartFailureState reports an agent's current start-failure record for callers -// outside the package (the dashboard status builder, the heartbeat). blocked is -// the fleet-visible bit: it means the agent has failed the same way +// StartFailureSnapshot is crux.StartFailureSnapshot; StartFailureState +// reports it for callers outside the package (the dashboard status builder, +// the heartbeat). Blocked means the agent has failed the same way // startFailureBlockThreshold times and is no longer being relaunched on the // tick. -type StartFailureSnapshot struct { - Reason string - Count int - LastAt time.Time - LastExitCode *int - LastSignal string - Blocked bool -} +type StartFailureSnapshot = crux.StartFailureSnapshot func (m *Manager) StartFailureState(name string) (StartFailureSnapshot, bool) { m.mu.RLock() diff --git a/src/pkg/agent/start_failure_test.go b/src/crux/agent/start_failure_test.go similarity index 100% rename from src/pkg/agent/start_failure_test.go rename to src/crux/agent/start_failure_test.go diff --git a/src/pkg/agent/start_launch_guard_test.go b/src/crux/agent/start_launch_guard_test.go similarity index 98% rename from src/pkg/agent/start_launch_guard_test.go rename to src/crux/agent/start_launch_guard_test.go index cd2fa356b4..1267c51222 100644 --- a/src/pkg/agent/start_launch_guard_test.go +++ b/src/crux/agent/start_launch_guard_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // These tests cover the per-agent launch guard that lets Manager.Start run its diff --git a/src/pkg/agent/status_fast_test.go b/src/crux/agent/status_fast_test.go similarity index 97% rename from src/pkg/agent/status_fast_test.go rename to src/crux/agent/status_fast_test.go index e9a096b23b..3e58a875b9 100644 --- a/src/pkg/agent/status_fast_test.go +++ b/src/crux/agent/status_fast_test.go @@ -3,6 +3,8 @@ package agent import ( "testing" "time" + + "github.com/hivecommons/hive/crux" ) // A restart holds the global m.mu in write mode across the whole tmux relaunch @@ -38,7 +40,7 @@ func TestGetStatusFastDoesNotBlockOnHeldWriteLock(t *testing.T) { <-held defer close(released) - done := make(chan *AgentProcess, 1) + done := make(chan *crux.Process, 1) go func() { proc, err := m.GetStatusFast("scanner") if err != nil { diff --git a/src/pkg/agent/stuck_login_diagnosis.go b/src/crux/agent/stuck_login_diagnosis.go similarity index 99% rename from src/pkg/agent/stuck_login_diagnosis.go rename to src/crux/agent/stuck_login_diagnosis.go index a8c5fcf234..d0e1481dbf 100644 --- a/src/pkg/agent/stuck_login_diagnosis.go +++ b/src/crux/agent/stuck_login_diagnosis.go @@ -4,7 +4,7 @@ import ( "fmt" "time" - "github.com/hivecommons/hive/pkg/claude" + "github.com/hivecommons/hive/crux/claude" ) // diagnoseStuckLogin explains why an agent is still sitting at a login prompt diff --git a/src/pkg/agent/stuck_login_diagnosis_test.go b/src/crux/agent/stuck_login_diagnosis_test.go similarity index 99% rename from src/pkg/agent/stuck_login_diagnosis_test.go rename to src/crux/agent/stuck_login_diagnosis_test.go index 7803243e67..e165b60c90 100644 --- a/src/pkg/agent/stuck_login_diagnosis_test.go +++ b/src/crux/agent/stuck_login_diagnosis_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // #4596: an agent can sit at a login prompt forever while every file-level diff --git a/src/pkg/agent/subagent_models.go b/src/crux/agent/subagent_models.go similarity index 94% rename from src/pkg/agent/subagent_models.go rename to src/crux/agent/subagent_models.go index c56a092395..ece6bee738 100644 --- a/src/pkg/agent/subagent_models.go +++ b/src/crux/agent/subagent_models.go @@ -5,11 +5,11 @@ import ( "strconv" "strings" - "github.com/hivecommons/hive/pkg/crux/types" + "github.com/hivecommons/hive/crux" ) -// SubAgentModel is an observed Copilot sub-agent row; see types.SubAgentModel. -type SubAgentModel = types.SubAgentModel +// SubAgentModel is an observed Copilot sub-agent row; see crux.SubAgentModel. +type SubAgentModel = crux.SubAgentModel // Anchor the whole row: policy prose mentioning a model must not become a // model observation. Accept the TUI's status/tree decoration, not arbitrary text. diff --git a/src/pkg/agent/subagent_models_test.go b/src/crux/agent/subagent_models_test.go similarity index 98% rename from src/pkg/agent/subagent_models_test.go rename to src/crux/agent/subagent_models_test.go index 09d1d8cc77..f75591625c 100644 --- a/src/pkg/agent/subagent_models_test.go +++ b/src/crux/agent/subagent_models_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestSubAgentModelRows(t *testing.T) { diff --git a/src/pkg/agent/task_mcp_connection_test.go b/src/crux/agent/task_mcp_connection_test.go similarity index 99% rename from src/pkg/agent/task_mcp_connection_test.go rename to src/crux/agent/task_mcp_connection_test.go index 2f8ce2d0ee..318578f9f8 100644 --- a/src/pkg/agent/task_mcp_connection_test.go +++ b/src/crux/agent/task_mcp_connection_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/taskmcp" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/taskmcp" ) func TestTaskMCPConnectionAddedForClaudeLaunch(t *testing.T) { diff --git a/src/pkg/agent/task_mcp_scope.go b/src/crux/agent/task_mcp_scope.go similarity index 95% rename from src/pkg/agent/task_mcp_scope.go rename to src/crux/agent/task_mcp_scope.go index b68e459d72..c7e382af00 100644 --- a/src/pkg/agent/task_mcp_scope.go +++ b/src/crux/agent/task_mcp_scope.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" - "github.com/hivecommons/hive/pkg/config" - "github.com/hivecommons/hive/pkg/taskmcp" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/taskmcp" ) func (m *Manager) ActiveLaunches() []taskmcp.LaunchScope { diff --git a/src/pkg/agent/terminal.go b/src/crux/agent/terminal.go similarity index 100% rename from src/pkg/agent/terminal.go rename to src/crux/agent/terminal.go diff --git a/src/pkg/agent/terminal_seams_test.go b/src/crux/agent/terminal_seams_test.go similarity index 100% rename from src/pkg/agent/terminal_seams_test.go rename to src/crux/agent/terminal_seams_test.go diff --git a/src/pkg/agent/terminal_send_literal_test.go b/src/crux/agent/terminal_send_literal_test.go similarity index 98% rename from src/pkg/agent/terminal_send_literal_test.go rename to src/crux/agent/terminal_send_literal_test.go index dd02380a83..496a1358ce 100644 --- a/src/pkg/agent/terminal_send_literal_test.go +++ b/src/crux/agent/terminal_send_literal_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // A kick is typed into the pane as 400-rune chunks via `tmux send-keys -l`. diff --git a/src/pkg/agent/terminal_tmux_hermetic_test.go b/src/crux/agent/terminal_tmux_hermetic_test.go similarity index 100% rename from src/pkg/agent/terminal_tmux_hermetic_test.go rename to src/crux/agent/terminal_tmux_hermetic_test.go diff --git a/src/pkg/agent/testdata/kick_transcript/claude_session.golden b/src/crux/agent/testdata/kick_transcript/claude_session.golden similarity index 100% rename from src/pkg/agent/testdata/kick_transcript/claude_session.golden rename to src/crux/agent/testdata/kick_transcript/claude_session.golden diff --git a/src/pkg/agent/testdata/kick_transcript/claude_session.jsonl b/src/crux/agent/testdata/kick_transcript/claude_session.jsonl similarity index 100% rename from src/pkg/agent/testdata/kick_transcript/claude_session.jsonl rename to src/crux/agent/testdata/kick_transcript/claude_session.jsonl diff --git a/src/pkg/agent/testdata/kick_transcript/copilot_events.golden b/src/crux/agent/testdata/kick_transcript/copilot_events.golden similarity index 100% rename from src/pkg/agent/testdata/kick_transcript/copilot_events.golden rename to src/crux/agent/testdata/kick_transcript/copilot_events.golden diff --git a/src/pkg/agent/testdata/kick_transcript/copilot_events.jsonl b/src/crux/agent/testdata/kick_transcript/copilot_events.jsonl similarity index 100% rename from src/pkg/agent/testdata/kick_transcript/copilot_events.jsonl rename to src/crux/agent/testdata/kick_transcript/copilot_events.jsonl diff --git a/src/pkg/agent/thrash_coverage_test.go b/src/crux/agent/thrash_coverage_test.go similarity index 100% rename from src/pkg/agent/thrash_coverage_test.go rename to src/crux/agent/thrash_coverage_test.go diff --git a/src/pkg/agent/thrash_poller_wiring_test.go b/src/crux/agent/thrash_poller_wiring_test.go similarity index 97% rename from src/pkg/agent/thrash_poller_wiring_test.go rename to src/crux/agent/thrash_poller_wiring_test.go index de5bfeede8..72da489640 100644 --- a/src/pkg/agent/thrash_poller_wiring_test.go +++ b/src/crux/agent/thrash_poller_wiring_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/internal/testutil" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" ) // TestPollTmuxOutputForAgent_WiresThrashBreaker pins the poller-to-breaker diff --git a/src/pkg/agent/tmux/tmux.go b/src/crux/agent/tmux/tmux.go similarity index 100% rename from src/pkg/agent/tmux/tmux.go rename to src/crux/agent/tmux/tmux.go diff --git a/src/pkg/agent/tmux/tmux_test.go b/src/crux/agent/tmux/tmux_test.go similarity index 100% rename from src/pkg/agent/tmux/tmux_test.go rename to src/crux/agent/tmux/tmux_test.go diff --git a/src/pkg/agent/tmux_coverage_test.go b/src/crux/agent/tmux_coverage_test.go similarity index 99% rename from src/pkg/agent/tmux_coverage_test.go rename to src/crux/agent/tmux_coverage_test.go index 5b5bac150e..7fdea77396 100644 --- a/src/pkg/agent/tmux_coverage_test.go +++ b/src/crux/agent/tmux_coverage_test.go @@ -11,8 +11,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/internal/testutil" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" ) // tmuxAvailable reports whether a usable tmux binary is on PATH. All tmux-based diff --git a/src/pkg/agent/tmux_false_missing_test.go b/src/crux/agent/tmux_false_missing_test.go similarity index 98% rename from src/pkg/agent/tmux_false_missing_test.go rename to src/crux/agent/tmux_false_missing_test.go index d38f354c63..b68ac67c11 100644 --- a/src/pkg/agent/tmux_false_missing_test.go +++ b/src/crux/agent/tmux_false_missing_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func writeFakeTmux(t *testing.T, script string) string { diff --git a/src/pkg/agent/tmux_history_limit_test.go b/src/crux/agent/tmux_history_limit_test.go similarity index 100% rename from src/pkg/agent/tmux_history_limit_test.go rename to src/crux/agent/tmux_history_limit_test.go diff --git a/src/pkg/agent/tmux_inherited_env_test.go b/src/crux/agent/tmux_inherited_env_test.go similarity index 98% rename from src/pkg/agent/tmux_inherited_env_test.go rename to src/crux/agent/tmux_inherited_env_test.go index 29996dc349..74a728cb7b 100644 --- a/src/pkg/agent/tmux_inherited_env_test.go +++ b/src/crux/agent/tmux_inherited_env_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/internal/testutil" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/internal/testutil" ) // A tmux server inherits the environment of the process that starts it, and diff --git a/src/pkg/agent/tmux_lifecycle_hermetic_test.go b/src/crux/agent/tmux_lifecycle_hermetic_test.go similarity index 98% rename from src/pkg/agent/tmux_lifecycle_hermetic_test.go rename to src/crux/agent/tmux_lifecycle_hermetic_test.go index 772a093e54..9a7966082d 100644 --- a/src/pkg/agent/tmux_lifecycle_hermetic_test.go +++ b/src/crux/agent/tmux_lifecycle_hermetic_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestHermeticWaitForInputPromptForAgentSkipsConsentScreen(t *testing.T) { diff --git a/src/pkg/agent/tmux_scrollback_legibility_test.go b/src/crux/agent/tmux_scrollback_legibility_test.go similarity index 100% rename from src/pkg/agent/tmux_scrollback_legibility_test.go rename to src/crux/agent/tmux_scrollback_legibility_test.go diff --git a/src/pkg/agent/token_refresh_test.go b/src/crux/agent/token_refresh_test.go similarity index 99% rename from src/pkg/agent/token_refresh_test.go rename to src/crux/agent/token_refresh_test.go index c5ecc802ec..04e28824c1 100644 --- a/src/pkg/agent/token_refresh_test.go +++ b/src/crux/agent/token_refresh_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // syncedMinter is a race-safe fake for tests where the refresh loop mints diff --git a/src/pkg/agent/token_restart_boot_pane_test.go b/src/crux/agent/token_restart_boot_pane_test.go similarity index 100% rename from src/pkg/agent/token_restart_boot_pane_test.go rename to src/crux/agent/token_restart_boot_pane_test.go diff --git a/src/pkg/agent/tracing.go b/src/crux/agent/tracing.go similarity index 87% rename from src/pkg/agent/tracing.go rename to src/crux/agent/tracing.go index 3e9d445f13..8d82d5b7f0 100644 --- a/src/pkg/agent/tracing.go +++ b/src/crux/agent/tracing.go @@ -6,17 +6,19 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + + "github.com/hivecommons/hive/crux" ) // spanInstrumentationName is the instrumentation scope the runtime's spans // carry when no SpanStarter is installed. -const spanInstrumentationName = "github.com/hivecommons/hive/pkg/agent" +const spanInstrumentationName = "github.com/hivecommons/hive/crux/agent" // SpanStarter opens a trace span. It has the shape of pkg/tracing.StartSpan // so the host can install that function and keep its per-component reach // bookkeeping, while the runtime itself depends only on the OpenTelemetry // API (hivecommons/hive#13536). -type SpanStarter func(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) +type SpanStarter = crux.SpanStarter // SetSpanStarter installs the function that opens the runtime's trace spans. // Safe to call at any time; nil restores the default, which starts spans on diff --git a/src/pkg/agent/tracing_test.go b/src/crux/agent/tracing_test.go similarity index 100% rename from src/pkg/agent/tracing_test.go rename to src/crux/agent/tracing_test.go diff --git a/src/pkg/agent/trajectory_idle_test.go b/src/crux/agent/trajectory_idle_test.go similarity index 100% rename from src/pkg/agent/trajectory_idle_test.go rename to src/crux/agent/trajectory_idle_test.go diff --git a/src/pkg/agent/trajectory_view.go b/src/crux/agent/trajectory_view.go similarity index 97% rename from src/pkg/agent/trajectory_view.go rename to src/crux/agent/trajectory_view.go index dec0643bce..03f7b0902e 100644 --- a/src/pkg/agent/trajectory_view.go +++ b/src/crux/agent/trajectory_view.go @@ -1,7 +1,7 @@ package agent import ( - "github.com/hivecommons/hive/pkg/trajectory" + "github.com/hivecommons/hive/crux/trajectory" ) // trajectoryTranscriptLines is how many trailing pane lines are gathered per diff --git a/src/pkg/agent/transient_api_error_nudge_test.go b/src/crux/agent/transient_api_error_nudge_test.go similarity index 99% rename from src/pkg/agent/transient_api_error_nudge_test.go rename to src/crux/agent/transient_api_error_nudge_test.go index 48d48d4191..bfce8ff1b0 100644 --- a/src/pkg/agent/transient_api_error_nudge_test.go +++ b/src/crux/agent/transient_api_error_nudge_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // transient_api_error_nudge_test.go covers the #4697 recovery path: a CLI agent diff --git a/src/pkg/agent/tui_menu.go b/src/crux/agent/tui_menu.go similarity index 100% rename from src/pkg/agent/tui_menu.go rename to src/crux/agent/tui_menu.go diff --git a/src/pkg/agent/tui_menu_test.go b/src/crux/agent/tui_menu_test.go similarity index 99% rename from src/pkg/agent/tui_menu_test.go rename to src/crux/agent/tui_menu_test.go index c851d5ea1d..171731b45c 100644 --- a/src/pkg/agent/tui_menu_test.go +++ b/src/crux/agent/tui_menu_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // codexHooksMenuPane is the live Codex 0.163 /hooks screen from #13765. Its diff --git a/src/pkg/agent/turn_loss.go b/src/crux/agent/turn_loss.go similarity index 95% rename from src/pkg/agent/turn_loss.go rename to src/crux/agent/turn_loss.go index 7e30833465..951d60771b 100644 --- a/src/pkg/agent/turn_loss.go +++ b/src/crux/agent/turn_loss.go @@ -3,7 +3,7 @@ package agent import ( "time" - "github.com/hivecommons/hive/pkg/crux/types" + "github.com/hivecommons/hive/crux" ) // turnLossNow is the clock this file reads, seamed for tests in the same @@ -17,16 +17,16 @@ var turnLossNow = time.Now // loss (a few long turns vs. many short ones), which a handful conveys. const turnLossRecentCapacity = 20 -// TurnInterruption and TurnLoss are defined in pkg/crux/types as part of the +// TurnInterruption and TurnLoss are defined in pkg/crux as part of the // Crux runtime surface (hivecommons/hive#13536); aliased here. TurnLoss is // persisted through snapshot.AgentState into /data/hive-state.json — the // store that already carries RestartCount — rather than a new sidecar file, // per the RFC's explicit caution against adding durable store number five. -type TurnInterruption = types.TurnInterruption +type TurnInterruption = crux.TurnInterruption // TurnLoss is the per-agent accumulation that survives the restart it -// measures; see types.TurnLoss. -type TurnLoss = types.TurnLoss +// measures; see crux.TurnLoss. +type TurnLoss = crux.TurnLoss // noteTurnInterruptedLocked records one interrupted turn against the agent. // Callers must hold m.mu; it reads pane observations under paneMu, which is diff --git a/src/pkg/agent/turn_loss_test.go b/src/crux/agent/turn_loss_test.go similarity index 100% rename from src/pkg/agent/turn_loss_test.go rename to src/crux/agent/turn_loss_test.go diff --git a/src/pkg/agent/uid_isolation_migration_test.go b/src/crux/agent/uid_isolation_migration_test.go similarity index 100% rename from src/pkg/agent/uid_isolation_migration_test.go rename to src/crux/agent/uid_isolation_migration_test.go diff --git a/src/pkg/agent/uidmap.go b/src/crux/agent/uidmap.go similarity index 100% rename from src/pkg/agent/uidmap.go rename to src/crux/agent/uidmap.go diff --git a/src/pkg/agent/uidmap_internal_test.go b/src/crux/agent/uidmap_internal_test.go similarity index 100% rename from src/pkg/agent/uidmap_internal_test.go rename to src/crux/agent/uidmap_internal_test.go diff --git a/src/pkg/agent/uidmap_persist_test.go b/src/crux/agent/uidmap_persist_test.go similarity index 100% rename from src/pkg/agent/uidmap_persist_test.go rename to src/crux/agent/uidmap_persist_test.go diff --git a/src/pkg/agent/uidmap_stable_test.go b/src/crux/agent/uidmap_stable_test.go similarity index 97% rename from src/pkg/agent/uidmap_stable_test.go rename to src/crux/agent/uidmap_stable_test.go index 0646e65792..db7c6100a6 100644 --- a/src/pkg/agent/uidmap_stable_test.go +++ b/src/crux/agent/uidmap_stable_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestUIDMapAllocateUIDsAppendsNewNamesAfterMaxExistingUID(t *testing.T) { diff --git a/src/pkg/agent/uidmap_test.go b/src/crux/agent/uidmap_test.go similarity index 100% rename from src/pkg/agent/uidmap_test.go rename to src/crux/agent/uidmap_test.go diff --git a/src/pkg/agent/watchdog_fleet.go b/src/crux/agent/watchdog_fleet.go similarity index 97% rename from src/pkg/agent/watchdog_fleet.go rename to src/crux/agent/watchdog_fleet.go index c8f2781b69..728ad21d62 100644 --- a/src/pkg/agent/watchdog_fleet.go +++ b/src/crux/agent/watchdog_fleet.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/watchdog" + "github.com/hivecommons/hive/crux/watchdog" ) // WatchdogFleet implements watchdog.Fleet over a Manager. @@ -427,3 +427,11 @@ func newestMtimeBounded(root string, maxDepth int) (time.Time, bool) { scan(root, 0) return newest, !newest.IsZero() } + +// WatchdogSource is the runtime-owned half of the watchdog fleet +// (watchdog.Source) over this manager, with no governor hooks. `crux serve` +// exposes it over the API; crux/runtime.NewWatchdogFleet composes it with +// the hooks for a sidecar (hivecommons/hive#14138, stage 3). +func (m *Manager) WatchdogSource() watchdog.Source { + return WatchdogFleet{M: m} +} diff --git a/src/pkg/agent/watchdog_fleet_test.go b/src/crux/agent/watchdog_fleet_test.go similarity index 99% rename from src/pkg/agent/watchdog_fleet_test.go rename to src/crux/agent/watchdog_fleet_test.go index 69a3ba07bd..ad04901918 100644 --- a/src/pkg/agent/watchdog_fleet_test.go +++ b/src/crux/agent/watchdog_fleet_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/watchdog" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/watchdog" ) // newWatchdogTestManager builds a Manager with one agent per backend, marked diff --git a/src/pkg/agent/watsonx_backend_test.go b/src/crux/agent/watsonx_backend_test.go similarity index 99% rename from src/pkg/agent/watsonx_backend_test.go rename to src/crux/agent/watsonx_backend_test.go index 86f46efac9..a8b2db96c9 100644 --- a/src/pkg/agent/watsonx_backend_test.go +++ b/src/crux/agent/watsonx_backend_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // The bug this file locks down: diff --git a/src/pkg/agent/write_as_user_test.go b/src/crux/agent/write_as_user_test.go similarity index 100% rename from src/pkg/agent/write_as_user_test.go rename to src/crux/agent/write_as_user_test.go diff --git a/src/crux/agentmode.go b/src/crux/agentmode.go new file mode 100644 index 0000000000..f0dac8ed0e --- /dev/null +++ b/src/crux/agentmode.go @@ -0,0 +1,73 @@ +package crux + +import "github.com/hivecommons/hive/crux/agentmode" + +// AgentMode is the GitHub interaction tier an agent holds at an ACMM level. +// It is defined in the leaf crux/agentmode; consumers name it here so they do +// not import crux/agent for mode and capability parsing. +type AgentMode = agentmode.AgentMode + +const ( + ModeAdvisory = agentmode.ModeAdvisory + ModeIssuesOnly = agentmode.ModeIssuesOnly + ModeIssuesAndPRs = agentmode.ModeIssuesAndPRs + ModeIssuesPRsMerge = agentmode.ModeIssuesPRsMerge +) + +// ParseAgentMode converts a configured mode string such as "ADVISORY" to an +// AgentMode. Dashboard: agent cards and Linear agent config. +func ParseAgentMode(s string) (AgentMode, bool) { + return agentmode.ParseAgentMode(s) +} + +// DefaultAgentMode returns the mode an agent gets at an ACMM level with no +// hive.yaml override. Dashboard: the "(default)" indicators. +func DefaultAgentMode(name string, level int) AgentMode { + if name == "supervisor" { + return ModeAdvisory + } + switch level { + case 1: + return ModeAdvisory + case 2: + return ModeAdvisory + case 3: + if name == "quality" { + return ModeIssuesAndPRs + } + return ModeAdvisory + case 4: + switch name { + case "quality", "sec-check", "ci-maintainer": + return ModeIssuesAndPRs + case "scanner", "guide": + return ModeIssuesOnly + default: + return ModeAdvisory + } + case 5: + return ModeIssuesAndPRs + case 6: + if name == "scanner" { + return ModeIssuesPRsMerge + } + return ModeIssuesAndPRs + default: + return ModeAdvisory + } +} + +// AgentAvailableAtACMMLevel is the ACMM roster gate for level-scoped agents. +// Dashboard: status and pack rosters; hub: agent capability rows. +func AgentAvailableAtACMMLevel(name string, level int) bool { + switch name { + case "telemetry", "operations": + return level >= operabilityAgentMinACMMLevel + default: + return true + } +} + +// operabilityAgentMinACMMLevel is the ACMM level the operability agents +// (telemetry, operations) exist from. +const operabilityAgentMinACMMLevel = 5 diff --git a/src/pkg/agentmode/agentmode.go b/src/crux/agentmode/agentmode.go similarity index 100% rename from src/pkg/agentmode/agentmode.go rename to src/crux/agentmode/agentmode.go diff --git a/src/pkg/agentmode/agentmode_test.go b/src/crux/agentmode/agentmode_test.go similarity index 100% rename from src/pkg/agentmode/agentmode_test.go rename to src/crux/agentmode/agentmode_test.go diff --git a/src/pkg/agentmode/claim_comment_test.go b/src/crux/agentmode/claim_comment_test.go similarity index 100% rename from src/pkg/agentmode/claim_comment_test.go rename to src/crux/agentmode/claim_comment_test.go diff --git a/src/pkg/agentmode/reviewer_tier_test.go b/src/crux/agentmode/reviewer_tier_test.go similarity index 100% rename from src/pkg/agentmode/reviewer_tier_test.go rename to src/crux/agentmode/reviewer_tier_test.go diff --git a/src/crux/agentmode_test.go b/src/crux/agentmode_test.go new file mode 100644 index 0000000000..a7b60700c9 --- /dev/null +++ b/src/crux/agentmode_test.go @@ -0,0 +1,68 @@ +package crux_test + +import ( + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/agentmode" +) + +func TestParseAgentMode_MatchesLeaf(t *testing.T) { + for _, in := range []string{"ADVISORY", "issues-only", "ISSUES_AND_PRS", "issues+prs+merge", "", "bogus"} { + got, gotOK := crux.ParseAgentMode(in) + want, wantOK := agentmode.ParseAgentMode(in) + if got != want || gotOK != wantOK { + t.Errorf("crux.ParseAgentMode(%q) = (%v, %v), want leaf result (%v, %v)", in, got, gotOK, want, wantOK) + } + } + if m, ok := crux.ParseAgentMode("ADVISORY"); !ok || m != crux.ModeAdvisory { + t.Fatalf("crux.ParseAgentMode(ADVISORY) = (%v, %v), want (crux.ModeAdvisory, true)", m, ok) + } +} + +func TestModeConstants_AliasLeaf(t *testing.T) { + cases := map[crux.AgentMode]agentmode.AgentMode{ + crux.ModeAdvisory: agentmode.ModeAdvisory, + crux.ModeIssuesOnly: agentmode.ModeIssuesOnly, + crux.ModeIssuesAndPRs: agentmode.ModeIssuesAndPRs, + crux.ModeIssuesPRsMerge: agentmode.ModeIssuesPRsMerge, + } + for got, want := range cases { + if got != want { + t.Errorf("crux mode %v != leaf mode %v", got, want) + } + } +} + +func TestDefaultAgentMode_MatchesAgentPackage(t *testing.T) { + for _, name := range []string{"scanner", "quality", "supervisor", "guide", "governor"} { + for level := 0; level <= 7; level++ { + if got, want := crux.DefaultAgentMode(name, level), agent.DefaultAgentMode(name, level); got != want { + t.Errorf("crux.DefaultAgentMode(%q, %d) = %v, want %v", name, level, got, want) + } + } + } + if got := crux.DefaultAgentMode("scanner", 6); got != crux.ModeIssuesPRsMerge { + t.Fatalf("crux.DefaultAgentMode(scanner, 6) = %v, want crux.ModeIssuesPRsMerge", got) + } + if got := crux.DefaultAgentMode("supervisor", 6); got != crux.ModeAdvisory { + t.Fatalf("crux.DefaultAgentMode(supervisor, 6) = %v, want crux.ModeAdvisory", got) + } +} + +func TestAgentAvailableAtACMMLevel_MatchesAgentPackage(t *testing.T) { + for _, name := range []string{"telemetry", "operations", "scanner", "quality"} { + for level := 0; level <= 7; level++ { + if got, want := crux.AgentAvailableAtACMMLevel(name, level), agent.AgentAvailableAtACMMLevel(name, level); got != want { + t.Errorf("crux.AgentAvailableAtACMMLevel(%q, %d) = %v, want %v", name, level, got, want) + } + } + } + if crux.AgentAvailableAtACMMLevel("telemetry", 4) { + t.Fatal("telemetry should be gated below ACMM level 5") + } + if !crux.AgentAvailableAtACMMLevel("telemetry", 5) || !crux.AgentAvailableAtACMMLevel("scanner", 1) { + t.Fatal("telemetry at level 5 and scanner at any level should be available") + } +} diff --git a/src/pkg/agentparse/agentparse.go b/src/crux/agentparse/agentparse.go similarity index 100% rename from src/pkg/agentparse/agentparse.go rename to src/crux/agentparse/agentparse.go diff --git a/src/pkg/agentparse/agentparse_test.go b/src/crux/agentparse/agentparse_test.go similarity index 100% rename from src/pkg/agentparse/agentparse_test.go rename to src/crux/agentparse/agentparse_test.go diff --git a/src/pkg/agentsmd/agentsmd.go b/src/crux/agentsmd/agentsmd.go similarity index 100% rename from src/pkg/agentsmd/agentsmd.go rename to src/crux/agentsmd/agentsmd.go diff --git a/src/pkg/agentsmd/agentsmd_test.go b/src/crux/agentsmd/agentsmd_test.go similarity index 100% rename from src/pkg/agentsmd/agentsmd_test.go rename to src/crux/agentsmd/agentsmd_test.go diff --git a/src/pkg/agentsmd/injection_cap_test.go b/src/crux/agentsmd/injection_cap_test.go similarity index 100% rename from src/pkg/agentsmd/injection_cap_test.go rename to src/crux/agentsmd/injection_cap_test.go diff --git a/src/crux/api/api.go b/src/crux/api/api.go new file mode 100644 index 0000000000..d18bcc8437 --- /dev/null +++ b/src/crux/api/api.go @@ -0,0 +1,260 @@ +// Package api is the wire contract of `crux serve` (Crux Stage 3, +// hivecommons/hive#13536; src/docs/design/crux.md "Stage 3"): the request +// and response types, the routes, the event vocabulary and the address forms +// a client and server agree on. It imports only the standard library, so a +// harness that drives the runtime over the API depends on this package and +// nothing of the runtime itself. +// +// The API is HTTP+JSON under /v1. The server listens on a unix socket by +// default and optionally on TCP; TCP always requires a bearer token +// (Authorization: Bearer ), the socket requires one only when the +// server was given one. Streams (log follow, runtime events) are +// text/event-stream, one JSON object per `data:` line, `event:` naming the +// type. +// +// The CLI-shaped endpoints mirror one `crux` CLI verb each and the +// crux.Runtime methods that verb maps onto: +// +// GET /v1/health server is up; agent count +// GET /v1/agents ps Observer.AllStatuses +// POST /v1/agents run Definitions.AddAgent + Lifecycle.Start +// GET /v1/agents/{name} get Observer.GetStatus +// GET /v1/agents/{name}/logs logs Observer.GetOutput / GetBufferOutput / CaptureFullLog +// GET /v1/agents/{name}/logs?follow=1 logs -f Observer.GetBufferOutput, streamed as SSE +// POST /v1/agents/{name}/kick kick Kicker.SendKick +// POST /v1/agents/{name}/stop stop Lifecycle.Stop +// GET /v1/events — runtime events as SSE +// +// {name} is a name or ID; the server resolves it with Observer.ResolveAgent. +// +// The rest of crux.Runtime (hivecommons/hive#14138, stage 3) is in +// runtime.go: one route per family or read, and RuntimeRoutes, the table that +// maps every Runtime method to its route. +package api + +import ( + "encoding/json" + "time" +) + +// Version is the API version every route is prefixed with. +const Version = "v1" + +// Routes. Agent routes take the agent's name or ID in place of {name}; use +// AgentPath and friends to build them. +const ( + PathHealth = "/" + Version + "/health" + PathAgents = "/" + Version + "/agents" + PathEvents = "/" + Version + "/events" +) + +// AgentPath is the route of one agent. +func AgentPath(name string) string { return PathAgents + "/" + name } + +// LogsPath is the route of one agent's logs. +func LogsPath(name string) string { return AgentPath(name) + "/logs" } + +// KickPath is the route that kicks one agent. +func KickPath(name string) string { return AgentPath(name) + "/kick" } + +// StopPath is the route that stops one agent. +func StopPath(name string) string { return AgentPath(name) + "/stop" } + +// Log sources, the `source` query parameter of the logs route. They are the +// three readers `crux logs` has: the live pane (default), the ring buffer +// (--buffer) and the whole scrollback (--full). +const ( + LogSourcePane = "pane" + LogSourceBuffer = "buffer" + LogSourceFull = "full" +) + +// Query parameters of the logs route. +const ( + QuerySource = "source" + QueryLines = "n" + QueryFollow = "follow" +) + +// DefaultLogLines is how many lines the logs route returns when `n` is +// absent; it matches `crux logs`. +const DefaultLogLines = 100 + +// Media types. +const ( + ContentTypeJSON = "application/json" + ContentTypeEvent = "text/event-stream" +) + +// Health is the body of GET /v1/health. +type Health struct { + // Version is the API version the server speaks. + Version string `json:"version"` + // Agents is how many definitions the runtime holds. + Agents int `json:"agents"` + // Background lists the crux.Background loops the server runs in-process + // (the Background method names); empty when it runs none. + Background []string `json:"background,omitempty"` + // HiveCallback is the hive callback endpoint's reachability when the + // server was started with one (`crux serve --hive-callback`); absent + // otherwise. The same state is the hive_callback_unavailable condition + // on the event stream. + HiveCallback *HiveCallback `json:"hive_callback,omitempty"` + // StartedAt is when the server started and UptimeSeconds how long ago + // that was at the time of the answer (#14138 stage 4): a supervisor + // that restarted the server sees the uptime reset. + StartedAt time.Time `json:"started_at"` + UptimeSeconds float64 `json:"uptime_seconds"` +} + +// HiveCallback is the wire form of the callback client's status. +type HiveCallback struct { + Address string `json:"address"` + Reachable bool `json:"reachable"` + // LastError is the most recent failure, "" when reachable. + LastError string `json:"last_error,omitempty"` + // Since is when the current outage began; absent when reachable. + Since *time.Time `json:"since,omitempty"` + // FailedCalls counts calls that failed closed during the current outage. + FailedCalls int `json:"failed_calls,omitempty"` + // LastSuccess is when a call last reached hive; absent if never. + LastSuccess *time.Time `json:"last_success,omitempty"` +} + +// Process is the wire projection of one agent: what `crux ps --json` prints, +// plus the fields a harness needs to decide what to do next (kick refusals, +// pause provenance, the busy condition). It is a subset of crux.Process by +// design: the full snapshot carries engine-internal state that is neither +// stable nor meaningful off-process. +type Process struct { + Name string `json:"name"` + ID string `json:"id"` + State string `json:"state"` + Backend string `json:"backend"` + Model string `json:"model,omitempty"` + PID int `json:"pid,omitempty"` + + Paused bool `json:"paused"` + PausedReason string `json:"paused_reason,omitempty"` + PausedTrigger string `json:"paused_trigger,omitempty"` + PausedBy string `json:"paused_by,omitempty"` + + StartedAt *time.Time `json:"started_at,omitempty"` + LastKick *time.Time `json:"last_kick,omitempty"` + RestartCount int `json:"restart_count"` + + LastRestartReason string `json:"last_restart_reason,omitempty"` + KickRefused bool `json:"kick_refused,omitempty"` + KickRefusalReason string `json:"kick_refusal_reason,omitempty"` + KicksUndeliverable int `json:"kicks_undeliverable,omitempty"` + BusyCondition string `json:"busy_condition,omitempty"` + StartFailureClass string `json:"start_failure_class,omitempty"` +} + +// ProcessList is the body of GET /v1/agents, sorted by name. +type ProcessList struct { + Processes []Process `json:"processes"` +} + +// RunRequest is the body of POST /v1/agents. Name is required. Definition, +// when present, is an agent definition document in the runtime's own +// configuration shape (the JSON form of crux/config.AgentConfig, which is +// the per-agent overlay hive reads from agents_dir) and is registered with +// Definitions.AddAgent before Lifecycle.Start; Name must then match +// ^[a-z0-9][a-z0-9_-]*$ (at most 64 characters) and not already be defined +// on the server (409 CodeConflict otherwise: the runtime does not replace a +// definition). The server fills the defaults a start depends on (id and +// role from Name, replicas 1, enabled unless the document says otherwise); +// everything else is the caller's. Without a definition, Name (or an agent +// ID) must already be defined on the server. Parsing the portable +// AgentDefinition format is the caller's job, as it is the host's job for +// the CLI. +type RunRequest struct { + Name string `json:"name"` + Definition json.RawMessage `json:"definition,omitempty"` +} + +// KickRequest is the body of POST /v1/agents/{name}/kick. Async selects +// Kicker.SendKickAsync; the response's Started then says whether delivery +// began, and GET /v1/agents/{name}/kick reports the dispatch. +type KickRequest struct { + Message string `json:"message"` + Async bool `json:"async,omitempty"` + // Source records the kick's origin on its kick record + // (Kicker.SendKickWithSource); empty means unattributed. + Source string `json:"source,omitempty"` +} + +// AgentResponse is the body of a successful run, kick, stop or other +// per-agent action: the resolved agent name and its snapshot after the +// action. Started is set only for an async kick. +type AgentResponse struct { + Agent string `json:"agent"` + Process *Process `json:"process,omitempty"` + Started *bool `json:"started,omitempty"` +} + +// Logs is the body of GET /v1/agents/{name}/logs without follow. Lines is +// set for the pane and buffer sources; Header and Content for full. +type Logs struct { + Agent string `json:"agent"` + Source string `json:"source"` + Header string `json:"header,omitempty"` + Lines []string `json:"lines,omitempty"` + Content string `json:"content,omitempty"` +} + +// LogChunk is one `data:` record of a followed log stream: the lines +// appended to the ring buffer since the previous chunk. +type LogChunk struct { + Agent string `json:"agent"` + Lines []string `json:"lines"` +} + +// Event is one `data:` record of GET /v1/events. Type is one of the Event* +// constants; Agent is empty for runtime-wide events. State carries the new +// state for EventState; Detail is free text for the kick relay. Data is the +// typed payload of the observer events in runtime.go (KickOutcome, +// PauseTransition, ...), absent for the lifecycle events. +type Event struct { + Time time.Time `json:"time"` + Type string `json:"type"` + Agent string `json:"agent,omitempty"` + State string `json:"state,omitempty"` + Detail string `json:"detail,omitempty"` + Data json.RawMessage `json:"data,omitempty"` +} + +// Event types. The lifecycle ones are emitted by the server for actions it +// performed; EventState is emitted when the server observes an agent's state +// change for any reason (crash, restart, pause); EventKick relays the +// runtime's kick observer (Kicker.SetKickObserver) with the observer's event +// name in Detail. +const ( + EventStarted = "agent.started" + EventStopped = "agent.stopped" + EventKicked = "agent.kicked" + EventState = "agent.state" + EventKick = "agent.kick" + // EventLog is the type of every record on a followed log stream. + EventLog = "agent.log" +) + +// Error is the body of every non-2xx response. +type Error struct { + // Error is the message. + Error string `json:"error"` + // Code is a stable machine-readable kind; see the Code* constants. + Code string `json:"code,omitempty"` +} + +// Error codes. +const ( + CodeUnauthorized = "unauthorized" + CodeBadRequest = "bad_request" + CodeNotFound = "not_found" + // CodeConflict is a run with a definition for a name the server already + // holds; the runtime does not replace definitions. + CodeConflict = "conflict" + CodeRuntime = "runtime_error" +) diff --git a/src/crux/api/api_test.go b/src/crux/api/api_test.go new file mode 100644 index 0000000000..a0f24a3eb0 --- /dev/null +++ b/src/crux/api/api_test.go @@ -0,0 +1,131 @@ +package api + +import ( + "bytes" + "errors" + "strings" + "testing" +) + +func TestPaths(t *testing.T) { + if AgentPath("fixer") != "/v1/agents/fixer" || LogsPath("fixer") != "/v1/agents/fixer/logs" || + KickPath("fixer") != "/v1/agents/fixer/kick" || StopPath("fixer") != "/v1/agents/fixer/stop" { + t.Fatalf("paths: %s %s %s %s", AgentPath("fixer"), LogsPath("fixer"), KickPath("fixer"), StopPath("fixer")) + } +} + +func TestParseAddress(t *testing.T) { + cases := []struct { + in string + want Address + }{ + {"unix:///run/crux.sock", Address{Network: "unix", Path: "/run/crux.sock", BaseURL: unixBaseURL}}, + {"/run/crux.sock", Address{Network: "unix", Path: "/run/crux.sock", BaseURL: unixBaseURL}}, + {"./crux.sock", Address{Network: "unix", Path: "crux.sock", BaseURL: unixBaseURL}}, + {"../x/crux.sock", Address{Network: "unix", Path: "../x/crux.sock", BaseURL: unixBaseURL}}, + {" http://127.0.0.1:7777/ ", Address{Network: "tcp", Host: "127.0.0.1:7777", BaseURL: "http://127.0.0.1:7777"}}, + {"https://crux.example:7777", Address{Network: "tcp", Host: "crux.example:7777", BaseURL: "https://crux.example:7777"}}, + {"127.0.0.1:7777", Address{Network: "tcp", Host: "127.0.0.1:7777", BaseURL: "http://127.0.0.1:7777"}}, + {"[::1]:7777", Address{Network: "tcp", Host: "[::1]:7777", BaseURL: "http://[::1]:7777"}}, + } + for _, c := range cases { + got, err := ParseAddress(c.in) + if err != nil { + t.Errorf("%q: %v", c.in, err) + continue + } + if got != c.want { + t.Errorf("%q = %+v, want %+v", c.in, got, c.want) + } + } + + bad := []string{"", " ", "unix://", "http://", "ftp://x:1", "localhost", "host:", "host:abc", "[::1:80", "http://%zz"} + for _, in := range bad { + if _, err := ParseAddress(in); err == nil { + t.Errorf("%q should not parse", in) + } + } + + // Plain http carries the bearer token in the clear, so it is loopback-only; + // the same hosts are fine over https. + cleartext := []string{"http://crux.example:7777", "http://10.0.0.5:7777", "http://0.0.0.0:7777", "http://[::]:7777", "crux.example:7777", "10.0.0.5:7777", "0.0.0.0:7777", ":7777"} + for _, in := range cleartext { + if _, err := ParseAddress(in); err == nil || !strings.Contains(err.Error(), "loopback") { + t.Errorf("%q must be refused as cleartext to a non-loopback host, got %v", in, err) + } + } + for _, in := range []string{"https://10.0.0.5:7777", "https://0.0.0.0:7777", "http://localhost:7777", "http://[::ffff:127.0.0.1]:7777", "localhost:7777", "127.1.2.3:7777"} { + if _, err := ParseAddress(in); err != nil { + t.Errorf("%q: %v", in, err) + } + } +} + +func TestLoopbackHost(t *testing.T) { + yes := []string{"127.0.0.1:1", "127.255.0.9:80", "[::1]:1", "::1", "localhost", "LOCALHOST:7777", "[::ffff:127.0.0.1]:1", "[::1%lo]:1", "127.0.0.1"} + for _, h := range yes { + if !LoopbackHost(h) { + t.Errorf("LoopbackHost(%q) = false, want true", h) + } + } + no := []string{"", ":7777", "0.0.0.0:7777", "[::]:7777", "10.0.0.5:7777", "192.168.1.1", "crux.example:7777", "localhost.example:1", "[::ffff:10.0.0.5]:1", "127.0.0.1.nip.io:1"} + for _, h := range no { + if LoopbackHost(h) { + t.Errorf("LoopbackHost(%q) = true, want false", h) + } + } +} + +func TestWriteAndReadEvents(t *testing.T) { + var buf bytes.Buffer + if err := WriteEvent(&buf, EventStarted, Event{Type: EventStarted, Agent: "fixer"}); err != nil { + t.Fatal(err) + } + buf.WriteString(": keep-alive\n\n") + buf.WriteString("id: 7\nevent: " + EventLog + "\ndata: {\"agent\":\"fixer\",\n") + buf.WriteString("data:\"lines\":[\"a\"]}\n\n") + buf.WriteString("event: trailing\ndata: {\"type\":\"trailing\"}") // no final blank line + + var got []string + err := ReadEvents(&buf, func(typ string, data []byte) error { + got = append(got, typ+"|"+string(data)) + return nil + }) + if err != nil { + t.Fatal(err) + } + want := []string{ + EventStarted + `|{"time":"0001-01-01T00:00:00Z","type":"agent.started","agent":"fixer"}`, + EventLog + "|{\"agent\":\"fixer\",\n\"lines\":[\"a\"]}", + `trailing|{"type":"trailing"}`, + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("events:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +func TestReadEventsStopsOnCallbackError(t *testing.T) { + stop := errors.New("stop") + in := strings.NewReader("event: a\ndata: 1\n\nevent: b\ndata: 2\n\n") + n := 0 + err := ReadEvents(in, func(string, []byte) error { n++; return stop }) + if !errors.Is(err, stop) || n != 1 { + t.Fatalf("err %v after %d records", err, n) + } +} + +type failingReader struct{} + +func (failingReader) Read([]byte) (int, error) { return 0, errors.New("boom") } + +func TestReadEventsReportsReadErrors(t *testing.T) { + if err := ReadEvents(failingReader{}, func(string, []byte) error { return nil }); err == nil || err.Error() != "boom" { + t.Fatalf("err = %v", err) + } +} + +func TestWriteEventRejectsUnmarshalable(t *testing.T) { + if err := WriteEvent(&bytes.Buffer{}, "x", make(chan int)); err == nil { + t.Fatal("expected a marshal error") + } +} diff --git a/src/crux/api/callback/callback.go b/src/crux/api/callback/callback.go new file mode 100644 index 0000000000..5a24974f72 --- /dev/null +++ b/src/crux/api/callback/callback.go @@ -0,0 +1,625 @@ +// Package callback is the wire contract of the hive-side callback endpoint +// the Crux runtime calls for hive-provided capabilities (hivecommons/hive#14138 +// stage 2; src/docs/design/crux.md "Credentials and hive-provided +// capabilities"). It imports only the standard library and crux/api (itself +// stdlib-only), so a hive that mounts it links nothing of the runtime and a +// runtime that calls it links nothing of hive. +// +// The direction is runtime → hive. The runtime never holds the GitHub App +// private key, the canary registry or any other hive-owned credential; it +// asks hive for a short-lived scoped token, a scan verdict, a created PR, and +// reports accounting and conditions back. Capabilities is the one interface +// set both ends implement: *Client over HTTP, hive's pkg/cruxhost directly. +// Every method fails closed: an unreachable or refusing hive is an error the +// caller treats as a refusal, never a less-safe default. +// +// Transport follows crux/api: HTTP+JSON under /v1, api.Error bodies with +// stable codes, a 0600 unix socket in a uid-owned 0700 directory, optional +// TCP only behind a bearer token, and a token-less server that answers only +// requests it saw arrive on its own socket. +// +// GET /v1/health server is up; which capabilities exist +// POST /v1/tokens mint a scoped GitHub token +// POST /v1/github/rest record one relayed GitHub REST response +// GET /v1/github/rate-limit shared core-bucket snapshot +// POST /v1/usage record inference token usage +// POST /v1/scan/canary scan a GitHub write body for canaries +// POST /v1/scan/text scan tool payload text +// POST /v1/conditions observe or clear a runtime condition +// GET /v1/linear/credential the Linear credential to inject +// POST /v1/pulls open a pull request as the hive's App +// GET /v1/policy the host policy snapshot (stage 4) +// POST /v1/policy/repo-auto-merge whether auto-merge is enabled for a repo +// PUT /v1/inference/routes/{agent} route an agent's inference traffic (stage 4) +// DELETE /v1/inference/routes/{agent} clear that route +// POST /v1/effects begin a journaled external mutation (stage 4) +// POST /v1/effects/{id}/end report how it ended +// POST /v1/tokens/mint issue an agent's short-lived mint credential (stage 4) +// GET /v1/bob/credential the Bob API key and where it came from (stage 4) +package callback + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "time" +) + +// Version is the API version every route is prefixed with. +const Version = "v1" + +// Routes. +const ( + PathHealth = "/" + Version + "/health" + PathTokens = "/" + Version + "/tokens" + PathGitHubREST = "/" + Version + "/github/rest" + PathGitHubRateLimit = "/" + Version + "/github/rate-limit" + PathUsage = "/" + Version + "/usage" + PathScanCanary = "/" + Version + "/scan/canary" + PathScanText = "/" + Version + "/scan/text" + PathConditions = "/" + Version + "/conditions" + PathLinearCredential = "/" + Version + "/linear/credential" + PathPulls = "/" + Version + "/pulls" + PathPolicy = "/" + Version + "/policy" + PathRepoAutoMerge = "/" + Version + "/policy/repo-auto-merge" + PathInferenceRoutes = "/" + Version + "/inference/routes" + PathEffects = "/" + Version + "/effects" + PathAgentMint = "/" + Version + "/tokens/mint" + PathBobCredential = "/" + Version + "/bob/credential" +) + +// InferenceRoutePath is the route for one agent's inference route. +func InferenceRoutePath(agent string) string { + return PathInferenceRoutes + "/" + url.PathEscape(agent) +} + +// EffectEndPath is the route that ends the effect begun as id. +func EffectEndPath(id string) string { + return PathEffects + "/" + url.PathEscape(id) + "/end" +} + +// Environment variables. +const ( + // SocketEnvVar overrides the default unix socket path hive serves the + // callback endpoint on and the runtime dials. + SocketEnvVar = "CRUX_HIVE_SOCKET" + // TokenEnvVar is the bearer token hive requires and the runtime presents. + // Optional on the socket (the file mode is the credential), required + // for TCP. + TokenEnvVar = "CRUX_HIVE_TOKEN" + // ListenEnvVar is an optional loopback TCP address hive also serves on + // (the k8s sidecar-container option, where the containers share the + // pod's network namespace). It always requires TokenEnvVar and is + // refused off loopback: the listener is plain http. + ListenEnvVar = "CRUX_HIVE_LISTEN" + // SocketName is the socket file name; it sits next to the runtime's + // crux.sock in the same private directory. + SocketName = "hive.sock" + // socketDirName mirrors crux/config.ServeSocketDirName. + socketDirName = "crux" +) + +// File modes, the same as the runtime socket's. +const ( + SocketMode os.FileMode = 0o600 + SocketDirMode os.FileMode = 0o700 +) + +// Media type. +const ContentTypeJSON = "application/json" + +// Error codes carried in api.Error bodies. +const ( + CodeUnauthorized = "unauthorized" + CodeBadRequest = "bad_request" + // CodeNotConfigured is a route for a capability this hive does not have + // (no GitHub App, no canary registry, no Linear). The runtime treats it + // exactly as the nil seam it stands in for. + CodeNotConfigured = "not_configured" + // CodeUpstream is a configured capability whose own upstream (GitHub, + // Linear) refused; the message says why. + CodeUpstream = "upstream_error" + // CodeHive is an unexpected hive-side failure. + CodeHive = "hive_error" + // CodeEffectDenied is POST /v1/effects refused by hive's mutation + // boundary (the fence or the journal said no); the runtime maps it to + // effects.ErrDenied. + CodeEffectDenied = "effect_denied" + // CodeEffectUnknown is an end for an id hive is not holding: it was + // never begun, or already ended, or hive restarted in between. + CodeEffectUnknown = "effect_unknown" +) + +// ConditionClassUnavailable is the runtime condition class reserved for +// "the hive callback endpoint is unreachable"; Client.Status carries it +// locally and stage 4 surfaces it on the runtime's health route. +const ConditionClassUnavailable = "hive_callback_unavailable" + +// DefaultTimeout bounds one callback when the caller's context has no +// deadline of its own. Fail-closed answers are cheap, so a dead socket +// costs at most this per call. +const DefaultTimeout = 5 * time.Second + +// Sentinel errors a caller can test with errors.Is. Both are refusals. +var ( + // ErrUnavailable wraps a transport failure, timeout or 5xx: hive could + // not answer. + ErrUnavailable = errors.New("hive callback endpoint unavailable") + // ErrUnauthorized wraps a 401: hive rejected the runtime's credential. + // It is never retried with another credential. + ErrUnauthorized = errors.New("hive callback endpoint rejected the bearer token") + // ErrNotConfigured wraps a 404 CodeNotConfigured. + ErrNotConfigured = errors.New("capability not configured on this hive") + // ErrEffectDenied wraps a 403 CodeEffectDenied. + ErrEffectDenied = errors.New("hive mutation boundary denied the effect") + // ErrEffectUnknown wraps a 404 CodeEffectUnknown. + ErrEffectUnknown = errors.New("hive is not holding that effect") +) + +// DefaultSocket is the socket path hive and the runtime agree on when +// neither is told otherwise: $CRUX_HIVE_SOCKET if set, else +// $XDG_RUNTIME_DIR/crux/hive.sock, else /crux-/hive.sock. The +// rule is crux/config.DefaultServeSocket's with a different file name so the +// two sockets share one private directory. +func DefaultSocket() string { + if p := os.Getenv(SocketEnvVar); p != "" { + return p + } + return filepath.Join(DefaultSocketDir(), SocketName) +} + +// DefaultSocketDir is the directory DefaultSocket places the socket in. +func DefaultSocketDir() string { + if dir := os.Getenv("XDG_RUNTIME_DIR"); dir != "" { + return filepath.Join(dir, socketDirName) + } + return filepath.Join(os.TempDir(), fmt.Sprintf("%s-%d", socketDirName, os.Getuid())) +} + +// Health is the body of GET /v1/health. +type Health struct { + Version string `json:"version"` + // Capabilities names the capabilities this hive has configured, as the + // Capability* constants. + Capabilities []string `json:"capabilities"` +} + +// Capability names, as Health reports them. +const ( + CapabilityTokens = "tokens" + CapabilityGitHubREST = "github_rest" + CapabilityUsage = "usage" + CapabilityCanary = "canary" + CapabilityText = "text" + CapabilityConditions = "conditions" + CapabilityLinear = "linear" + CapabilityPulls = "pulls" + // CapabilityPolicy covers both policy routes. It is served only by a + // Capabilities that also implements PolicySource. + CapabilityPolicy = "policy" + // CapabilityInferenceRoutes covers the inference route routes; served + // only by a Capabilities that also implements InferenceRouter. + CapabilityInferenceRoutes = "inference_routes" + // CapabilityEffects covers the effect routes; served only by a + // Capabilities that also implements EffectBoundary. + CapabilityEffects = "effects" + // CapabilityAgentMint covers POST /v1/tokens/mint; served only by a + // Capabilities that also implements AgentMinter (hive's opt-in mint). + CapabilityAgentMint = "agent_mint" + // CapabilityBob covers GET /v1/bob/credential; served only by a + // Capabilities that also implements BobCredentials. + CapabilityBob = "bob" +) + +// MintTokenRequest is the body of POST /v1/tokens. Tier is the permission +// tier (pushbroker.DefaultTier when empty). Repo is "owner/name" and Repos +// bare repository names; the server narrows the token to the union of the +// two (installation-wide when both are empty, the existing +// ScopedTokenForRepos contract). TTLSeconds is a request; the response says +// what was granted. +type MintTokenRequest struct { + Agent string `json:"agent,omitempty"` + Repo string `json:"repo,omitempty"` + Repos []string `json:"repos,omitempty"` + Tier string `json:"tier,omitempty"` + TTLSeconds int `json:"ttl_seconds,omitempty"` +} + +// MintTokenResponse is the body of a successful POST /v1/tokens. +type MintTokenResponse struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` +} + +// String prints the response with the token redacted so an accidental %v in +// a log line cannot leak it. +func (r MintTokenResponse) String() string { + return fmt.Sprintf("callback.MintTokenResponse{Token: , ExpiresAt: %s}", r.ExpiresAt.Format(time.RFC3339)) +} + +// GoString is String, for %#v. +func (r MintTokenResponse) GoString() string { return r.String() } + +// RESTRecord is the body of POST /v1/github/rest: one relayed GitHub REST +// response. Injected says the daemon's credential was used (its rate-limit +// headers describe the shared core bucket) rather than the agent's own. +// RateLimit carries only the X-RateLimit-* response headers. +type RESTRecord struct { + Caller string `json:"caller"` + Method string `json:"method"` + Path string `json:"path"` + Status int `json:"status"` + Injected bool `json:"injected"` + RateLimit map[string]string `json:"rate_limit,omitempty"` +} + +// RateLimitHeaderPrefix selects the response headers RESTRecord carries. +const RateLimitHeaderPrefix = "X-Ratelimit-" + +// RateLimitSnapshot is the body of GET /v1/github/rate-limit. OK is false +// when hive has no snapshot yet. +type RateLimitSnapshot struct { + Remaining int `json:"remaining"` + Reset time.Time `json:"reset"` + OK bool `json:"ok"` +} + +// Usage kinds, the three UsageSink record paths. +const ( + UsageKindGateway = "gateway" + UsageKindCopilot = "copilot" + UsageKindCopilotCache = "copilot_cache" +) + +// UsageRecord is the body of POST /v1/usage. +type UsageRecord struct { + Agent string `json:"agent"` + Model string `json:"model"` + Kind string `json:"kind"` + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + CacheRead int64 `json:"cache_read_tokens,omitempty"` + CacheCreation int64 `json:"cache_creation_tokens,omitempty"` +} + +// CanaryScanRequest is the body of POST /v1/scan/canary. +type CanaryScanRequest struct { + Agent string `json:"agent"` + Text string `json:"text"` + Source string `json:"source"` +} + +// CanaryScanResponse is the body of a successful canary scan. It never +// carries the matched canary token: hive knows which one it planted, and +// the runtime only needs to refuse. +type CanaryScanResponse struct { + Detected bool `json:"detected"` + Agent string `json:"agent,omitempty"` + Source string `json:"source,omitempty"` +} + +// Scan directions. +const ( + ScanInput = "input" + ScanOutput = "output" +) + +// TextScanRequest is the body of POST /v1/scan/text. +type TextScanRequest struct { + Direction string `json:"direction"` + Text string `json:"text"` +} + +// TextScanResponse is the body of a successful text scan; it is the wire +// form of toolapprove.TextVerdict. +type TextScanResponse struct { + Blocked bool `json:"blocked"` + CriticalInjection bool `json:"critical_injection,omitempty"` + Findings []TextFinding `json:"findings,omitempty"` +} + +// TextFinding is one rule hit; Snippet is already masked by the scanner. +// High is the severity bit toolapprove turns into a violation. +type TextFinding struct { + Kind string `json:"kind"` + Snippet string `json:"snippet,omitempty"` + Rule string `json:"rule,omitempty"` + High bool `json:"high,omitempty"` +} + +// Condition actions. +const ( + ConditionObserve = "observe" + ConditionClear = "clear" +) + +// ConditionReport is the body of POST /v1/conditions: one Observe or Clear +// of crux.ConditionReporter. +type ConditionReport struct { + Action string `json:"action"` + Class string `json:"class"` + Key string `json:"key"` + Message string `json:"message,omitempty"` + Diagnostics string `json:"diagnostics,omitempty"` +} + +// LinearCredential is the body of GET /v1/linear/credential. Both fields +// empty means hive has none to inject. +type LinearCredential struct { + AccessToken string `json:"access_token,omitempty"` + APIKey string `json:"api_key,omitempty"` +} + +// Empty reports whether no credential is available. +func (c LinearCredential) Empty() bool { return c.AccessToken == "" && c.APIKey == "" } + +// String redacts both secrets. +func (c LinearCredential) String() string { + return fmt.Sprintf("callback.LinearCredential{AccessToken: %s, APIKey: %s}", redactedIf(c.AccessToken), redactedIf(c.APIKey)) +} + +// GoString is String, for %#v. +func (c LinearCredential) GoString() string { return c.String() } + +func redactedIf(s string) string { + if s == "" { + return `""` + } + return "" +} + +// CreatePRRequest is the body of POST /v1/pulls. +type CreatePRRequest struct { + Repo string `json:"repo"` + Head string `json:"head"` + Base string `json:"base"` + Title string `json:"title"` + Body string `json:"body,omitempty"` +} + +// CreatePRResponse is the body of a successful POST /v1/pulls: the fields +// of crux.PRResult. +type CreatePRResponse struct { + Number int `json:"number"` + URL string `json:"url"` + Author string `json:"author,omitempty"` + AlreadyExisted bool `json:"already_existed,omitempty"` + DuplicateTree bool `json:"duplicate_tree,omitempty"` + DuplicateReason string `json:"duplicate_reason,omitempty"` +} + +// Capabilities is the full capability set, as both ends see it: *Client +// implements it over HTTP, hive's pkg/cruxhost directly, and the runtime's +// crux/hivecap adapters are written against it so either serves. A nil +// error means hive answered; the error cases are the sentinels above, an +// *api.StatusError for anything else the server said, or a plain error. +type Capabilities interface { + TokenMinter + GitHubAccounting + UsageRecorder + CanaryScanner + TextScanner + ConditionSink + LinearCredentials + PRCreator +} + +// TokenMinter mints a scoped GitHub token. +type TokenMinter interface { + MintToken(ctx context.Context, req MintTokenRequest) (MintTokenResponse, error) +} + +// GitHubAccounting records relayed REST traffic and reads the shared +// core-bucket snapshot. +type GitHubAccounting interface { + RecordREST(ctx context.Context, rec RESTRecord) error + RateLimit(ctx context.Context) (RateLimitSnapshot, error) +} + +// UsageRecorder records inference token usage. +type UsageRecorder interface { + RecordUsage(ctx context.Context, rec UsageRecord) error +} + +// CanaryScanner scans a GitHub write body for planted canaries. +type CanaryScanner interface { + ScanCanary(ctx context.Context, req CanaryScanRequest) (CanaryScanResponse, error) +} + +// TextScanner scans tool payload text. +type TextScanner interface { + ScanText(ctx context.Context, req TextScanRequest) (TextScanResponse, error) +} + +// ConditionSink receives runtime condition observations and clears. +type ConditionSink interface { + ReportCondition(ctx context.Context, rep ConditionReport) error +} + +// LinearCredentials returns the Linear credential to inject. +type LinearCredentials interface { + LinearCredential(ctx context.Context) (LinearCredential, error) +} + +// PRCreator opens a pull request as the hive's App. +type PRCreator interface { + CreatePR(ctx context.Context, req CreatePRRequest) (CreatePRResponse, error) +} + +// Policy is the body of GET /v1/policy: the values hive hands an in-process +// runtime through the crux.Host policy setters (SetBuildInfo, +// SetExplainModeDefaultResolver, SetKickLogSourceResolver, +// SetGatewayBackendChecker, SetRepoAutoMergeEnabledResolver, +// SetDeadSessionRecoveryOwner, SetSandboxConfig), as one snapshot a sidecar +// runtime polls. An absent (nil / false) field is a setter hive has not +// called; the runtime leaves its own default in place for it. +type Policy struct { + Build *BuildInfo `json:"build,omitempty"` + // ExplainModeDefault is the hive-wide default explain mode, already + // resolved against hive's environment fallback. + ExplainModeDefault *string `json:"explain_mode_default,omitempty"` + // KickLogSource is the kick log source in force, already resolved + // against hive's environment override. + KickLogSource *string `json:"kick_log_source,omitempty"` + // GatewayBackends names the configured model gateways the runtime treats + // as inference-routable backends (matched case-insensitively). + GatewayBackends *[]string `json:"gateway_backends,omitempty"` + // RepoAutoMerge says hive answers POST /v1/policy/repo-auto-merge; the + // per-repository answer is a call, not part of the snapshot. + RepoAutoMerge bool `json:"repo_auto_merge,omitempty"` + // DeadSessionRecoveryOwnedElsewhere is SetDeadSessionRecoveryOwner's + // argument: true when hive's watchdog owns dead-session restarts. + DeadSessionRecoveryOwnedElsewhere *bool `json:"dead_session_recovery_owned_elsewhere,omitempty"` + // Sandbox is the agent_sandbox block (crux/config.AgentSandboxConfig) + // as JSON. It stays raw so this package keeps importing only the + // standard library and crux/api. + Sandbox json.RawMessage `json:"sandbox,omitempty"` +} + +// BuildInfo is the hive version and commit recorded in kick log headers. +type BuildInfo struct { + Version string `json:"version"` + Commit string `json:"commit"` +} + +// RepoAutoMergeRequest is the body of POST /v1/policy/repo-auto-merge. +type RepoAutoMergeRequest struct { + Repo string `json:"repo"` +} + +// RepoAutoMergeResponse is its answer. +type RepoAutoMergeResponse struct { + Enabled bool `json:"enabled"` +} + +// PolicySource is the policy half of the callback contract. It is separate +// from Capabilities so existing implementations stay valid: the server +// answers the policy routes only when its Caps also implement it (404 +// CodeNotConfigured otherwise); *Client always does. +type PolicySource interface { + Policy(ctx context.Context) (Policy, error) + RepoAutoMergeEnabled(ctx context.Context, repo string) (bool, error) +} + +// InferenceRoute is the body of PUT /v1/inference/routes/{agent}: the +// backend and model the manager resolved for the agent at launch, which +// hive's inference proxy turns into the upstream it forwards that agent's +// API traffic to (the in-process SetInferenceCallbacks setRoute arguments). +type InferenceRoute struct { + Backend string `json:"backend"` + Model string `json:"model"` +} + +// InferenceRouter is the inference-route half of the callback contract, +// optional like PolicySource: the server answers the routes only when its +// Caps implements it; *Client always does. Set replaces the agent's route; +// Clear removes it (the clearRoute callback). +type InferenceRouter interface { + SetInferenceRoute(ctx context.Context, agent string, route InferenceRoute) error + ClearInferenceRoute(ctx context.Context, agent string) error +} + +// EffectClaim is the body of POST /v1/effects: the fields of effects.Claim. +// The runtime is about to perform one external mutation (today: a sandbox +// branch push) and asks hive's mutation boundary to journal it, exactly as +// an in-process SetSandboxMutationBoundary would wrap it. +type EffectClaim struct { + Repo string `json:"repo"` + Kind string `json:"kind"` + Target string `json:"target"` + Actor string `json:"actor,omitempty"` + Inputs map[string]string `json:"inputs,omitempty"` +} + +// BeginEffectResponse is the body of a successful POST /v1/effects: hive +// admitted the effect and is holding the claim until EffectEndPath(ID) is +// called. A denial is 403 CodeEffectDenied, not a body. +type BeginEffectResponse struct { + ID string `json:"id"` +} + +// EndEffectRequest is the body of POST /v1/effects/{id}/end: the result of +// the runtime-side effect. Error is the effect's own error text, "" when it +// succeeded; hive journals it and releases the claim. +type EndEffectRequest struct { + Provenance string `json:"provenance,omitempty"` + Error string `json:"error,omitempty"` +} + +// EndEffectResponse is the body of a successful end: what hive's boundary +// returned around the effect. Error is non-empty when the boundary itself +// failed after the effect ran (the journal could not record it); the +// runtime surfaces it like the in-process boundary's return. +type EndEffectResponse struct { + Provenance string `json:"provenance,omitempty"` + Error string `json:"error,omitempty"` +} + +// EffectBoundary is the mutation-boundary half of the contract, optional +// like PolicySource. Begin blocks until hive's boundary has admitted or +// denied the claim; End delivers the effect's outcome and returns the +// boundary's. A runtime that begins and never ends (it died) leaves hive to +// time the claim out on its own TTL. +type EffectBoundary interface { + BeginEffect(ctx context.Context, claim EffectClaim) (BeginEffectResponse, error) + EndEffect(ctx context.Context, id string, req EndEffectRequest) (EndEffectResponse, error) +} + +// AgentMintRequest is the body of POST /v1/tokens/mint: the arguments of +// crux.AgentMintIssuer.MintAgentToken. Tier is the agent's trust tier, the +// same one its App token is scoped to. +type AgentMintRequest struct { + Agent string `json:"agent"` + Tier string `json:"tier,omitempty"` +} + +// AgentMintResponse is the body of a successful POST /v1/tokens/mint. An +// empty Token with a nil error is how a hive whose mint is configured but +// disabled answers (crux.AgentMintIssuer's "" contract); a hive without the +// mint at all answers 404 not_configured. +type AgentMintResponse struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at,omitempty"` +} + +// String prints the response with the token redacted. +func (r AgentMintResponse) String() string { + return fmt.Sprintf("callback.AgentMintResponse{Token: %s, ExpiresAt: %s}", redactedIf(r.Token), r.ExpiresAt.Format(time.RFC3339)) +} + +// GoString is String, for %#v. +func (r AgentMintResponse) GoString() string { return r.String() } + +// AgentMinter is the mint half of the contract, optional like PolicySource: +// the server answers POST /v1/tokens/mint only when its Caps implements it; +// *Client always does. It is crux.AgentMintIssuer with a context. +type AgentMinter interface { + MintAgentToken(ctx context.Context, req AgentMintRequest) (AgentMintResponse, error) +} + +// BobCredential is the body of GET /v1/bob/credential: the IBM Bob API key +// hive resolved from its config (SetBobAPIKeyResolver) and the loggable +// source it came from ("file:", "env:", "" when unconfigured; +// SetBobKeySourceResolver). APIKey is never logged. +type BobCredential struct { + APIKey string `json:"api_key,omitempty"` + Source string `json:"source,omitempty"` +} + +// String prints the credential with the key redacted. +func (c BobCredential) String() string { + return fmt.Sprintf("callback.BobCredential{APIKey: %s, Source: %q}", redactedIf(c.APIKey), c.Source) +} + +// GoString is String, for %#v. +func (c BobCredential) GoString() string { return c.String() } + +// BobCredentials is the Bob half of the contract, optional like +// PolicySource. +type BobCredentials interface { + BobCredential(ctx context.Context) (BobCredential, error) +} diff --git a/src/crux/api/callback/callback_test.go b/src/crux/api/callback/callback_test.go new file mode 100644 index 0000000000..37ff184e52 --- /dev/null +++ b/src/crux/api/callback/callback_test.go @@ -0,0 +1,1107 @@ +package callback + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" +) + +const ( + testToken = "ghs_minted_secret_do_not_log_0123456789" + testBearer = "bearer-shared-secret-ABC" + testLinearTok = "lin_oauth_secret_XYZ" + testLinearKey = "lin_api_secret_QRS" + notConfiguredC = "__none__" +) + +// fakeCaps is a Capabilities that records what it was asked and answers +// from fields. absent lists capabilities Configured denies. +type fakeCaps struct { + mu sync.Mutex + absent map[string]bool + fail error + mint []MintTokenRequest + rest []RESTRecord + usage []UsageRecord + canary []CanaryScanRequest + text []TextScanRequest + conds []ConditionReport + pulls []CreatePRRequest + linear LinearCredential + snap RateLimitSnapshot + detect bool + blocked bool + policy Policy + merge []string + routes map[string]InferenceRoute + effects []EffectClaim + ends map[string]EndEffectRequest + deny bool + mints []AgentMintRequest + bob BobCredential +} + +func (f *fakeCaps) Configured(c string) bool { return !f.absent[c] } + +func (f *fakeCaps) MintToken(_ context.Context, req MintTokenRequest) (MintTokenResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.mint = append(f.mint, req) + if f.fail != nil { + return MintTokenResponse{}, f.fail + } + return MintTokenResponse{Token: testToken, ExpiresAt: time.Date(2026, 10, 25, 13, 0, 0, 0, time.UTC)}, nil +} + +func (f *fakeCaps) RecordREST(_ context.Context, rec RESTRecord) error { + f.mu.Lock() + defer f.mu.Unlock() + f.rest = append(f.rest, rec) + return f.fail +} + +func (f *fakeCaps) RateLimit(context.Context) (RateLimitSnapshot, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.snap, f.fail +} + +func (f *fakeCaps) RecordUsage(_ context.Context, rec UsageRecord) error { + f.mu.Lock() + defer f.mu.Unlock() + f.usage = append(f.usage, rec) + return f.fail +} + +func (f *fakeCaps) ScanCanary(_ context.Context, req CanaryScanRequest) (CanaryScanResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.canary = append(f.canary, req) + if f.fail != nil { + return CanaryScanResponse{}, f.fail + } + return CanaryScanResponse{Detected: f.detect, Agent: req.Agent, Source: req.Source}, nil +} + +func (f *fakeCaps) ScanText(_ context.Context, req TextScanRequest) (TextScanResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.text = append(f.text, req) + if f.fail != nil { + return TextScanResponse{}, f.fail + } + if f.blocked { + return TextScanResponse{Blocked: true, CriticalInjection: true, Findings: []TextFinding{{Kind: "injection", Snippet: "ign***", Rule: "inj-1"}}}, nil + } + return TextScanResponse{}, nil +} + +func (f *fakeCaps) ReportCondition(_ context.Context, rep ConditionReport) error { + f.mu.Lock() + defer f.mu.Unlock() + f.conds = append(f.conds, rep) + return f.fail +} + +func (f *fakeCaps) LinearCredential(context.Context) (LinearCredential, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.linear, f.fail +} + +func (f *fakeCaps) CreatePR(_ context.Context, req CreatePRRequest) (CreatePRResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.pulls = append(f.pulls, req) + if f.fail != nil { + return CreatePRResponse{}, f.fail + } + return CreatePRResponse{Number: 42, URL: "https://github.com/o/r/pull/42", Author: "hive[bot]", AlreadyExisted: true, DuplicateTree: true, DuplicateReason: "same-tree"}, nil +} + +func (f *fakeCaps) Policy(context.Context) (Policy, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.policy, f.fail +} + +func (f *fakeCaps) MintAgentToken(_ context.Context, req AgentMintRequest) (AgentMintResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.mints = append(f.mints, req) + if f.fail != nil { + return AgentMintResponse{}, f.fail + } + if req.Tier == "disabled" { + return AgentMintResponse{}, nil + } + return AgentMintResponse{Token: "mint-" + req.Agent, ExpiresAt: time.Unix(1_800_000_000, 0).UTC()}, nil +} + +func (f *fakeCaps) BobCredential(context.Context) (BobCredential, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.bob, f.fail +} + +func (f *fakeCaps) RepoAutoMergeEnabled(_ context.Context, repo string) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.merge = append(f.merge, repo) + return repo == "o/merge", f.fail +} + +func (f *fakeCaps) SetInferenceRoute(_ context.Context, agent string, route InferenceRoute) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.routes == nil { + f.routes = map[string]InferenceRoute{} + } + f.routes[agent] = route + return f.fail +} + +func (f *fakeCaps) ClearInferenceRoute(_ context.Context, agent string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.routes, agent) + return f.fail +} + +func (f *fakeCaps) BeginEffect(_ context.Context, claim EffectClaim) (BeginEffectResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.effects = append(f.effects, claim) + if f.fail != nil { + return BeginEffectResponse{}, f.fail + } + if f.deny { + return BeginEffectResponse{}, fmt.Errorf("%w: fence held by other", ErrEffectDenied) + } + return BeginEffectResponse{ID: fmt.Sprintf("e%d", len(f.effects))}, nil +} + +func (f *fakeCaps) EndEffect(_ context.Context, id string, req EndEffectRequest) (EndEffectResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.fail != nil { + return EndEffectResponse{}, f.fail + } + if id != "e1" { + return EndEffectResponse{}, ErrEffectUnknown + } + if f.ends == nil { + f.ends = map[string]EndEffectRequest{} + } + f.ends[id] = req + return EndEffectResponse{Provenance: req.Provenance, Error: req.Error}, nil +} + +var ( + _ Capabilities = (*fakeCaps)(nil) + _ PolicySource = (*fakeCaps)(nil) + _ InferenceRouter = (*fakeCaps)(nil) + _ EffectBoundary = (*fakeCaps)(nil) +) + +// shortSocketDir returns a private directory whose socket paths fit in +// sun_path on macOS (t.TempDir is too long there). +func shortSocketDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("", "cbk") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + if err := os.Chmod(dir, 0o700); err != nil { + t.Fatal(err) + } + return dir +} + +// serveUnix runs srv on a fresh socket and returns a client for it plus the +// captured log output. +func serveUnix(t *testing.T, srv *Server, clientToken string) (*Client, *bytes.Buffer) { + t.Helper() + var logs bytes.Buffer + srv.Logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + sock := filepath.Join(shortSocketDir(t), SocketName) + ls, err := Listen(ListenConfig{Socket: sock, Token: srv.Token}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- srv.Serve(ctx, ls) }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + t.Errorf("Serve: %v", err) + } + }) + c, err := NewClient("unix://"+sock, clientToken) + if err != nil { + t.Fatal(err) + } + waitReady(t, c) + return c, &logs +} + +// waitReady makes one Health call. The socket is bound by Listen before +// Serve starts, so the connection queues in the backlog and the call +// returns once Serve accepts: no polling needed. +func waitReady(t *testing.T, c *Client) { + t.Helper() + if _, err := c.Health(context.Background()); err != nil && !errors.Is(err, ErrUnauthorized) { + t.Fatalf("endpoint never became ready: %v", err) + } +} + +func TestEveryCapabilityOverUnixSocket(t *testing.T) { + caps := &fakeCaps{detect: true, blocked: true, linear: LinearCredential{AccessToken: testLinearTok}, snap: RateLimitSnapshot{Remaining: 4000, Reset: time.Unix(1_800_000_000, 0).UTC(), OK: true}} + c, logs := serveUnix(t, NewServer(caps, ""), "") + ctx := context.Background() + + h, err := c.Health(ctx) + if err != nil || h.Version != Version || len(h.Capabilities) != len(allCapabilities) { + t.Fatalf("Health = %+v, %v", h, err) + } + + tok, err := c.MintToken(ctx, MintTokenRequest{Agent: "scanner", Repo: "o/r", Tier: "contributor", TTLSeconds: 600}) + if err != nil || tok.Token != testToken || tok.ExpiresAt.IsZero() { + t.Fatalf("MintToken = %v, %v", tok, err) + } + if got := caps.mint[0]; got.Agent != "scanner" || got.Repo != "o/r" || got.Tier != "contributor" || got.TTLSeconds != 600 { + t.Errorf("mint request not carried: %+v", got) + } + + if err := c.RecordREST(ctx, RESTRecord{Caller: "agent:a", Method: "GET", Path: "/repos/o/r", Status: 200, Injected: true, RateLimit: map[string]string{"X-Ratelimit-Remaining": "4999"}}); err != nil { + t.Fatal(err) + } + if got := caps.rest[0]; !got.Injected || got.RateLimit["X-Ratelimit-Remaining"] != "4999" || got.Caller != "agent:a" { + t.Errorf("rest record not carried: %+v", got) + } + snap, err := c.RateLimit(ctx) + if err != nil || !snap.OK || snap.Remaining != 4000 || !snap.Reset.Equal(caps.snap.Reset) { + t.Fatalf("RateLimit = %+v, %v", snap, err) + } + + if err := c.RecordUsage(ctx, UsageRecord{Agent: "a", Model: "m", Kind: UsageKindCopilotCache, InputTokens: 1, OutputTokens: 2, CacheRead: 3, CacheCreation: 4}); err != nil { + t.Fatal(err) + } + if got := caps.usage[0]; got.Kind != UsageKindCopilotCache || got.CacheCreation != 4 { + t.Errorf("usage not carried: %+v", got) + } + + cs, err := c.ScanCanary(ctx, CanaryScanRequest{Agent: "a", Text: "body", Source: "github:POST /x"}) + if err != nil || !cs.Detected || cs.Agent != "a" || cs.Source != "github:POST /x" { + t.Fatalf("ScanCanary = %+v, %v", cs, err) + } + + ts, err := c.ScanText(ctx, TextScanRequest{Direction: ScanInput, Text: "ignore previous"}) + if err != nil || !ts.Blocked || !ts.CriticalInjection || len(ts.Findings) != 1 || ts.Findings[0].Rule != "inj-1" { + t.Fatalf("ScanText = %+v, %v", ts, err) + } + + if err := c.ReportCondition(ctx, ConditionReport{Action: ConditionObserve, Class: "agent_stall", Key: "a", Message: "m", Diagnostics: "d"}); err != nil { + t.Fatal(err) + } + if err := c.ReportCondition(ctx, ConditionReport{Action: ConditionClear, Class: "agent_stall", Key: "a"}); err != nil { + t.Fatal(err) + } + if len(caps.conds) != 2 || caps.conds[0].Diagnostics != "d" || caps.conds[1].Action != ConditionClear { + t.Errorf("conditions not carried: %+v", caps.conds) + } + + lc, err := c.LinearCredential(ctx) + if err != nil || lc.AccessToken != testLinearTok || lc.Empty() { + t.Fatalf("LinearCredential = %v, %v", lc, err) + } + + pr, err := c.CreatePR(ctx, CreatePRRequest{Repo: "o/r", Head: "h", Base: "v5", Title: "t", Body: "b"}) + if err != nil || pr.Number != 42 || !pr.AlreadyExisted || !pr.DuplicateTree || pr.DuplicateReason != "same-tree" || pr.Author != "hive[bot]" { + t.Fatalf("CreatePR = %+v, %v", pr, err) + } + if caps.pulls[0].Body != "b" { + t.Errorf("pull request body not carried: %+v", caps.pulls[0]) + } + + st := c.Status() + if !st.Reachable || st.FailedCalls != 0 || st.LastSuccess.IsZero() || st.LastError != "" { + t.Errorf("Status after success = %+v", st) + } + + // The server logged method/path/status for every call and nothing + // sensitive. + for _, secret := range []string{testToken, testLinearTok, "ignore previous", "body"} { + if strings.Contains(logs.String(), secret) { + t.Errorf("server log contains %q:\n%s", secret, logs.String()) + } + } + if !strings.Contains(logs.String(), PathTokens) { + t.Errorf("server log has no route lines:\n%s", logs.String()) + } +} + +func TestTokenNeverPrinted(t *testing.T) { + r := MintTokenResponse{Token: testToken, ExpiresAt: time.Now()} + for _, s := range []string{fmt.Sprint(r), fmt.Sprintf("%v", r), fmt.Sprintf("%+v", r), fmt.Sprintf("%#v", r), fmt.Sprintf("%s", r), fmt.Sprint(&r)} { + if strings.Contains(s, testToken) { + t.Errorf("formatted response leaks the token: %s", s) + } + } + l := LinearCredential{AccessToken: testLinearTok, APIKey: testLinearKey} + for _, s := range []string{fmt.Sprint(l), fmt.Sprintf("%+v", l), fmt.Sprintf("%#v", l)} { + if strings.Contains(s, testLinearTok) || strings.Contains(s, testLinearKey) { + t.Errorf("formatted credential leaks a secret: %s", s) + } + } + if !strings.Contains(fmt.Sprint(LinearCredential{}), `""`) { + t.Error("empty credential should print as empty, not redacted") + } + // JSON still carries it: that is the wire. + data, _ := json.Marshal(r) + if !strings.Contains(string(data), testToken) { + t.Error("JSON form must carry the token") + } +} + +func TestClientErrorsNeverQuoteBearer(t *testing.T) { + caps := &fakeCaps{} + c, logs := serveUnix(t, NewServer(caps, testBearer), "wrong-"+testBearer) + _, err := c.MintToken(context.Background(), MintTokenRequest{}) + if !errors.Is(err, ErrUnauthorized) { + t.Fatalf("err = %v, want ErrUnauthorized", err) + } + if strings.Contains(err.Error(), testBearer) { + t.Errorf("error quotes the bearer: %v", err) + } + if strings.Contains(logs.String(), testBearer) { + t.Errorf("server log quotes the bearer:\n%s", logs.String()) + } + if len(caps.mint) != 0 { + t.Error("an unauthorized request must not reach the capability") + } + if st := c.Status(); !st.Reachable { + t.Errorf("a 401 is hive answering; Status = %+v", st) + } +} + +func TestBearerTokenAccepted(t *testing.T) { + caps := &fakeCaps{} + c, _ := serveUnix(t, NewServer(caps, testBearer), testBearer) + if _, err := c.MintToken(context.Background(), MintTokenRequest{Repo: "o/r"}); err != nil { + t.Fatal(err) + } +} + +func TestTokenlessServerRefusesOffSocket(t *testing.T) { + // Handler mounted on a plain TCP httptest server with no unix tag: the + // token-less rule must refuse, exactly as crux/serve does. + caps := &fakeCaps{} + ts := httptest.NewServer(NewServer(caps, "").Handler()) + defer ts.Close() + c, err := NewClient(ts.URL, "") + if err != nil { + t.Fatal(err) + } + if _, err := c.Health(context.Background()); !errors.Is(err, ErrUnauthorized) { + t.Fatalf("off-socket health = %v, want ErrUnauthorized", err) + } + // The same handler with the unix tag admits it. + ts2 := httptest.NewUnstartedServer(NewServer(caps, "").Handler()) + ts2.Config.BaseContext = func(net.Listener) context.Context { return WithUnixNetwork(context.Background()) } + ts2.Start() + defer ts2.Close() + c2, _ := NewClient(ts2.URL, "") + if _, err := c2.Health(context.Background()); err != nil { + t.Fatalf("tagged health = %v", err) + } + // With a token, TCP is fine once the token is presented. + ts3 := httptest.NewServer(NewServer(caps, testBearer).Handler()) + defer ts3.Close() + c3, _ := NewClient(ts3.URL, testBearer) + if _, err := c3.Health(context.Background()); err != nil { + t.Fatalf("tcp+token health = %v", err) + } +} + +func TestUnreachableFailsClosed(t *testing.T) { + sock := filepath.Join(shortSocketDir(t), "nobody.sock") + c, err := NewClient("unix://"+sock, "") + if err != nil { + t.Fatal(err) + } + c.SetTimeout(200 * time.Millisecond) + ctx := context.Background() + + if _, err := c.MintToken(ctx, MintTokenRequest{Repo: "o/r"}); !errors.Is(err, ErrUnavailable) { + t.Errorf("MintToken = %v, want ErrUnavailable", err) + } + if err := c.RecordREST(ctx, RESTRecord{Method: "GET", Path: "/"}); !errors.Is(err, ErrUnavailable) { + t.Errorf("RecordREST = %v", err) + } + if snap, err := c.RateLimit(ctx); !errors.Is(err, ErrUnavailable) || snap.OK { + t.Errorf("RateLimit = %+v, %v", snap, err) + } + if err := c.RecordUsage(ctx, UsageRecord{Kind: UsageKindGateway}); !errors.Is(err, ErrUnavailable) { + t.Errorf("RecordUsage = %v", err) + } + if _, err := c.ScanCanary(ctx, CanaryScanRequest{}); !errors.Is(err, ErrUnavailable) { + t.Errorf("ScanCanary = %v", err) + } + if _, err := c.ScanText(ctx, TextScanRequest{Direction: ScanInput}); !errors.Is(err, ErrUnavailable) { + t.Errorf("ScanText = %v", err) + } + if err := c.ReportCondition(ctx, ConditionReport{Action: ConditionObserve, Class: "x"}); !errors.Is(err, ErrUnavailable) { + t.Errorf("ReportCondition = %v", err) + } + if lc, err := c.LinearCredential(ctx); !errors.Is(err, ErrUnavailable) || !lc.Empty() { + t.Errorf("LinearCredential = %v, %v", lc, err) + } + if _, err := c.CreatePR(ctx, CreatePRRequest{}); !errors.Is(err, ErrUnavailable) { + t.Errorf("CreatePR = %v", err) + } + + st := c.Status() + if st.Reachable || st.FailedCalls != 9 || st.Since.IsZero() || st.LastError == "" { + t.Errorf("Status = %+v", st) + } + if strings.Contains(st.LastError, "http://") { + t.Errorf("transport error should not carry the URL: %s", st.LastError) + } +} + +func TestStatusRecoversAfterOutage(t *testing.T) { + caps := &fakeCaps{} + sock := filepath.Join(shortSocketDir(t), SocketName) + c, err := NewClient("unix://"+sock, "") + if err != nil { + t.Fatal(err) + } + c.SetTimeout(200 * time.Millisecond) + if _, err := c.Health(context.Background()); !errors.Is(err, ErrUnavailable) { + t.Fatalf("pre-start health = %v", err) + } + if c.Status().Reachable { + t.Fatal("status should be unreachable") + } + ls, err := Listen(ListenConfig{Socket: sock}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- NewServer(caps, "").Serve(ctx, ls) }() + waitReady(t, c) + st := c.Status() + if !st.Reachable || st.FailedCalls != 0 || !st.Since.IsZero() { + t.Errorf("status after recovery = %+v", st) + } + cancel() + if err := <-done; err != nil { + t.Fatal(err) + } + if _, err := os.Stat(sock); !errors.Is(err, os.ErrNotExist) { + t.Errorf("socket file not removed on shutdown: %v", err) + } +} + +func TestServerErrorsBecomeUnavailableAndUpstream(t *testing.T) { + caps := &fakeCaps{fail: errors.New("github said no")} + c, _ := serveUnix(t, NewServer(caps, ""), "") + ctx := context.Background() + _, err := c.MintToken(ctx, MintTokenRequest{Repo: "o/r"}) + if !errors.Is(err, ErrUnavailable) { + t.Fatalf("502 should be ErrUnavailable: %v", err) + } + var se *api.StatusError + if !errors.As(err, &se) || se.Status != http.StatusBadGateway || se.Body.Code != CodeUpstream || !strings.Contains(se.Body.Error, "github said no") { + t.Errorf("status error = %+v", se) + } + for _, call := range []func() error{ + func() error { return c.RecordREST(ctx, RESTRecord{Method: "GET", Path: "/"}) }, + func() error { _, e := c.RateLimit(ctx); return e }, + func() error { return c.RecordUsage(ctx, UsageRecord{Kind: UsageKindGateway}) }, + func() error { _, e := c.ScanCanary(ctx, CanaryScanRequest{}); return e }, + func() error { _, e := c.ScanText(ctx, TextScanRequest{Direction: ScanOutput}); return e }, + func() error { return c.ReportCondition(ctx, ConditionReport{Action: ConditionClear, Class: "x"}) }, + func() error { _, e := c.LinearCredential(ctx); return e }, + func() error { + _, e := c.CreatePR(ctx, CreatePRRequest{Repo: "r", Head: "h", Base: "b", Title: "t"}) + return e + }, + } { + if err := call(); !errors.Is(err, ErrUnavailable) { + t.Errorf("capability failure = %v, want ErrUnavailable", err) + } + } +} + +func TestNotConfigured(t *testing.T) { + caps := &fakeCaps{absent: map[string]bool{CapabilityTokens: true, CapabilityCanary: true, CapabilityLinear: true}} + c, _ := serveUnix(t, NewServer(caps, ""), "") + ctx := context.Background() + h, err := c.Health(ctx) + if err != nil { + t.Fatal(err) + } + for _, name := range h.Capabilities { + if caps.absent[name] { + t.Errorf("health lists absent capability %s", name) + } + } + if len(h.Capabilities) != len(allCapabilities)-3 { + t.Errorf("health capabilities = %v", h.Capabilities) + } + if _, err := c.MintToken(ctx, MintTokenRequest{}); !errors.Is(err, ErrNotConfigured) { + t.Errorf("MintToken = %v, want ErrNotConfigured", err) + } + if _, err := c.ScanCanary(ctx, CanaryScanRequest{}); !errors.Is(err, ErrNotConfigured) { + t.Errorf("ScanCanary = %v", err) + } + if _, err := c.LinearCredential(ctx); !errors.Is(err, ErrNotConfigured) { + t.Errorf("LinearCredential = %v", err) + } + if len(caps.mint)+len(caps.canary) != 0 { + t.Error("absent capabilities must not be dispatched") + } + if !c.Status().Reachable { + t.Error("not_configured is hive answering") + } + // A capability that reports ErrNotConfigured itself is also a 404. + caps2 := &fakeCaps{fail: ErrNotConfigured} + c2, _ := serveUnix(t, NewServer(caps2, ""), "") + if _, err := c2.CreatePR(ctx, CreatePRRequest{Repo: "r", Head: "h", Base: "b", Title: "t"}); !errors.Is(err, ErrNotConfigured) { + t.Errorf("CreatePR = %v", err) + } + // A nil Caps has nothing. + c3, _ := serveUnix(t, NewServer(nil, ""), "") + h3, err := c3.Health(ctx) + if err != nil || len(h3.Capabilities) != 0 { + t.Errorf("nil caps health = %+v, %v", h3, err) + } +} + +func TestServerValidation(t *testing.T) { + caps := &fakeCaps{} + c, _ := serveUnix(t, NewServer(caps, ""), "") + ctx := context.Background() + cases := []struct { + name string + call func() error + }{ + {"negative ttl", func() error { _, e := c.MintToken(ctx, MintTokenRequest{TTLSeconds: -1}); return e }}, + {"rest without method", func() error { return c.RecordREST(ctx, RESTRecord{Path: "/"}) }}, + {"usage bad kind", func() error { return c.RecordUsage(ctx, UsageRecord{Kind: "bogus"}) }}, + {"text bad direction", func() error { _, e := c.ScanText(ctx, TextScanRequest{Direction: "sideways"}); return e }}, + {"condition bad action", func() error { return c.ReportCondition(ctx, ConditionReport{Action: "poke", Class: "x"}) }}, + {"condition no class", func() error { return c.ReportCondition(ctx, ConditionReport{Action: ConditionObserve}) }}, + {"pull missing fields", func() error { _, e := c.CreatePR(ctx, CreatePRRequest{Repo: "r"}); return e }}, + } + for _, tc := range cases { + err := tc.call() + var se *api.StatusError + if !errors.As(err, &se) || se.Status != http.StatusBadRequest || se.Body.Code != CodeBadRequest { + t.Errorf("%s: err = %v, want 400 bad_request", tc.name, err) + } + if errors.Is(err, ErrUnavailable) { + t.Errorf("%s: a 400 is not an outage", tc.name) + } + } + if n := len(caps.mint) + len(caps.rest) + len(caps.usage) + len(caps.text) + len(caps.conds) + len(caps.pulls); n != 0 { + t.Errorf("invalid requests reached the capability %d times", n) + } + // Malformed JSON. + req, _ := http.NewRequest(http.MethodPost, c.addr.BaseURL+PathTokens, strings.NewReader("{nope")) + resp, err := c.http.Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("malformed JSON status = %d", resp.StatusCode) + } + // Unknown route is a plain 404 without CodeNotConfigured: not an outage, + // not "not configured". + err = c.do(ctx, http.MethodGet, "/v1/nothing", nil, nil) + if errors.Is(err, ErrNotConfigured) || errors.Is(err, ErrUnavailable) || !api.IsNotFound(err) { + t.Errorf("unknown route = %v", err) + } +} + +func TestMintTokenServerFillsExpiryAndRefusesEmpty(t *testing.T) { + // Expiry defaulted when the capability does not know it. + srv := NewServer(capsFunc{fakeCaps: &fakeCaps{}, mint: func(MintTokenRequest) (MintTokenResponse, error) { + return MintTokenResponse{Token: "t"}, nil + }}, "") + c, _ := serveUnix(t, srv, "") + before := time.Now() + resp, err := c.MintToken(context.Background(), MintTokenRequest{}) + if err != nil { + t.Fatal(err) + } + if d := resp.ExpiresAt.Sub(before); d < installationTokenTTL-time.Minute || d > installationTokenTTL+time.Minute { + t.Errorf("defaulted expiry %v, want ~%v", d, installationTokenTTL) + } + // An empty token from the capability is a 502, never handed out. + srv2 := NewServer(capsFunc{fakeCaps: &fakeCaps{}, mint: func(MintTokenRequest) (MintTokenResponse, error) { + return MintTokenResponse{}, nil + }}, "") + c2, _ := serveUnix(t, srv2, "") + if _, err := c2.MintToken(context.Background(), MintTokenRequest{}); !errors.Is(err, ErrUnavailable) { + t.Errorf("empty token = %v, want ErrUnavailable", err) + } +} + +// capsFunc is a Capabilities with a pluggable MintToken; the rest refuse. +type capsFunc struct { + *fakeCaps + mint func(MintTokenRequest) (MintTokenResponse, error) +} + +func (c capsFunc) MintToken(_ context.Context, req MintTokenRequest) (MintTokenResponse, error) { + return c.mint(req) +} + +func TestClientEmptyTokenFromWire(t *testing.T) { + // A server that bypasses the Server type and returns {"token":""}. + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"token":"","expires_at":"2026-01-01T00:00:00Z"}`)) + })) + defer ts.Close() + c, _ := NewClient(ts.URL, "x") + if _, err := c.MintToken(context.Background(), MintTokenRequest{}); !errors.Is(err, ErrUnavailable) { + t.Errorf("empty wire token = %v", err) + } + // Garbage JSON from the server is also unavailable. + ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`nope`)) + })) + defer ts2.Close() + c2, _ := NewClient(ts2.URL, "x") + if _, err := c2.RateLimit(context.Background()); !errors.Is(err, ErrUnavailable) { + t.Errorf("garbage body = %v", err) + } +} + +func TestClientTimeoutAndContext(t *testing.T) { + block := make(chan struct{}) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-block: + case <-r.Context().Done(): + } + })) + defer ts.Close() + defer close(block) + c, _ := NewClient(ts.URL, "x") + c.SetTimeout(100 * time.Millisecond) + start := time.Now() + if _, err := c.Health(context.Background()); !errors.Is(err, ErrUnavailable) { + t.Errorf("timeout = %v", err) + } + if time.Since(start) > 2*time.Second { + t.Error("default timeout not applied") + } + // A caller deadline wins over the default. + c.SetTimeout(0) + if c.timeout != DefaultTimeout { + t.Errorf("SetTimeout(0) = %v, want default", c.timeout) + } + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if _, err := c.Health(ctx); !errors.Is(err, ErrUnavailable) { + t.Errorf("caller deadline = %v", err) + } + // nil context is tolerated. + if _, err := c.Health(nil); !errors.Is(err, ErrUnavailable) { //nolint:staticcheck // deliberate nil + t.Errorf("nil ctx = %v", err) + } +} + +func TestNewClientAndSetHTTPClient(t *testing.T) { + if _, err := NewClient("", ""); err == nil { + t.Error("empty address accepted") + } + c, err := NewClient("unix:///tmp/x.sock", "") + if err != nil { + t.Fatal(err) + } + if c.Address().Network != "unix" { + t.Errorf("address = %+v", c.Address()) + } + tr := c.http.Transport + c.SetHTTPClient(&http.Client{}) + if c.http.Transport != tr { + t.Error("SetHTTPClient dropped the socket transport") + } + own := &http.Transport{} + c.SetHTTPClient(&http.Client{Transport: own}) + if c.http.Transport != own { + t.Error("SetHTTPClient overrode the caller's transport") + } +} + +func TestListenRefusals(t *testing.T) { + dir := shortSocketDir(t) + // TCP without a token. + cfg := ListenConfig{Socket: filepath.Join(dir, "a.sock"), Listen: "127.0.0.1:0"} + if _, err := Listen(cfg); err == nil || !strings.Contains(err.Error(), "bearer token") { + t.Errorf("tcp without token = %v", err) + } + // TCP off loopback, even with a token: plain http would carry the + // token and minted installation tokens in cleartext. + for _, addr := range []string{"0.0.0.0:0", "[::]:0", "10.0.0.1:0", "example.invalid:0"} { + cfg := ListenConfig{Socket: filepath.Join(dir, "c.sock"), Listen: addr, Token: "t"} + if _, err := Listen(cfg); err == nil || !strings.Contains(err.Error(), "loopback") { + t.Errorf("tcp off loopback %s = %v", addr, err) + } + } + // TCP with a token binds both. + ls, err := Listen(ListenConfig{Socket: filepath.Join(dir, "b.sock"), Listen: "127.0.0.1:0", Token: "t"}) + if err != nil { + t.Fatal(err) + } + if addrs := ls.Addrs(); len(addrs) != 2 || !strings.HasPrefix(addrs[0], "unix://") || !strings.HasPrefix(addrs[1], "http://") { + t.Errorf("addrs = %v", addrs) + } + fi, err := os.Stat(filepath.Join(dir, "b.sock")) + if err != nil || fi.Mode().Perm() != SocketMode { + t.Errorf("socket mode = %v, %v", fi, err) + } + // A second listener on a live socket is refused. + if _, err := Listen(ListenConfig{Socket: filepath.Join(dir, "b.sock")}); err == nil || !strings.Contains(err.Error(), "already listening") { + t.Errorf("double listen = %v", err) + } + ls.Close() + // A stale socket file is replaced. + if err := os.WriteFile(filepath.Join(dir, "b.sock"), nil, 0o600); err != nil { + t.Fatal(err) + } + ls2, err := Listen(ListenConfig{Socket: filepath.Join(dir, "b.sock")}) + if err != nil { + t.Fatalf("stale socket not replaced: %v", err) + } + ls2.Close() + // A directory readable by others is refused. + loose := filepath.Join(dir, "loose") + if err := os.Mkdir(loose, 0o755); err != nil { + t.Fatal(err) + } + if _, err := Listen(ListenConfig{Socket: filepath.Join(loose, "c.sock")}); err == nil || !strings.Contains(err.Error(), "readable by others") { + t.Errorf("loose dir = %v", err) + } + // A symlinked directory is refused. + link := filepath.Join(dir, "link") + if err := os.Symlink(dir, link); err != nil { + t.Fatal(err) + } + if _, err := Listen(ListenConfig{Socket: filepath.Join(link, "d.sock")}); err == nil || !strings.Contains(err.Error(), "symlink") { + t.Errorf("symlink dir = %v", err) + } + // A file where the directory should be. + file := filepath.Join(dir, "file") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := Listen(ListenConfig{Socket: filepath.Join(file, "e.sock")}); err == nil { + t.Error("file as dir accepted") + } + if err := ownedPrivateDir(file); err == nil || !strings.Contains(err.Error(), "not a directory") { + t.Errorf("ownedPrivateDir(file) = %v", err) + } + if err := ownedPrivateDir(filepath.Join(dir, "missing")); err == nil { + t.Error("missing dir accepted") + } + // Default socket path. + t.Setenv(SocketEnvVar, "") + t.Setenv("XDG_RUNTIME_DIR", "/run/user/7") + if got := DefaultSocket(); got != "/run/user/7/crux/hive.sock" { + t.Errorf("DefaultSocket = %s", got) + } + t.Setenv("XDG_RUNTIME_DIR", "") + if got := DefaultSocket(); !strings.HasSuffix(got, fmt.Sprintf("crux-%d/hive.sock", os.Getuid())) { + t.Errorf("DefaultSocket fallback = %s", got) + } + t.Setenv(SocketEnvVar, "/x/y.sock") + if got := DefaultSocket(); got != "/x/y.sock" { + t.Errorf("DefaultSocket env = %s", got) + } + // Validate defaults the socket. + var empty ListenConfig + if err := empty.Validate(); err != nil || empty.Socket != "/x/y.sock" { + t.Errorf("Validate = %v, %+v", err, empty) + } +} + +func TestServeReportsListenerError(t *testing.T) { + // Closing a listener under Serve makes it return the error. + dir := shortSocketDir(t) + sock := filepath.Join(dir, "s.sock") + ls, err := Listen(ListenConfig{Socket: sock}) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- NewServer(&fakeCaps{}, "").Serve(context.Background(), ls) }() + c, err := NewClient("unix://"+sock, "") + if err != nil { + t.Fatal(err) + } + waitReady(t, c) // Serve is accepting before the listener is pulled. + _ = ls.Unix.Close() + select { + case err := <-done: + if err == nil { + t.Error("Serve returned nil after its listener died") + } + case <-time.After(5 * time.Second): + t.Fatal("Serve did not return") + } +} + +func TestNotConfiguredSentinelStringAndCodes(t *testing.T) { + if ConditionClassUnavailable != "hive_callback_unavailable" { + t.Error("condition class is a wire name; do not change it") + } + for _, p := range []string{PathTokens, PathGitHubREST, PathGitHubRateLimit, PathUsage, PathScanCanary, PathScanText, PathConditions, PathLinearCredential, PathPulls, PathHealth} { + if !strings.HasPrefix(p, "/"+Version+"/") { + t.Errorf("route %s is not under /%s", p, Version) + } + } +} + +// bareCaps has no Configured method: the server assumes every capability +// except the optional-interface ones (policy, inference routes, effects). +type bareCaps struct{ Capabilities } + +func TestCapabilitiesWithoutConfiguredHaveAll(t *testing.T) { + c, _ := serveUnix(t, NewServer(bareCaps{&fakeCaps{}}, ""), "") + h, err := c.Health(context.Background()) + if err != nil || len(h.Capabilities) != len(allCapabilities)-5 { + t.Errorf("health = %+v, %v", h, err) + } + for _, name := range h.Capabilities { + if name == CapabilityPolicy || name == CapabilityInferenceRoutes || name == CapabilityEffects { + t.Errorf("health lists %s for a Capabilities without the optional interface", name) + } + } + ctx := context.Background() + if _, err := c.Policy(ctx); !errors.Is(err, ErrNotConfigured) { + t.Errorf("Policy without PolicySource = %v, want ErrNotConfigured", err) + } + if err := c.SetInferenceRoute(ctx, "a", InferenceRoute{Backend: "litellm"}); !errors.Is(err, ErrNotConfigured) { + t.Errorf("SetInferenceRoute without InferenceRouter = %v, want ErrNotConfigured", err) + } + if _, err := c.BeginEffect(ctx, EffectClaim{Repo: "o/r", Kind: "k", Target: "t"}); !errors.Is(err, ErrNotConfigured) { + t.Errorf("BeginEffect without EffectBoundary = %v, want ErrNotConfigured", err) + } +} + +// The inference route routes carry the per-agent route to hive's proxy and +// validate the agent and backend. +func TestInferenceRouteRoutes(t *testing.T) { + caps := &fakeCaps{} + c, _ := serveUnix(t, NewServer(caps, ""), "") + ctx := context.Background() + if err := c.SetInferenceRoute(ctx, "agent one", InferenceRoute{Backend: "litellm", Model: "gpt"}); err != nil { + t.Fatal(err) + } + caps.mu.Lock() + got := caps.routes["agent one"] + caps.mu.Unlock() + if got != (InferenceRoute{Backend: "litellm", Model: "gpt"}) { + t.Fatalf("route = %+v", got) + } + if err := c.SetInferenceRoute(ctx, "a", InferenceRoute{}); err == nil || errors.Is(err, ErrNotConfigured) { + t.Fatalf("SetInferenceRoute without backend = %v, want bad request", err) + } + if err := c.ClearInferenceRoute(ctx, "agent one"); err != nil { + t.Fatal(err) + } + caps.mu.Lock() + _, still := caps.routes["agent one"] + caps.mu.Unlock() + if still { + t.Fatal("route not cleared") + } + caps.fail = errors.New("proxy down") + if err := c.ClearInferenceRoute(ctx, "a"); !errors.Is(err, ErrUnavailable) { + t.Fatalf("failing clear = %v, want ErrUnavailable (502 upstream)", err) + } +} + +// The effect routes begin and end a journaled mutation; a denial is +// ErrEffectDenied, an unknown id ErrEffectUnknown. +func TestEffectRoutes(t *testing.T) { + caps := &fakeCaps{} + c, _ := serveUnix(t, NewServer(caps, ""), "") + ctx := context.Background() + claim := EffectClaim{Repo: "o/r", Kind: "branch_push", Target: "feat", Actor: "agent", Inputs: map[string]string{"commit": "abc"}} + begun, err := c.BeginEffect(ctx, claim) + if err != nil || begun.ID != "e1" { + t.Fatalf("BeginEffect = %+v, %v", begun, err) + } + caps.mu.Lock() + gotClaim := caps.effects[0] + caps.mu.Unlock() + if gotClaim.Repo != claim.Repo || gotClaim.Kind != claim.Kind || gotClaim.Target != claim.Target || gotClaim.Actor != claim.Actor || gotClaim.Inputs["commit"] != "abc" { + t.Fatalf("claim = %+v", gotClaim) + } + ended, err := c.EndEffect(ctx, "e1", EndEffectRequest{Provenance: "abc", Error: "push failed"}) + if err != nil || ended.Provenance != "abc" || ended.Error != "push failed" { + t.Fatalf("EndEffect = %+v, %v", ended, err) + } + if _, err := c.EndEffect(ctx, "nope", EndEffectRequest{}); !errors.Is(err, ErrEffectUnknown) { + t.Fatalf("EndEffect(unknown) = %v, want ErrEffectUnknown", err) + } + if _, err := c.BeginEffect(ctx, EffectClaim{Repo: "o/r"}); err == nil || errors.Is(err, ErrNotConfigured) { + t.Fatalf("BeginEffect(incomplete) = %v, want bad request", err) + } + caps.deny = true + if _, err := c.BeginEffect(ctx, claim); !errors.Is(err, ErrEffectDenied) { + t.Fatalf("denied BeginEffect = %v, want ErrEffectDenied", err) + } + if st := c.Status(); !st.Reachable { + t.Fatal("a denial is hive answering, not an outage") + } + caps.deny, caps.fail = false, errors.New("journal broken") + if _, err := c.BeginEffect(ctx, claim); err == nil || errors.Is(err, ErrEffectDenied) || !errors.Is(err, ErrUnavailable) { + t.Fatalf("failing BeginEffect = %v, want ErrUnavailable (502 upstream)", err) + } + for _, p := range []string{InferenceRoutePath("a b"), EffectEndPath("x/y"), PathEffects, PathInferenceRoutes} { + if !strings.HasPrefix(p, "/"+Version+"/") || strings.Contains(p, " ") || strings.Contains(p, "x/y") { + t.Errorf("route %s is malformed", p) + } + } +} + +// The policy routes carry the snapshot verbatim and answer the per-repo +// auto-merge question; an absent policy capability is not_configured. +func TestPolicyRoutes(t *testing.T) { + explain, owned := "on", true + gateways := []string{"litellm-eu"} + want := Policy{ + Build: &BuildInfo{Version: "v5.1.0", Commit: "abc123"}, + ExplainModeDefault: &explain, + GatewayBackends: &gateways, + RepoAutoMerge: true, + DeadSessionRecoveryOwnedElsewhere: &owned, + Sandbox: json.RawMessage(`{"enabled":true,"image":"img"}`), + } + caps := &fakeCaps{policy: want} + c, _ := serveUnix(t, NewServer(caps, ""), "") + ctx := context.Background() + got, err := c.Policy(ctx) + if err != nil { + t.Fatal(err) + } + if got.Build == nil || *got.Build != *want.Build || got.ExplainModeDefault == nil || *got.ExplainModeDefault != "on" || + got.KickLogSource != nil || got.GatewayBackends == nil || len(*got.GatewayBackends) != 1 || !got.RepoAutoMerge || + got.DeadSessionRecoveryOwnedElsewhere == nil || !*got.DeadSessionRecoveryOwnedElsewhere || string(got.Sandbox) != string(want.Sandbox) { + t.Fatalf("Policy = %+v", got) + } + if ok, err := c.RepoAutoMergeEnabled(ctx, "o/merge"); err != nil || !ok { + t.Fatalf("RepoAutoMergeEnabled(o/merge) = %v, %v", ok, err) + } + if ok, err := c.RepoAutoMergeEnabled(ctx, "o/other"); err != nil || ok { + t.Fatalf("RepoAutoMergeEnabled(o/other) = %v, %v", ok, err) + } + if _, err := c.RepoAutoMergeEnabled(ctx, ""); err == nil || errors.Is(err, ErrNotConfigured) { + t.Fatalf("RepoAutoMergeEnabled(\"\") = %v, want bad request", err) + } + + absent := &fakeCaps{absent: map[string]bool{CapabilityPolicy: true}} + c2, _ := serveUnix(t, NewServer(absent, ""), "") + if _, err := c2.Policy(ctx); !errors.Is(err, ErrNotConfigured) { + t.Fatalf("absent Policy = %v, want ErrNotConfigured", err) + } + if _, err := c2.RepoAutoMergeEnabled(ctx, "o/r"); !errors.Is(err, ErrNotConfigured) { + t.Fatalf("absent RepoAutoMergeEnabled = %v, want ErrNotConfigured", err) + } + for _, p := range []string{PathPolicy, PathRepoAutoMerge} { + if !strings.HasPrefix(p, "/"+Version+"/") { + t.Errorf("route %s is not under /%s", p, Version) + } + } +} + +func TestListenUnbindablePath(t *testing.T) { + dir := shortSocketDir(t) + long := filepath.Join(dir, strings.Repeat("x", 120)+".sock") + if _, err := Listen(ListenConfig{Socket: long}); err == nil { + t.Error("over-long socket path accepted") + } + // A directory that cannot be created (parent is a file). + file := filepath.Join(dir, "f") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := Listen(ListenConfig{Socket: filepath.Join(file, "sub", "s.sock")}); err == nil { + t.Error("uncreatable directory accepted") + } +} + +func TestAgentMintAndBobOverTheWire(t *testing.T) { + caps := &fakeCaps{bob: BobCredential{APIKey: "bob-secret", Source: "file:/run/secrets/bob"}} + c, logs := serveUnix(t, NewServer(caps, ""), "") + ctx := context.Background() + + got, err := c.MintAgentToken(ctx, AgentMintRequest{Agent: "fixer", Tier: "staff"}) + if err != nil || got.Token != "mint-fixer" || got.ExpiresAt.IsZero() { + t.Fatalf("MintAgentToken = %+v, %v", got, err) + } + if len(caps.mints) != 1 || caps.mints[0].Tier != "staff" { + t.Fatalf("mint requests = %+v", caps.mints) + } + // A configured-but-disabled mint answers an empty token without error. + if got, err := c.MintAgentToken(ctx, AgentMintRequest{Agent: "fixer", Tier: "disabled"}); err != nil || got.Token != "" { + t.Fatalf("disabled mint = %+v, %v", got, err) + } + if _, err := c.MintAgentToken(ctx, AgentMintRequest{}); err == nil || errors.Is(err, ErrNotConfigured) { + t.Fatalf("MintAgentToken without agent = %v, want bad request", err) + } + bob, err := c.BobCredential(ctx) + if err != nil || bob.APIKey != "bob-secret" || bob.Source != "file:/run/secrets/bob" { + t.Fatalf("BobCredential = %+v, %v", bob, err) + } + if s := fmt.Sprint(bob, got); strings.Contains(s, "bob-secret") || strings.Contains(s, "mint-fixer") { + t.Fatalf("secret printed: %s", s) + } + if l := logs.String(); strings.Contains(l, "bob-secret") || strings.Contains(l, "mint-fixer") { + t.Fatalf("secret logged:\n%s", l) + } + + absent := &fakeCaps{absent: map[string]bool{CapabilityAgentMint: true, CapabilityBob: true}} + c2, _ := serveUnix(t, NewServer(absent, ""), "") + if _, err := c2.MintAgentToken(ctx, AgentMintRequest{Agent: "a"}); !errors.Is(err, ErrNotConfigured) { + t.Fatalf("absent mint = %v", err) + } + if _, err := c2.BobCredential(ctx); !errors.Is(err, ErrNotConfigured) { + t.Fatalf("absent bob = %v", err) + } + // Upstream failures map to the standard sentinel. + failing := &fakeCaps{fail: errors.New("key store down")} + c3, _ := serveUnix(t, NewServer(failing, ""), "") + if _, err := c3.BobCredential(ctx); !errors.Is(err, ErrUnavailable) { + t.Fatalf("failing bob = %v", err) + } + if _, err := c3.MintAgentToken(ctx, AgentMintRequest{Agent: "a"}); !errors.Is(err, ErrUnavailable) { + t.Fatalf("failing mint = %v", err) + } +} diff --git a/src/crux/api/callback/client.go b/src/crux/api/callback/client.go new file mode 100644 index 0000000000..2bee134fb3 --- /dev/null +++ b/src/crux/api/callback/client.go @@ -0,0 +1,337 @@ +package callback + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/hivecommons/hive/crux/api" +) + +// Client calls the hive-side callback endpoint. It implements Capabilities +// over HTTP and is what `crux serve` builds in sidecar mode. It has no +// logger: nothing it does is logged here, so no token, body or bearer can +// leak through it, and its errors never quote a request or response body. +// The zero value is not usable; call NewClient. +type Client struct { + addr api.Address + token string + http *http.Client + timeout time.Duration + + mu sync.Mutex + status Status +} + +// Status is the client's view of the endpoint: whether the last call +// reached hive, and when not, since when and how many calls have failed +// closed. It is the runtime condition ConditionClassUnavailable describes. +type Status struct { + Reachable bool + // LastError is the most recent failure, "" when Reachable. + LastError string + // Since is when the current outage began (zero when Reachable). + Since time.Time + // FailedCalls counts calls that failed closed during the current outage. + FailedCalls int + // LastSuccess is when a call last reached hive (zero if never). + LastSuccess time.Time +} + +// NewClient parses server (the forms api.ParseAddress accepts) and returns a +// client that presents token as a bearer token when it is non-empty. A +// unix-socket address gets a transport that dials the socket. +func NewClient(server, token string) (*Client, error) { + addr, err := api.ParseAddress(server) + if err != nil { + return nil, err + } + c := &Client{addr: addr, token: token, http: &http.Client{}, timeout: DefaultTimeout} + c.status.Reachable = true + if addr.Network == "unix" { + path := addr.Path + c.http.Transport = &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", path) + }, + } + } + return c, nil +} + +// Address is the parsed endpoint address. +func (c *Client) Address() api.Address { return c.addr } + +// SetTimeout replaces DefaultTimeout as the per-call bound applied when the +// caller's context has no deadline. Non-positive restores the default. +func (c *Client) SetTimeout(d time.Duration) { + if d <= 0 { + d = DefaultTimeout + } + c.timeout = d +} + +// SetHTTPClient replaces the underlying http.Client, keeping the socket +// transport when the address is a unix socket and the caller's client has +// none of its own. +func (c *Client) SetHTTPClient(h *http.Client) { + if h.Transport == nil && c.http != nil { + h.Transport = c.http.Transport + } + c.http = h +} + +// Status returns the current reachability snapshot. +func (c *Client) Status() Status { + c.mu.Lock() + defer c.mu.Unlock() + return c.status +} + +// Health is GET /v1/health. +func (c *Client) Health(ctx context.Context) (Health, error) { + var out Health + err := c.do(ctx, http.MethodGet, PathHealth, nil, &out) + return out, err +} + +// MintToken is POST /v1/tokens. +func (c *Client) MintToken(ctx context.Context, req MintTokenRequest) (MintTokenResponse, error) { + var out MintTokenResponse + if err := c.do(ctx, http.MethodPost, PathTokens, req, &out); err != nil { + return MintTokenResponse{}, err + } + if out.Token == "" { + return MintTokenResponse{}, fmt.Errorf("%w: hive returned an empty token", ErrUnavailable) + } + return out, nil +} + +// RecordREST is POST /v1/github/rest. +func (c *Client) RecordREST(ctx context.Context, rec RESTRecord) error { + return c.do(ctx, http.MethodPost, PathGitHubREST, rec, nil) +} + +// RateLimit is GET /v1/github/rate-limit. +func (c *Client) RateLimit(ctx context.Context) (RateLimitSnapshot, error) { + var out RateLimitSnapshot + err := c.do(ctx, http.MethodGet, PathGitHubRateLimit, nil, &out) + return out, err +} + +// RecordUsage is POST /v1/usage. +func (c *Client) RecordUsage(ctx context.Context, rec UsageRecord) error { + return c.do(ctx, http.MethodPost, PathUsage, rec, nil) +} + +// ScanCanary is POST /v1/scan/canary. +func (c *Client) ScanCanary(ctx context.Context, req CanaryScanRequest) (CanaryScanResponse, error) { + var out CanaryScanResponse + err := c.do(ctx, http.MethodPost, PathScanCanary, req, &out) + return out, err +} + +// ScanText is POST /v1/scan/text. +func (c *Client) ScanText(ctx context.Context, req TextScanRequest) (TextScanResponse, error) { + var out TextScanResponse + err := c.do(ctx, http.MethodPost, PathScanText, req, &out) + return out, err +} + +// ReportCondition is POST /v1/conditions. +func (c *Client) ReportCondition(ctx context.Context, rep ConditionReport) error { + return c.do(ctx, http.MethodPost, PathConditions, rep, nil) +} + +// LinearCredential is GET /v1/linear/credential. +func (c *Client) LinearCredential(ctx context.Context) (LinearCredential, error) { + var out LinearCredential + err := c.do(ctx, http.MethodGet, PathLinearCredential, nil, &out) + return out, err +} + +// CreatePR is POST /v1/pulls. +func (c *Client) CreatePR(ctx context.Context, req CreatePRRequest) (CreatePRResponse, error) { + var out CreatePRResponse + err := c.do(ctx, http.MethodPost, PathPulls, req, &out) + return out, err +} + +// Policy is GET /v1/policy. +func (c *Client) Policy(ctx context.Context) (Policy, error) { + var out Policy + err := c.do(ctx, http.MethodGet, PathPolicy, nil, &out) + return out, err +} + +// RepoAutoMergeEnabled is POST /v1/policy/repo-auto-merge. +func (c *Client) RepoAutoMergeEnabled(ctx context.Context, repo string) (bool, error) { + var out RepoAutoMergeResponse + err := c.do(ctx, http.MethodPost, PathRepoAutoMerge, RepoAutoMergeRequest{Repo: repo}, &out) + return out.Enabled, err +} + +// SetInferenceRoute is PUT /v1/inference/routes/{agent}. +func (c *Client) SetInferenceRoute(ctx context.Context, agent string, route InferenceRoute) error { + return c.do(ctx, http.MethodPut, InferenceRoutePath(agent), route, nil) +} + +// ClearInferenceRoute is DELETE /v1/inference/routes/{agent}. +func (c *Client) ClearInferenceRoute(ctx context.Context, agent string) error { + return c.do(ctx, http.MethodDelete, InferenceRoutePath(agent), nil, nil) +} + +// BeginEffect is POST /v1/effects. +func (c *Client) BeginEffect(ctx context.Context, claim EffectClaim) (BeginEffectResponse, error) { + var out BeginEffectResponse + if err := c.do(ctx, http.MethodPost, PathEffects, claim, &out); err != nil { + return BeginEffectResponse{}, err + } + if out.ID == "" { + return BeginEffectResponse{}, fmt.Errorf("%w: hive returned no effect id", ErrUnavailable) + } + return out, nil +} + +// EndEffect is POST /v1/effects/{id}/end. +func (c *Client) EndEffect(ctx context.Context, id string, req EndEffectRequest) (EndEffectResponse, error) { + var out EndEffectResponse + err := c.do(ctx, http.MethodPost, EffectEndPath(id), req, &out) + return out, err +} + +var ( + _ Capabilities = (*Client)(nil) + _ PolicySource = (*Client)(nil) + _ InferenceRouter = (*Client)(nil) + _ EffectBoundary = (*Client)(nil) +) + +// do performs one call. Transport failures, timeouts and 5xx become +// ErrUnavailable; 401 becomes ErrUnauthorized; 404 CodeNotConfigured becomes +// ErrNotConfigured; other non-2xx are *api.StatusError. Every outcome +// updates Status. +func (c *Client) do(ctx context.Context, method, path string, in, out any) (err error) { + defer func() { c.observe(err) }() + if ctx == nil { + ctx = context.Background() + } + if _, has := ctx.Deadline(); !has { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, c.timeout) + defer cancel() + } + var body io.Reader + if in != nil { + data, err := json.Marshal(in) + if err != nil { + return err + } + body = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext(ctx, method, c.addr.BaseURL+path, body) + if err != nil { + return err + } + if in != nil { + req.Header.Set("Content-Type", ContentTypeJSON) + } + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("%w: %s %s: %w", ErrUnavailable, method, path, redactURLError(err)) + } + defer func() { _ = resp.Body.Close() }() + if err := c.checkStatus(resp, method, path); err != nil { + return err + } + if out == nil { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16)) + return nil + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("%w: decoding %s %s: %w", ErrUnavailable, method, path, err) + } + return nil +} + +// redactURLError strips the request URL from a transport error so a TCP +// endpoint's URL form never appears in an error a caller might log; the +// method and path are added back by the caller. +func redactURLError(err error) error { + var ue *url.Error + if errors.As(err, &ue) && ue.Err != nil { + return ue.Err + } + return err +} + +func (c *Client) checkStatus(resp *http.Response, method, path string) error { + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return nil + } + se := &api.StatusError{Status: resp.StatusCode} + data, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16)) + _ = json.Unmarshal(data, &se.Body) + switch { + case resp.StatusCode == http.StatusUnauthorized: + return fmt.Errorf("%w: %s %s", ErrUnauthorized, method, path) + case resp.StatusCode == http.StatusNotFound && se.Body.Code == CodeNotConfigured: + return fmt.Errorf("%w: %s %s", ErrNotConfigured, method, path) + case resp.StatusCode == http.StatusForbidden && se.Body.Code == CodeEffectDenied: + return fmt.Errorf("%w: %s", ErrEffectDenied, se.Body.Error) + case resp.StatusCode == http.StatusNotFound && se.Body.Code == CodeEffectUnknown: + return fmt.Errorf("%w: %s %s", ErrEffectUnknown, method, path) + case resp.StatusCode >= 500: + return fmt.Errorf("%w: %s %s: HTTP %d %s: %w", ErrUnavailable, method, path, resp.StatusCode, se.Body.Code, se) + } + return se +} + +// observe folds one call's outcome into Status. A refusal hive itself made +// (401, 404 not_configured, 4xx) still counts as reachable: hive answered. +func (c *Client) observe(err error) { + c.mu.Lock() + defer c.mu.Unlock() + now := time.Now() + if err != nil && errors.Is(err, ErrUnavailable) { + if c.status.Reachable { + c.status.Reachable = false + c.status.Since = now + c.status.FailedCalls = 0 + } + c.status.FailedCalls++ + c.status.LastError = err.Error() + return + } + c.status.Reachable = true + c.status.LastError = "" + c.status.Since = time.Time{} + c.status.FailedCalls = 0 + c.status.LastSuccess = now +} + +// MintAgentToken is POST /v1/tokens/mint. +func (c *Client) MintAgentToken(ctx context.Context, req AgentMintRequest) (AgentMintResponse, error) { + var out AgentMintResponse + err := c.do(ctx, http.MethodPost, PathAgentMint, req, &out) + return out, err +} + +// BobCredential is GET /v1/bob/credential. +func (c *Client) BobCredential(ctx context.Context) (BobCredential, error) { + var out BobCredential + err := c.do(ctx, http.MethodGet, PathBobCredential, nil, &out) + return out, err +} diff --git a/src/crux/api/callback/server.go b/src/crux/api/callback/server.go new file mode 100644 index 0000000000..0de624a61f --- /dev/null +++ b/src/crux/api/callback/server.go @@ -0,0 +1,677 @@ +package callback + +import ( + "context" + "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/hivecommons/hive/crux/api" +) + +const ( + // maxBodyBytes bounds a request body. Scan bodies are the largest; the + // proxy already caps the GitHub write bodies it scans well under this. + maxBodyBytes = 4 << 20 + // shutdownGrace is how long Serve waits for in-flight requests. + shutdownGrace = 5 * time.Second + // networkUnix tags requests that arrived on the unix socket. + networkUnix = "unix" + bearerPrefix = "Bearer " + // installationTokenTTL is the lifetime of a GitHub App installation + // token; a minter that does not know its token's expiry reports this. + installationTokenTTL = time.Hour +) + +type ctxKey int + +const networkKey ctxKey = iota + +// Server mounts a Capabilities implementation on the callback routes. Hive +// builds one over pkg/cruxhost; tests build one over a fake. Zero values are +// usable after Caps is set. +type Server struct { + // Caps answers the routes. A capability it reports as absent through + // Configured (optional) answers 404 CodeNotConfigured. + Caps Capabilities + // Token, when set, is required on every request as a bearer token. When + // empty only requests that arrived on the unix socket Serve opened are + // answered; anything mounted elsewhere is refused. + Token string + // Logger logs method, path and status only; never a body or a token. + // Nil discards. + Logger *slog.Logger +} + +// Configured is optionally implemented by a Capabilities to say which of +// its capabilities exist; the server turns an absent one into 404 +// CodeNotConfigured before dispatching and lists the present ones on +// /v1/health. A Capabilities without it is taken to have all of them. +type Configured interface { + Configured(capability string) bool +} + +// NewServer is Server{Caps: caps, Token: token}. +func NewServer(caps Capabilities, token string) *Server { + return &Server{Caps: caps, Token: token} +} + +func (s *Server) logger() *slog.Logger { + if s.Logger != nil { + return s.Logger + } + return slog.New(slog.DiscardHandler) +} + +// Handler is the routed, authenticated http.Handler. Mount it under Serve +// (which tags unix-socket requests) or, with a Token, anywhere. +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET "+PathHealth, s.handleHealth) + mux.HandleFunc("POST "+PathTokens, s.capability(CapabilityTokens, s.handleMintToken)) + mux.HandleFunc("POST "+PathGitHubREST, s.capability(CapabilityGitHubREST, s.handleRecordREST)) + mux.HandleFunc("GET "+PathGitHubRateLimit, s.capability(CapabilityGitHubREST, s.handleRateLimit)) + mux.HandleFunc("POST "+PathUsage, s.capability(CapabilityUsage, s.handleUsage)) + mux.HandleFunc("POST "+PathScanCanary, s.capability(CapabilityCanary, s.handleScanCanary)) + mux.HandleFunc("POST "+PathScanText, s.capability(CapabilityText, s.handleScanText)) + mux.HandleFunc("POST "+PathConditions, s.capability(CapabilityConditions, s.handleCondition)) + mux.HandleFunc("GET "+PathLinearCredential, s.capability(CapabilityLinear, s.handleLinear)) + mux.HandleFunc("POST "+PathPulls, s.capability(CapabilityPulls, s.handleCreatePR)) + mux.HandleFunc("GET "+PathPolicy, s.capability(CapabilityPolicy, s.handlePolicy)) + mux.HandleFunc("POST "+PathRepoAutoMerge, s.capability(CapabilityPolicy, s.handleRepoAutoMerge)) + mux.HandleFunc("PUT "+PathInferenceRoutes+"/{agent}", s.capability(CapabilityInferenceRoutes, s.handleSetInferenceRoute)) + mux.HandleFunc("DELETE "+PathInferenceRoutes+"/{agent}", s.capability(CapabilityInferenceRoutes, s.handleClearInferenceRoute)) + mux.HandleFunc("POST "+PathEffects, s.capability(CapabilityEffects, s.handleBeginEffect)) + mux.HandleFunc("POST "+PathEffects+"/{id}/end", s.capability(CapabilityEffects, s.handleEndEffect)) + mux.HandleFunc("POST "+PathAgentMint, s.capability(CapabilityAgentMint, s.handleAgentMint)) + mux.HandleFunc("GET "+PathBobCredential, s.capability(CapabilityBob, s.handleBobCredential)) + return s.authenticate(mux) +} + +// authenticate is the fail-closed gate, the same rule as crux/serve: no +// token → only tagged unix-socket requests; a token → every request must +// present it. +func (s *Server) authenticate(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + network, _ := r.Context().Value(networkKey).(string) + if s.Token == "" { + if network != networkUnix { + s.writeError(w, r, http.StatusUnauthorized, CodeUnauthorized, "this endpoint has no bearer token; only unix-socket requests are accepted") + return + } + next.ServeHTTP(w, r) + return + } + got, ok := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) + if !ok || subtle.ConstantTimeCompare([]byte(got), []byte(s.Token)) != 1 { + w.Header().Set("WWW-Authenticate", `Bearer realm="crux-hive"`) + s.writeError(w, r, http.StatusUnauthorized, CodeUnauthorized, "missing or invalid bearer token") + return + } + next.ServeHTTP(w, r) + }) +} + +func (s *Server) configured(capability string) bool { + if s.Caps == nil { + return false + } + switch capability { + case CapabilityPolicy: + if _, ok := s.Caps.(PolicySource); !ok { + return false + } + case CapabilityInferenceRoutes: + if _, ok := s.Caps.(InferenceRouter); !ok { + return false + } + case CapabilityEffects: + if _, ok := s.Caps.(EffectBoundary); !ok { + return false + } + case CapabilityAgentMint: + if _, ok := s.Caps.(AgentMinter); !ok { + return false + } + case CapabilityBob: + if _, ok := s.Caps.(BobCredentials); !ok { + return false + } + } + if c, ok := s.Caps.(Configured); ok { + return c.Configured(capability) + } + return true +} + +func (s *Server) capability(name string, h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !s.configured(name) { + s.writeError(w, r, http.StatusNotFound, CodeNotConfigured, fmt.Sprintf("this hive has no %s capability", name)) + return + } + h(w, r) + } +} + +// allCapabilities is the Health order. +var allCapabilities = []string{ + CapabilityTokens, CapabilityGitHubREST, CapabilityUsage, CapabilityCanary, + CapabilityText, CapabilityConditions, CapabilityLinear, CapabilityPulls, + CapabilityPolicy, CapabilityInferenceRoutes, CapabilityEffects, + CapabilityAgentMint, CapabilityBob, +} + +func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { + h := Health{Version: Version, Capabilities: []string{}} + for _, c := range allCapabilities { + if s.configured(c) { + h.Capabilities = append(h.Capabilities, c) + } + } + s.writeJSON(w, r, http.StatusOK, h) +} + +func (s *Server) handleMintToken(w http.ResponseWriter, r *http.Request) { + var req MintTokenRequest + if !s.readJSON(w, r, &req) { + return + } + if req.TTLSeconds < 0 { + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "ttl_seconds must not be negative") + return + } + resp, err := s.Caps.MintToken(r.Context(), req) + if err != nil { + s.writeCapabilityError(w, r, "minting token", err) + return + } + if resp.Token == "" { + s.writeError(w, r, http.StatusBadGateway, CodeUpstream, "minting token: empty token") + return + } + if resp.ExpiresAt.IsZero() { + resp.ExpiresAt = time.Now().Add(installationTokenTTL) + } + s.writeJSON(w, r, http.StatusOK, resp) +} + +func (s *Server) handleRecordREST(w http.ResponseWriter, r *http.Request) { + var rec RESTRecord + if !s.readJSON(w, r, &rec) { + return + } + if rec.Method == "" || rec.Path == "" { + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "method and path are required") + return + } + if err := s.Caps.RecordREST(r.Context(), rec); err != nil { + s.writeCapabilityError(w, r, "recording REST request", err) + return + } + s.writeNoContent(w, r) +} + +func (s *Server) handleRateLimit(w http.ResponseWriter, r *http.Request) { + snap, err := s.Caps.RateLimit(r.Context()) + if err != nil { + s.writeCapabilityError(w, r, "reading rate limit", err) + return + } + s.writeJSON(w, r, http.StatusOK, snap) +} + +func (s *Server) handleUsage(w http.ResponseWriter, r *http.Request) { + var rec UsageRecord + if !s.readJSON(w, r, &rec) { + return + } + switch rec.Kind { + case UsageKindGateway, UsageKindCopilot, UsageKindCopilotCache: + default: + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "kind must be gateway, copilot or copilot_cache") + return + } + if err := s.Caps.RecordUsage(r.Context(), rec); err != nil { + s.writeCapabilityError(w, r, "recording usage", err) + return + } + s.writeNoContent(w, r) +} + +func (s *Server) handleScanCanary(w http.ResponseWriter, r *http.Request) { + var req CanaryScanRequest + if !s.readJSON(w, r, &req) { + return + } + resp, err := s.Caps.ScanCanary(r.Context(), req) + if err != nil { + s.writeCapabilityError(w, r, "scanning for canaries", err) + return + } + s.writeJSON(w, r, http.StatusOK, resp) +} + +func (s *Server) handleScanText(w http.ResponseWriter, r *http.Request) { + var req TextScanRequest + if !s.readJSON(w, r, &req) { + return + } + if req.Direction != ScanInput && req.Direction != ScanOutput { + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "direction must be input or output") + return + } + resp, err := s.Caps.ScanText(r.Context(), req) + if err != nil { + s.writeCapabilityError(w, r, "scanning text", err) + return + } + s.writeJSON(w, r, http.StatusOK, resp) +} + +func (s *Server) handleCondition(w http.ResponseWriter, r *http.Request) { + var rep ConditionReport + if !s.readJSON(w, r, &rep) { + return + } + if rep.Action != ConditionObserve && rep.Action != ConditionClear { + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "action must be observe or clear") + return + } + if rep.Class == "" { + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "class is required") + return + } + if err := s.Caps.ReportCondition(r.Context(), rep); err != nil { + s.writeCapabilityError(w, r, "reporting condition", err) + return + } + s.writeNoContent(w, r) +} + +func (s *Server) handleLinear(w http.ResponseWriter, r *http.Request) { + cred, err := s.Caps.LinearCredential(r.Context()) + if err != nil { + s.writeCapabilityError(w, r, "resolving Linear credential", err) + return + } + s.writeJSON(w, r, http.StatusOK, cred) +} + +func (s *Server) handleCreatePR(w http.ResponseWriter, r *http.Request) { + var req CreatePRRequest + if !s.readJSON(w, r, &req) { + return + } + if req.Repo == "" || req.Head == "" || req.Base == "" || req.Title == "" { + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "repo, head, base and title are required") + return + } + resp, err := s.Caps.CreatePR(r.Context(), req) + if err != nil { + s.writeCapabilityError(w, r, "creating pull request", err) + return + } + s.writeJSON(w, r, http.StatusOK, resp) +} + +func (s *Server) handlePolicy(w http.ResponseWriter, r *http.Request) { + p, err := s.Caps.(PolicySource).Policy(r.Context()) + if err != nil { + s.writeCapabilityError(w, r, "reading policy", err) + return + } + s.writeJSON(w, r, http.StatusOK, p) +} + +func (s *Server) handleRepoAutoMerge(w http.ResponseWriter, r *http.Request) { + var req RepoAutoMergeRequest + if !s.readJSON(w, r, &req) { + return + } + if req.Repo == "" { + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "repo is required") + return + } + enabled, err := s.Caps.(PolicySource).RepoAutoMergeEnabled(r.Context(), req.Repo) + if err != nil { + s.writeCapabilityError(w, r, "resolving repo auto-merge", err) + return + } + s.writeJSON(w, r, http.StatusOK, RepoAutoMergeResponse{Enabled: enabled}) +} + +func (s *Server) handleSetInferenceRoute(w http.ResponseWriter, r *http.Request) { + agent := r.PathValue("agent") + var route InferenceRoute + if !s.readJSON(w, r, &route) { + return + } + if agent == "" || route.Backend == "" { + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "agent and backend are required") + return + } + if err := s.Caps.(InferenceRouter).SetInferenceRoute(r.Context(), agent, route); err != nil { + s.writeCapabilityError(w, r, "setting inference route", err) + return + } + s.writeNoContent(w, r) +} + +func (s *Server) handleClearInferenceRoute(w http.ResponseWriter, r *http.Request) { + agent := r.PathValue("agent") + if agent == "" { + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "agent is required") + return + } + if err := s.Caps.(InferenceRouter).ClearInferenceRoute(r.Context(), agent); err != nil { + s.writeCapabilityError(w, r, "clearing inference route", err) + return + } + s.writeNoContent(w, r) +} + +func (s *Server) handleBeginEffect(w http.ResponseWriter, r *http.Request) { + var claim EffectClaim + if !s.readJSON(w, r, &claim) { + return + } + if claim.Repo == "" || claim.Kind == "" || claim.Target == "" { + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "repo, kind and target are required") + return + } + resp, err := s.Caps.(EffectBoundary).BeginEffect(r.Context(), claim) + if err != nil { + if errors.Is(err, ErrEffectDenied) { + s.writeError(w, r, http.StatusForbidden, CodeEffectDenied, "beginning effect: "+err.Error()) + return + } + s.writeCapabilityError(w, r, "beginning effect", err) + return + } + if resp.ID == "" { + s.writeError(w, r, http.StatusBadGateway, CodeUpstream, "beginning effect: boundary returned no id") + return + } + s.writeJSON(w, r, http.StatusOK, resp) +} + +func (s *Server) handleEndEffect(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + var req EndEffectRequest + if !s.readJSON(w, r, &req) { + return + } + resp, err := s.Caps.(EffectBoundary).EndEffect(r.Context(), id, req) + if err != nil { + if errors.Is(err, ErrEffectUnknown) { + s.writeError(w, r, http.StatusNotFound, CodeEffectUnknown, "ending effect: "+err.Error()) + return + } + s.writeCapabilityError(w, r, "ending effect", err) + return + } + s.writeJSON(w, r, http.StatusOK, resp) +} + +// writeCapabilityError maps a Capabilities error: ErrNotConfigured → 404, +// anything else → 502 CodeUpstream with the message. The message is the +// implementation's; hive-side code is responsible for keeping secrets out +// of the errors it returns, as it already is for its logs. +func (s *Server) writeCapabilityError(w http.ResponseWriter, r *http.Request, verb string, err error) { + if errors.Is(err, ErrNotConfigured) { + s.writeError(w, r, http.StatusNotFound, CodeNotConfigured, verb+": "+err.Error()) + return + } + s.writeError(w, r, http.StatusBadGateway, CodeUpstream, verb+": "+err.Error()) +} + +func (s *Server) readJSON(w http.ResponseWriter, r *http.Request, v any) bool { + dec := json.NewDecoder(io.LimitReader(r.Body, maxBodyBytes)) + if err := dec.Decode(v); err != nil { + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "invalid JSON body") + return false + } + return true +} + +func (s *Server) writeJSON(w http.ResponseWriter, r *http.Request, status int, v any) { + w.Header().Set("Content-Type", ContentTypeJSON) + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) + s.logger().Debug("hive callback", "method", r.Method, "path", r.URL.Path, "status", status) +} + +func (s *Server) writeNoContent(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + s.logger().Debug("hive callback", "method", r.Method, "path", r.URL.Path, "status", http.StatusNoContent) +} + +func (s *Server) writeError(w http.ResponseWriter, r *http.Request, status int, code, msg string) { + w.Header().Set("Content-Type", ContentTypeJSON) + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(api.Error{Error: msg, Code: code}) + s.logger().Warn("hive callback refused", "method", r.Method, "path", r.URL.Path, "status", status, "code", code) +} + +// ListenConfig is what Serve binds. Zero values mean "unix socket at +// DefaultSocket, no TCP". +type ListenConfig struct { + // Socket is the unix socket path; empty selects DefaultSocket(). + Socket string + // Listen is an optional loopback TCP host:port; it requires Token. + Listen string + // Token is the bearer token Serve will require. It is here only so + // Validate can refuse TCP without one or off loopback; the Server + // carries it too. + Token string +} + +// Validate fills defaults and applies the fail-closed rules shared with +// crux/config.ServeConfig.Validate: no TCP without a token, and no TCP off +// loopback. The listener is plain http, so a routable bind would carry the +// bearer token and every installation token PathTokens mints across the +// network in cleartext. +func (c *ListenConfig) Validate() error { + if c.Socket == "" { + c.Socket = DefaultSocket() + } + if c.Listen != "" && c.Token == "" { + return fmt.Errorf("a TCP callback listener on %s needs a bearer token (%s); refusing to serve hive capabilities unauthenticated", c.Listen, TokenEnvVar) + } + if c.Listen != "" && !api.LoopbackHost(c.Listen) { + return fmt.Errorf("listen %s: a TCP callback listener must bind a loopback address (127.0.0.1, [::1] or localhost); the listener is plain http and the bearer token and minted installation tokens would cross the network in cleartext — put a TLS-terminating proxy in front for remote access", c.Listen) + } + return nil +} + +// Listeners is what Listen opened. +type Listeners struct { + Unix net.Listener + TCP net.Listener + path string +} + +// Addrs lists the bound addresses for logging. +func (l *Listeners) Addrs() []string { + var out []string + if l.Unix != nil { + out = append(out, "unix://"+l.path) + } + if l.TCP != nil { + out = append(out, "http://"+l.TCP.Addr().String()) + } + return out +} + +// Close closes every listener and removes the socket file. +func (l *Listeners) Close() { + if l.Unix != nil { + _ = l.Unix.Close() + _ = os.Remove(l.path) + } + if l.TCP != nil { + _ = l.TCP.Close() + } +} + +// Listen validates cfg and binds the unix socket (directory created 0700 if +// missing and checked to be this uid's private directory, a stale socket +// replaced, the socket 0600 before it is reachable at its path) and the +// optional TCP address. The rule is crux/serve.Listen's. +func Listen(cfg ListenConfig) (*Listeners, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + dir := filepath.Dir(cfg.Socket) + if err := os.MkdirAll(dir, SocketDirMode); err != nil { + return nil, fmt.Errorf("socket directory: %w", err) + } + if err := ownedPrivateDir(dir); err != nil { + return nil, fmt.Errorf("socket directory %s: %w", dir, err) + } + if conn, err := net.DialTimeout("unix", cfg.Socket, time.Second); err == nil { + _ = conn.Close() + return nil, fmt.Errorf("socket %s: another callback endpoint is already listening", cfg.Socket) + } + tmp := fmt.Sprintf("%s.%d.tmp", cfg.Socket, os.Getpid()) + _ = os.Remove(tmp) + ul, err := net.Listen("unix", tmp) + if err != nil { + return nil, fmt.Errorf("socket %s: %w", cfg.Socket, err) + } + if err := os.Chmod(tmp, SocketMode); err != nil { + _ = ul.Close() + _ = os.Remove(tmp) + return nil, fmt.Errorf("socket %s: chmod: %w", cfg.Socket, err) + } + if err := os.Rename(tmp, cfg.Socket); err != nil { + _ = ul.Close() + _ = os.Remove(tmp) + return nil, fmt.Errorf("socket %s: %w", cfg.Socket, err) + } + ls := &Listeners{Unix: ul, path: cfg.Socket} + if cfg.Listen != "" { + tl, err := net.Listen("tcp", cfg.Listen) + if err != nil { + ls.Close() + return nil, fmt.Errorf("listen %s: %w", cfg.Listen, err) + } + ls.TCP = tl + } + return ls, nil +} + +// ownedPrivateDir refuses a socket directory this user does not own +// outright: a real directory (not a symlink), owned by this uid, with no +// group or other permission bits. +func ownedPrivateDir(dir string) error { + fi, err := os.Lstat(dir) + if err != nil { + return err + } + if fi.Mode()&os.ModeSymlink != 0 { + return errors.New("is a symlink") + } + if !fi.IsDir() { + return errors.New("is not a directory") + } + if perm := fi.Mode().Perm(); perm&0o077 != 0 { + return fmt.Errorf("mode %04o is readable by others; want %04o", perm, SocketDirMode) + } + if st, ok := fi.Sys().(*syscall.Stat_t); ok && int(st.Uid) != os.Getuid() { + return fmt.Errorf("owned by uid %d, not this user (uid %d)", st.Uid, os.Getuid()) + } + return nil +} + +// Serve runs the endpoint on ls until ctx ends, then shuts down and closes +// ls. Requests on the unix listener are tagged so a token-less server +// accepts them. +func (s *Server) Serve(ctx context.Context, ls *Listeners) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + handler := s.Handler() + errc := make(chan error, 2) + var servers []*http.Server + serveOn := func(network string, l net.Listener) { + srv := &http.Server{ + Handler: handler, + ReadHeaderTimeout: 10 * time.Second, + BaseContext: func(net.Listener) context.Context { + return context.WithValue(ctx, networkKey, network) + }, + } + servers = append(servers, srv) + go func() { + if err := srv.Serve(l); err != nil && !errors.Is(err, http.ErrServerClosed) { + errc <- fmt.Errorf("%s: %w", network, err) + return + } + errc <- nil + }() + } + if ls.Unix != nil { + serveOn(networkUnix, ls.Unix) + } + if ls.TCP != nil { + serveOn("tcp", ls.TCP) + } + var err error + select { + case <-ctx.Done(): + case err = <-errc: + cancel() + } + for _, srv := range servers { + sctx, scancel := context.WithTimeout(context.Background(), shutdownGrace) + _ = srv.Shutdown(sctx) + scancel() + } + ls.Close() + return err +} + +// WithUnixNetwork tags ctx as a unix-socket request context, for a caller +// that mounts Handler on its own listener (an httptest server over a unix +// socket, or a hive that already owns an http.Server) and wants the +// token-less rule to admit it. Use it as the http.Server's BaseContext. +func WithUnixNetwork(ctx context.Context) context.Context { + return context.WithValue(ctx, networkKey, networkUnix) +} + +func (s *Server) handleAgentMint(w http.ResponseWriter, r *http.Request) { + var req AgentMintRequest + if !s.readJSON(w, r, &req) { + return + } + if req.Agent == "" { + s.writeError(w, r, http.StatusBadRequest, CodeBadRequest, "agent is required") + return + } + resp, err := s.Caps.(AgentMinter).MintAgentToken(r.Context(), req) + if err != nil { + s.writeCapabilityError(w, r, "minting agent token", err) + return + } + s.writeJSON(w, r, http.StatusOK, resp) +} + +func (s *Server) handleBobCredential(w http.ResponseWriter, r *http.Request) { + cred, err := s.Caps.(BobCredentials).BobCredential(r.Context()) + if err != nil { + s.writeCapabilityError(w, r, "reading bob credential", err) + return + } + s.writeJSON(w, r, http.StatusOK, cred) +} diff --git a/src/crux/api/client.go b/src/crux/api/client.go new file mode 100644 index 0000000000..202555e5ab --- /dev/null +++ b/src/crux/api/client.go @@ -0,0 +1,367 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strconv" + "sync" + "time" +) + +// Client drives a `crux serve` over the API. It is the Go form of the curl +// calls in src/docs/crux-serve.md and what the `crux` CLI uses in remote +// mode (--server / CRUX_SERVER). The zero value is not usable; call +// NewClient. +type Client struct { + addr Address + token string + http *http.Client + + mu sync.Mutex + status Status +} + +// ErrUnreachable marks a call that never got an answer from the server: +// the dial or the request failed at the transport. A host distinguishes +// it from a route that answered with an error (*StatusError): the runtime +// is gone, not disagreeing. +var ErrUnreachable = errors.New("crux runtime unreachable") + +// Status is the client's view of the server's reachability, updated by +// every call and stream. +type Status struct { + Reachable bool + // LastError is the most recent transport failure, "" when Reachable. + LastError string + // Since is when the current outage began (zero when Reachable). + Since time.Time + // FailedCalls counts calls that failed at the transport during the + // current outage. + FailedCalls int + // LastSuccess is when the server last answered (zero if never). + LastSuccess time.Time +} + +// Status returns the reachability as of the last call. A client that has +// never called reports Reachable (nothing is known against it). +func (c *Client) Status() Status { + c.mu.Lock() + defer c.mu.Unlock() + return c.status +} + +// observe records the outcome of one transport attempt. +func (c *Client) observe(err error) { + c.mu.Lock() + defer c.mu.Unlock() + now := time.Now() + if err != nil { + if c.status.Reachable { + c.status.Since = now + c.status.FailedCalls = 0 + } + c.status.Reachable = false + c.status.FailedCalls++ + c.status.LastError = err.Error() + return + } + c.status = Status{Reachable: true, LastSuccess: now} +} + +// NewClient parses server (see ParseAddress) and returns a client that +// presents token as a bearer token when it is non-empty. A unix-socket +// address gets a transport that dials the socket; TCP uses the default. +func NewClient(server, token string) (*Client, error) { + addr, err := ParseAddress(server) + if err != nil { + return nil, err + } + c := &Client{addr: addr, token: token, http: &http.Client{}, status: Status{Reachable: true}} + if addr.Network == "unix" { + path := addr.Path + c.http.Transport = &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", path) + }, + } + } + return c, nil +} + +// Address is the parsed server address the client was built with. +func (c *Client) Address() Address { return c.addr } + +// SetHTTPClient replaces the underlying http.Client, keeping the socket +// transport when the address is a unix socket and the caller's client has +// none of its own. +func (c *Client) SetHTTPClient(h *http.Client) { + if h.Transport == nil && c.http != nil { + h.Transport = c.http.Transport + } + c.http = h +} + +// StatusError is a non-2xx response: the HTTP status and the server's +// api.Error body when it sent one. +type StatusError struct { + Status int + Body Error +} + +func (e *StatusError) Error() string { + if e.Body.Error != "" { + return e.Body.Error + } + return fmt.Sprintf("crux server returned HTTP %d", e.Status) +} + +// IsNotFound reports whether err is a 404 from the server. +func IsNotFound(err error) bool { + var se *StatusError + return errors.As(err, &se) && se.Status == http.StatusNotFound +} + +// IsUnauthorized reports whether err is a 401 from the server. +func IsUnauthorized(err error) bool { + var se *StatusError + return errors.As(err, &se) && se.Status == http.StatusUnauthorized +} + +// Health is GET /v1/health. +func (c *Client) Health(ctx context.Context) (Health, error) { + var out Health + err := c.do(ctx, http.MethodGet, PathHealth, nil, &out) + return out, err +} + +// List is GET /v1/agents. +func (c *Client) List(ctx context.Context) ([]Process, error) { + var out ProcessList + if err := c.do(ctx, http.MethodGet, PathAgents, nil, &out); err != nil { + return nil, err + } + return out.Processes, nil +} + +// Get is GET /v1/agents/{name}. +func (c *Client) Get(ctx context.Context, name string) (Process, error) { + var out Process + err := c.do(ctx, http.MethodGet, AgentPath(url.PathEscape(name)), nil, &out) + return out, err +} + +// Run is POST /v1/agents: register the definition (when given) and start. +func (c *Client) Run(ctx context.Context, req RunRequest) (AgentResponse, error) { + var out AgentResponse + err := c.do(ctx, http.MethodPost, PathAgents, req, &out) + return out, err +} + +// Logs is GET /v1/agents/{name}/logs for one source; lines <= 0 means the +// server default. +func (c *Client) Logs(ctx context.Context, name, source string, lines int) (Logs, error) { + q := url.Values{} + if source != "" { + q.Set(QuerySource, source) + } + if lines > 0 { + q.Set(QueryLines, strconv.Itoa(lines)) + } + var out Logs + err := c.do(ctx, http.MethodGet, LogsPath(url.PathEscape(name))+"?"+q.Encode(), nil, &out) + return out, err +} + +// FollowLogs is GET /v1/agents/{name}/logs?follow=1: fn is called with +// each chunk of new lines until ctx ends, the server closes the stream, or +// fn returns an error (which is returned). A terminal api.EventState record +// (the server lost the agent's output) is returned as an error. +func (c *Client) FollowLogs(ctx context.Context, name string, fn func(LogChunk) error) error { + return c.stream(ctx, LogsPath(url.PathEscape(name))+"?"+QueryFollow+"=1", func(typ string, data []byte) error { + switch typ { + case EventLog: + var chunk LogChunk + if err := json.Unmarshal(data, &chunk); err != nil { + return fmt.Errorf("log stream: %w", err) + } + return fn(chunk) + case EventState: + var ev Event + _ = json.Unmarshal(data, &ev) + return fmt.Errorf("log stream ended: %s", ev.Detail) + } + return nil + }) +} + +// Kick is POST /v1/agents/{name}/kick. +func (c *Client) Kick(ctx context.Context, name, message string) (AgentResponse, error) { + var out AgentResponse + err := c.do(ctx, http.MethodPost, KickPath(url.PathEscape(name)), KickRequest{Message: message}, &out) + return out, err +} + +// Stop is POST /v1/agents/{name}/stop. +func (c *Client) Stop(ctx context.Context, name string) (AgentResponse, error) { + var out AgentResponse + err := c.do(ctx, http.MethodPost, StopPath(url.PathEscape(name)), nil, &out) + return out, err +} + +// Events is GET /v1/events: fn is called with each runtime event until ctx +// ends, the server closes the stream, or fn returns an error. +func (c *Client) Events(ctx context.Context, fn func(Event) error) error { + return c.stream(ctx, PathEvents, func(typ string, data []byte) error { + var ev Event + if err := json.Unmarshal(data, &ev); err != nil { + return fmt.Errorf("event stream: %w", err) + } + if ev.Type == "" { + ev.Type = typ + } + return fn(ev) + }) +} + +// Do sends one JSON request to a route and decodes the JSON response into +// out (when non-nil). Non-2xx is a *StatusError. It is the generic call the +// typed crux.Runtime client (crux/api/client) builds every method on; the +// CLI-shaped methods above are conveniences over it. +func (c *Client) Do(ctx context.Context, method, path string, in, out any) error { + return c.do(ctx, method, path, in, out) +} + +// do sends one JSON request and decodes a JSON response into out (when +// non-nil). Non-2xx is a *StatusError. +func (c *Client) do(ctx context.Context, method, path string, in, out any) error { + var body io.Reader + if in != nil { + data, err := json.Marshal(in) + if err != nil { + return err + } + body = bytes.NewReader(data) + } + req, err := c.request(ctx, method, path, body) + if err != nil { + return err + } + if in != nil { + req.Header.Set("Content-Type", ContentTypeJSON) + } + resp, err := c.http.Do(req) + if err != nil { + return c.unreachable(ctx, err) + } + c.observe(nil) + defer func() { _ = resp.Body.Close() }() + if err := checkStatus(resp); err != nil { + return err + } + if out == nil { + return nil + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("crux server %s: decoding %s %s: %w", c.addr.target(), method, path, err) + } + return nil +} + +// stream opens an SSE route and feeds its records to fn. +func (c *Client) stream(ctx context.Context, path string, fn func(typ string, data []byte) error) error { + req, err := c.request(ctx, http.MethodGet, path, nil) + if err != nil { + return err + } + req.Header.Set("Accept", ContentTypeEvent) + resp, err := c.http.Do(req) + if err != nil { + return c.unreachable(ctx, err) + } + c.observe(nil) + defer func() { _ = resp.Body.Close() }() + if err := checkStatus(resp); err != nil { + return err + } + err = ReadEvents(resp.Body, fn) + if ctx.Err() != nil { + // The caller ended the stream; the transport error that follows is + // not news. + return nil + } + return err +} + +// unreachable wraps a transport failure as ErrUnreachable and records it, +// unless the caller cancelled its own context first (a cancelled caller is +// not evidence about the server; an expired deadline is). +func (c *Client) unreachable(ctx context.Context, err error) error { + wrapped := fmt.Errorf("%w: crux server %s: %w", ErrUnreachable, c.addr.target(), err) + // A caller that cancelled its own request learned nothing about the + // server; one whose deadline expired waiting on the transport did. + if !errors.Is(ctx.Err(), context.Canceled) { + c.observe(wrapped) + } + return wrapped +} + +func (c *Client) request(ctx context.Context, method, path string, body io.Reader) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, method, c.addr.BaseURL+path, body) + if err != nil { + return nil, err + } + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + return req, nil +} + +// checkStatus turns a non-2xx response into a *StatusError, reading the +// api.Error body when there is one. +func checkStatus(resp *http.Response) error { + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return nil + } + se := &StatusError{Status: resp.StatusCode} + data, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16)) + _ = json.Unmarshal(data, &se.Body) + return se +} + +// target is the address as a human reads it in an error. +func (a Address) target() string { + if a.Network == "unix" { + return "unix://" + a.Path + } + return a.BaseURL +} + +// WaitReady polls GET /v1/health until it answers or ctx ends; it is for a +// caller that just started `crux serve` and needs to know when to begin. +func (c *Client) WaitReady(ctx context.Context, interval time.Duration) error { + if interval <= 0 { + interval = 100 * time.Millisecond + } + t := time.NewTicker(interval) + defer t.Stop() + for { + if _, err := c.Health(ctx); err == nil { + return nil + } else if IsUnauthorized(err) { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + } + } +} diff --git a/src/crux/api/client/client.go b/src/crux/api/client/client.go new file mode 100644 index 0000000000..99dc73ac2a --- /dev/null +++ b/src/crux/api/client/client.go @@ -0,0 +1,966 @@ +// Package client is the sidecar half of the Crux runtime seam +// (hivecommons/hive#14138, stage 3): Runtime implements crux.Runtime by +// calling a `crux serve` process over the Crux API, so a host built against +// the interface runs unchanged whether the agents live in its process or in +// a sidecar. crux/runtime.New returns one for config.ModeSidecar. +// +// Every method maps to a route in api.RuntimeRoutes; TestParity in this +// package holds the interface and the table to each other, so a method +// cannot be added to crux.Runtime without an endpoint. Methods whose +// signature has no error return log the failure through Logger and answer +// their zero value, never a guess: a read that fails reports "unknown" +// (ok=false), a write that fails is dropped with a log line. Authorizer +// methods fail closed: anything but a 204 is a denial. +// +// The Background family is a no-op here. The loops run inside the serving +// process (serve.Server.Background), which lists them on /v1/health. +package client + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "net/url" + "sync" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/trajectory" + "github.com/hivecommons/hive/crux/watchdog" +) + +// DefaultTimeout bounds one API call when the method has no context of its +// own. Kicks and restarts can take a while (tmux round trips, CLI boot). +const DefaultTimeout = 2 * time.Minute + +// eventsRetry is how long the event stream waits before reconnecting after +// the server dropped it. +const eventsRetry = time.Second + +// Runtime is crux.Runtime over a `crux serve` process. Construct with New +// or Wrap; the zero value is not usable. +type Runtime struct { + c *api.Client + // Logger receives dropped-write and failed-read diagnostics; nil discards. + Logger *slog.Logger + // Timeout bounds calls that have no context parameter; zero means + // DefaultTimeout. + Timeout time.Duration + + mu sync.Mutex + kickObserver func(agentName, event, detail string) + eventObserver func(api.Event) + cancelEvents context.CancelFunc + eventsDone chan struct{} + retry time.Duration +} + +// New connects to the server at addr (see api.ParseAddress) with token. +func New(addr, token string) (*Runtime, error) { + c, err := api.NewClient(addr, token) + if err != nil { + return nil, err + } + return Wrap(c), nil +} + +// Wrap returns a Runtime over an existing api.Client. +func Wrap(c *api.Client) *Runtime { + return &Runtime{c: c, Logger: slog.New(slog.DiscardHandler), retry: eventsRetry} +} + +// Client returns the underlying api.Client. +func (r *Runtime) Client() *api.Client { return r.c } + +// Ready blocks until the server answers its health route or ctx ends. +func (r *Runtime) Ready(ctx context.Context, interval time.Duration) error { + return r.c.WaitReady(ctx, interval) +} + +func (r *Runtime) ctx() (context.Context, context.CancelFunc) { + d := r.Timeout + if d <= 0 { + d = DefaultTimeout + } + return context.WithTimeout(context.Background(), d) +} + +func (r *Runtime) logger() *slog.Logger { + if r.Logger == nil { + return slog.New(slog.DiscardHandler) + } + return r.Logger +} + +// call is one request with a fresh timeout context. +func (r *Runtime) call(method, path string, in, out any) error { + ctx, cancel := r.ctx() + defer cancel() + return unavailable(r.c.Do(ctx, method, path, in, out)) +} + +// callCtx is one request under the caller's context. +func (r *Runtime) callCtx(ctx context.Context, method, path string, in, out any) error { + return unavailable(r.c.Do(ctx, method, path, in, out)) +} + +// unavailable marks a transport failure as crux.ErrRuntimeUnavailable so a +// host that only knows crux can tell "runtime gone" from "runtime said no". +func unavailable(err error) error { + if err != nil && errors.Is(err, api.ErrUnreachable) { + return fmt.Errorf("%w: %w", crux.ErrRuntimeUnavailable, err) + } + return err +} + +// Status is the server's reachability as of the last call. +func (r *Runtime) Status() api.Status { return r.c.Status() } + +// drop logs a failed write whose method has no error return. +func (r *Runtime) drop(method string, err error, attrs ...any) { + r.logger().Warn("crux client: "+method+" dropped", append(attrs, "err", err)...) +} + +func esc(name string) string { return url.PathEscape(name) } + +// ---- Lifecycle ---- + +func (r *Runtime) Start(ctx context.Context, name string) error { + return r.callCtx(ctx, http.MethodPost, api.StartPath(esc(name)), nil, nil) +} + +func (r *Runtime) Stop(name string) error { + return r.call(http.MethodPost, api.StopPath(esc(name)), nil, nil) +} + +func (r *Runtime) Restart(ctx context.Context, name string) error { + return r.callCtx(ctx, http.MethodPost, api.RestartPath(esc(name)), api.RestartRequest{}, nil) +} + +func (r *Runtime) RestartWithBootstrap(ctx context.Context, name, prompt string) error { + return r.callCtx(ctx, http.MethodPost, api.RestartPath(esc(name)), api.RestartRequest{Bootstrap: prompt}, nil) +} + +func (r *Runtime) Pause(name, trigger, reason string) error { + return r.call(http.MethodPost, api.PausePath(esc(name)), api.PauseRequest{Trigger: trigger, Reason: reason}, nil) +} + +func (r *Runtime) PauseBy(name, trigger, reason, by string) error { + return r.call(http.MethodPost, api.PausePath(esc(name)), api.PauseRequest{Trigger: trigger, Reason: reason, By: by}, nil) +} + +func (r *Runtime) PauseByCause(name, trigger, reason, by string, cause crux.PauseCausation) error { + return r.call(http.MethodPost, api.PausePath(esc(name)), api.PauseRequest{ + Trigger: trigger, Reason: reason, By: by, + Cause: &api.PauseCausation{Depth: cause.Depth, HookName: cause.HookName, OriginTransition: cause.OriginTransition}, + }, nil) +} + +func (r *Runtime) Resume(ctx context.Context, name, trigger, reason string) error { + return r.callCtx(ctx, http.MethodPost, api.ResumePath(esc(name)), api.ResumeRequest{Trigger: trigger, Reason: reason}, nil) +} + +func (r *Runtime) IsPaused(name string) bool { + var p api.Process + if err := r.call(http.MethodGet, api.AgentPath(esc(name)), nil, &p); err != nil { + return false + } + return p.Paused +} + +func (r *Runtime) ResetRestartCount(name string) error { + return r.call(http.MethodDelete, api.RestartCountPath(esc(name)), nil, nil) +} + +// ---- Observer ---- + +// tail is the pane filter a wire snapshot gets: the server already applied +// the runtime's current-session filter, so the client keeps the last n. +func tail(lines []string, n int) []string { + if n <= 0 || n >= len(lines) { + return lines + } + return lines[len(lines)-n:] +} + +func decodeSnapshot(snap api.Snapshot) (*crux.Process, error) { + var p crux.Process + if err := json.Unmarshal(snap.Process, &p); err != nil { + return nil, fmt.Errorf("decoding snapshot of %s: %w", snap.Agent, err) + } + if len(snap.Pane) > 0 { + p.SetPaneCapture(snap.Pane, tail) + } + return &p, nil +} + +func (r *Runtime) snapshot(name string, fast bool) (*crux.Process, error) { + path := api.SnapshotPath(esc(name)) + if fast { + path += "?" + api.QueryFast + "=1" + } + var snap api.Snapshot + if err := r.call(http.MethodGet, path, nil, &snap); err != nil { + return nil, err + } + return decodeSnapshot(snap) +} + +func (r *Runtime) GetStatus(name string) (*crux.Process, error) { return r.snapshot(name, false) } +func (r *Runtime) GetStatusFast(name string) (*crux.Process, error) { return r.snapshot(name, true) } + +func (r *Runtime) AllStatuses() map[string]*crux.Process { + var list api.SnapshotList + if err := r.call(http.MethodGet, api.PathSnapshots, nil, &list); err != nil { + r.drop("AllStatuses", err) + return nil + } + out := make(map[string]*crux.Process, len(list.Snapshots)) + for _, snap := range list.Snapshots { + p, err := decodeSnapshot(snap) + if err != nil { + r.drop("AllStatuses", err, "agent", snap.Agent) + continue + } + out[snap.Agent] = p + } + return out +} + +func (r *Runtime) ResolveAgent(nameOrID string) string { + var p api.Process + if err := r.call(http.MethodGet, api.AgentPath(esc(nameOrID)), nil, &p); err != nil || p.Name == "" { + return nameOrID + } + return p.Name +} + +func (r *Runtime) logs(name, source string, lines int) (api.Logs, error) { + ctx, cancel := r.ctx() + defer cancel() + return r.c.Logs(ctx, name, source, lines) +} + +func (r *Runtime) GetOutput(name string, lines int) ([]string, error) { + out, err := r.logs(name, api.LogSourcePane, lines) + return out.Lines, err +} + +func (r *Runtime) GetBufferOutput(name string, lines int) ([]string, error) { + out, err := r.logs(name, api.LogSourceBuffer, lines) + return out.Lines, err +} + +func (r *Runtime) CaptureFullLog(name string) (string, error) { + out, err := r.logs(name, api.LogSourceFull, 0) + return out.Content, err +} + +// FullLogHeader reads the header off the full-log route; the content it +// carries alongside is discarded. +func (r *Runtime) FullLogHeader(name string) string { + out, err := r.logs(name, api.LogSourceFull, 0) + if err != nil { + return "" + } + return out.Header +} + +// ---- Kicker ---- + +func (r *Runtime) SendKick(name string, message string) error { + return r.call(http.MethodPost, api.KickPath(esc(name)), api.KickRequest{Message: message}, nil) +} + +func (r *Runtime) SendKickWithSource(name string, message string, source string) error { + return r.call(http.MethodPost, api.KickPath(esc(name)), api.KickRequest{Message: message, Source: source}, nil) +} + +func (r *Runtime) SendKickAsync(name string, message string) (bool, error) { + var resp api.AgentResponse + if err := r.call(http.MethodPost, api.KickPath(esc(name)), api.KickRequest{Message: message, Async: true}, &resp); err != nil { + return false, err + } + return resp.Started != nil && *resp.Started, nil +} + +func (r *Runtime) ValidateKick(name string) error { + return r.call(http.MethodPost, api.KickValidatePath(esc(name)), nil, nil) +} + +func (r *Runtime) KickDispatchState(name string) (crux.KickDispatch, bool) { + var resp api.KickDispatchResponse + if err := r.call(http.MethodGet, api.KickPath(esc(name)), nil, &resp); err != nil || resp.Dispatch == nil { + return crux.KickDispatch{}, false + } + d := resp.Dispatch + return crux.KickDispatch{Agent: d.Agent, Phase: d.Phase, Error: d.Error, QueuedAt: d.QueuedAt, SettledAt: d.SettledAt}, true +} + +func (r *Runtime) ListKickLogs(name string) ([]crux.KickLogInfo, error) { + var list api.KickLogList + if err := r.call(http.MethodGet, api.KickLogsPath(esc(name)), nil, &list); err != nil { + return nil, err + } + out := make([]crux.KickLogInfo, 0, len(list.Logs)) + for _, l := range list.Logs { + out = append(out, crux.KickLogInfo{ID: l.ID, Timestamp: l.Timestamp, SizeBytes: l.SizeBytes, Reason: l.Reason, Source: l.Source}) + } + return out, nil +} + +func (r *Runtime) ReadKickLog(name, id string) (string, error) { + var log api.KickLog + if err := r.call(http.MethodGet, api.KickLogPath(esc(name), esc(id)), nil, &log); err != nil { + return "", err + } + return log.Content, nil +} + +// SetKickObserver subscribes to the server's event stream and relays the +// kick relay events (api.EventKick) to fn; nil unsubscribes. The stream +// reconnects after a drop until unsubscribed. SetEventObserver shares the +// same stream. +func (r *Runtime) SetKickObserver(fn func(agentName, event, detail string)) { + r.mu.Lock() + r.kickObserver = fn + r.mu.Unlock() + r.syncStream() +} + +// SetEventObserver delivers every event on the stream to fn (nil stops): +// the Host observer events (api.EventKickOutcome, api.EventPauseTransition, +// ...) a sidecar host would otherwise never see. +func (r *Runtime) SetEventObserver(fn func(api.Event)) { + r.mu.Lock() + r.eventObserver = fn + r.mu.Unlock() + r.syncStream() +} + +// syncStream starts the event stream when an observer is set and none runs, +// and stops it when the last observer is cleared. +func (r *Runtime) syncStream() { + r.mu.Lock() + want := r.kickObserver != nil || r.eventObserver != nil + running := r.cancelEvents != nil + var cancel context.CancelFunc + var done chan struct{} + if want && !running { + ctx, c := context.WithCancel(context.Background()) + r.cancelEvents = c + r.eventsDone = make(chan struct{}) + go r.streamEvents(ctx, r.eventsDone) + } + if !want && running { + cancel, done = r.cancelEvents, r.eventsDone + r.cancelEvents, r.eventsDone = nil, nil + } + r.mu.Unlock() + if cancel != nil { + cancel() + <-done + } +} + +func (r *Runtime) streamEvents(ctx context.Context, done chan struct{}) { + defer close(done) + for { + err := r.c.Events(ctx, func(ev api.Event) error { + r.dispatch(ev) + return nil + }) + if ctx.Err() != nil { + return + } + if err != nil { + r.logger().Warn("crux client: event stream dropped; reconnecting", "err", err) + } + select { + case <-ctx.Done(): + return + case <-time.After(r.retry): + } + } +} + +func (r *Runtime) dispatch(ev api.Event) { + r.mu.Lock() + kick, all := r.kickObserver, r.eventObserver + r.mu.Unlock() + if ev.Type == api.EventKick && kick != nil { + kick(ev.Agent, ev.State, ev.Detail) + } + if all != nil { + all(ev) + } +} + +// ---- Definitions ---- + +func (r *Runtime) AddAgent(name string, cfg config.AgentConfig) { + raw, err := json.Marshal(cfg) + if err != nil { + r.drop("AddAgent", err, "agent", name) + return + } + if err := r.call(http.MethodPost, api.PathDefinitions, api.DefinitionRequest{Name: name, Definition: raw}, nil); err != nil { + r.drop("AddAgent", err, "agent", name) + } +} + +func (r *Runtime) RemoveAgent(name string) { + if err := r.call(http.MethodDelete, api.AgentPath(esc(name)), nil, nil); err != nil && !api.IsNotFound(err) { + r.drop("RemoveAgent", err, "agent", name) + } +} + +func (r *Runtime) UpdateConfig(name string, cfg config.AgentConfig) error { + raw, err := json.Marshal(cfg) + if err != nil { + return err + } + return r.call(http.MethodPut, api.DefinitionPath(esc(name)), api.DefinitionRequest{Name: name, Definition: raw}, nil) +} + +func (r *Runtime) ReconcileAgents(configs map[string]config.AgentConfig) []string { + req := api.ReconcileRequest{Definitions: make(map[string]json.RawMessage, len(configs))} + for name, cfg := range configs { + raw, err := json.Marshal(cfg) + if err != nil { + r.drop("ReconcileAgents", err, "agent", name) + return nil + } + req.Definitions[name] = raw + } + var out api.Names + if err := r.call(http.MethodPost, api.PathReconcile, req, &out); err != nil { + r.drop("ReconcileAgents", err) + return nil + } + return out.Names +} + +func (r *Runtime) SetModelOverride(name, model string) error { + return r.call(http.MethodPut, api.OverridesPath(esc(name)), api.OverridesRequest{Model: &model}, nil) +} + +func (r *Runtime) SetBackendOverride(name, backend string) error { + return r.call(http.MethodPut, api.OverridesPath(esc(name)), api.OverridesRequest{Backend: &backend}, nil) +} + +func (r *Runtime) PinCLI(name, version string) error { + return r.call(http.MethodPut, api.PinPath(esc(name), api.PinCLI), api.PinRequest{Value: version}, nil) +} + +func (r *Runtime) PinModel(name, model string) error { + return r.call(http.MethodPut, api.PinPath(esc(name), api.PinModel), api.PinRequest{Value: model}, nil) +} + +func (r *Runtime) UnpinCLI(name string) error { + return r.call(http.MethodDelete, api.PinPath(esc(name), api.PinCLI), nil, nil) +} + +func (r *Runtime) UnpinModel(name string) error { + return r.call(http.MethodDelete, api.PinPath(esc(name), api.PinModel), nil, nil) +} + +func (r *Runtime) ClearModeOverrides(names []string) { + if err := r.call(http.MethodPost, api.PathModesClear, api.Names{Names: names}, nil); err != nil { + r.drop("ClearModeOverrides", err) + } +} + +func (r *Runtime) SyncModeFiles(level int) { + if err := r.call(http.MethodPost, api.PathModesSync, api.Level{Level: level}, nil); err != nil { + r.drop("SyncModeFiles", err) + } +} + +func (r *Runtime) RefreshInferenceRoutes(backend string) { + if err := r.call(http.MethodPost, api.PathInference, api.Backend{Backend: backend}, nil); err != nil { + r.drop("RefreshInferenceRoutes", err) + } +} + +// ---- Fleet ---- + +func (r *Runtime) fleet() (api.FleetInfo, error) { + var info api.FleetInfo + err := r.call(http.MethodGet, api.PathFleet, nil, &info) + return info, err +} + +func (r *Runtime) BreakerState() (bool, []string) { + var b api.Breaker + if err := r.call(http.MethodGet, api.PathBreaker, nil, &b); err != nil { + r.drop("BreakerState", err) + return false, nil + } + return b.Engaged, b.Paused +} + +// names posts/gets a Names list under the caller's context. +func (r *Runtime) names(ctx context.Context, method, verb, path string) []string { + var out api.Names + if err := r.c.Do(ctx, verb, path, nil, &out); err != nil { + r.drop(method, err) + return nil + } + return out.Names +} + +func (r *Runtime) EngageBreaker() []string { + ctx, cancel := r.ctx() + defer cancel() + return r.names(ctx, "EngageBreaker", http.MethodPost, api.PathBreakerOn) +} + +func (r *Runtime) ReleaseBreaker(ctx context.Context) []string { + return r.names(ctx, "ReleaseBreaker", http.MethodPost, api.PathBreakerOff) +} + +func (r *Runtime) SetACMMLevel(level int) { + if err := r.call(http.MethodPut, api.PathACMM, api.Level{Level: level}, nil); err != nil { + r.drop("SetACMMLevel", err) + } +} + +func (r *Runtime) GetACMMLevel() int { + info, err := r.fleet() + if err != nil { + r.drop("GetACMMLevel", err) + } + return info.ACMMLevel +} + +func (r *Runtime) RelaunchBobAgentsAwaitingKey(ctx context.Context) []string { + return r.names(ctx, "RelaunchBobAgentsAwaitingKey", http.MethodPost, api.PathRelaunchBob) +} + +func (r *Runtime) SharedDevUID() bool { + info, err := r.fleet() + if err != nil { + r.drop("SharedDevUID", err) + } + return info.SharedDevUID +} + +// ---- Credentials ---- + +func (r *Runtime) SubmitLoginCode(name, code string) error { + return r.call(http.MethodPost, api.LoginCodePath(esc(name)), api.LoginCodeRequest{Code: code}, nil) +} + +func (r *Runtime) copilot() api.CopilotCredential { + var c api.CopilotCredential + if err := r.call(http.MethodGet, api.PathCopilotCred, nil, &c); err != nil { + r.drop("CopilotToken", err) + } + return c +} + +func (r *Runtime) CopilotToken() string { return r.copilot().Token } +func (r *Runtime) CopilotTokenSource() string { return r.copilot().Source } + +func (r *Runtime) SetCopilotToken(token string) { + if err := r.call(http.MethodPut, api.PathCopilotCred, api.CopilotCredential{Token: token}, nil); err != nil { + r.drop("SetCopilotToken", err) + } +} + +func (r *Runtime) ActivateCopilotToken(token string) error { + return r.call(http.MethodPut, api.PathCopilotCred, api.CopilotCredential{Token: token, Activate: true}, nil) +} + +func (r *Runtime) ReloadClaudeToken() { + if err := r.call(http.MethodPost, api.PathClaudeReload, nil, nil); err != nil { + r.drop("ReloadClaudeToken", err) + } +} + +func (r *Runtime) ClaudeCredentialCandidatePaths() []string { + var c api.ClaudeCredential + if err := r.call(http.MethodGet, api.PathClaudeCred, nil, &c); err != nil { + r.drop("ClaudeCredentialCandidatePaths", err) + return nil + } + return c.CandidatePaths +} + +func (r *Runtime) OmpAgentHomes() []string { + var h api.OmpHomes + if err := r.call(http.MethodGet, api.PathOmpHomes, nil, &h); err != nil { + r.drop("OmpAgentHomes", err) + return nil + } + return h.Homes +} + +// ---- Resumer ---- + +func (r *Runtime) session(name string) (api.Session, bool) { + var s api.Session + if err := r.call(http.MethodGet, api.SessionPath(esc(name)), nil, &s); err != nil { + if !api.IsNotFound(err) { + r.drop("Session", err, "agent", name) + } + return api.Session{}, false + } + return s, true +} + +func (r *Runtime) ResumeHandle(name string) (crux.ResumeHandle, bool) { + s, ok := r.session(name) + if !ok || s.Handle == nil { + return crux.ResumeHandle{}, false + } + h := s.Handle + return crux.ResumeHandle{Backend: h.Backend, SessionID: h.SessionID, Transcript: h.Transcript, Command: h.Command, LastActive: h.LastActive, CapturedAt: h.CapturedAt}, true +} + +// SendResumeKick maps the server's resume codes back onto the crux +// sentinels so errors.Is works for a sidecar host as it does in-process. +func (r *Runtime) SendResumeKick(name, message, sessionID string) error { + err := r.call(http.MethodPost, api.SessionKickPath(esc(name)), api.ResumeKickRequest{Message: message, SessionID: sessionID}, nil) + var se *api.StatusError + if errors.As(err, &se) { + switch se.Body.Code { + case api.CodeResumeBusy: + return fmt.Errorf("%w: %s", crux.ErrResumeBusy, se.Body.Error) + case api.CodeResumeSessionGone: + return fmt.Errorf("%w: %s", crux.ErrResumeSessionGone, se.Body.Error) + case api.CodeResumeUnsupported: + return fmt.Errorf("%w: %s", crux.ErrResumeUnsupported, se.Body.Error) + } + } + return err +} + +func (r *Runtime) SessionID(name string) (string, bool) { + s, ok := r.session(name) + if !ok || !s.HasSession { + return "", false + } + return s.SessionID, true +} + +func (r *Runtime) SessionMissing(name string) bool { + s, _ := r.session(name) + return s.Missing +} + +// ---- Supervisor ---- + +func (r *Runtime) supervision(name string) (api.Supervision, bool) { + var s api.Supervision + if err := r.call(http.MethodGet, api.SupervisionPath(esc(name)), nil, &s); err != nil { + if !api.IsNotFound(err) { + r.drop("Supervision", err, "agent", name) + } + return api.Supervision{}, false + } + return s, true +} + +func (r *Runtime) GovernorMayStart(name string) bool { + s, _ := r.supervision(name) + return s.MayStart +} + +func (r *Runtime) MarkStartupLaunchQueued(names []string) { + if err := r.call(http.MethodPost, api.PathStartupQueued, api.Names{Names: names}, nil); err != nil { + r.drop("MarkStartupLaunchQueued", err) + } +} + +func (r *Runtime) CheckAndRestartCrashedAgents(ctx context.Context) []string { + return r.names(ctx, "CheckAndRestartCrashedAgents", http.MethodPost, api.PathRestartCrashed) +} + +func (r *Runtime) ConsentWedgedAgents() []string { + info, err := r.fleet() + if err != nil { + r.drop("ConsentWedgedAgents", err) + } + return info.ConsentWedged +} + +func (r *Runtime) RefreshAgentTokens(ctx context.Context) { + if err := r.callCtx(ctx, http.MethodPost, api.PathTokensRefresh, nil, nil); err != nil { + r.drop("RefreshAgentTokens", err) + } +} + +func (r *Runtime) AgentCredentialRefreshable(agentName string) (bool, bool) { + s, ok := r.supervision(agentName) + return s.CredentialRefreshable, ok && s.CredentialKnown +} + +func (r *Runtime) AgentCapabilities(agentName string) (bool, bool, bool, bool) { + s, ok := r.supervision(agentName) + if !ok || s.Capabilities == nil { + return false, false, false, false + } + return s.Capabilities.CanOpenIssue, s.Capabilities.CanOpenPR, s.Capabilities.CanMerge, true +} + +func (r *Runtime) EffectiveBackend(agentName string) (string, bool) { + s, ok := r.supervision(agentName) + if !ok || s.Backend == nil { + return "", false + } + return *s.Backend, true +} + +func (r *Runtime) InvocationMetadata(agentName string) (string, string, string, bool) { + s, ok := r.supervision(agentName) + if !ok || s.Invocation == nil { + return "", "", "", false + } + return s.Invocation.Backend, s.Invocation.Model, s.Invocation.Effort, true +} + +func (r *Runtime) RestartTelemetry(name string) (int, int, time.Time, string, bool) { + s, ok := r.supervision(name) + if !ok || s.Restarts == nil { + return 0, 0, time.Time{}, "", false + } + t := s.Restarts + return t.Total, t.Last24h, t.LastRestartAt, t.LastReason, true +} + +func (r *Runtime) StartFailureState(name string) (crux.StartFailureSnapshot, bool) { + s, ok := r.supervision(name) + if !ok || s.StartFailure == nil { + return crux.StartFailureSnapshot{}, false + } + f := s.StartFailure + return crux.StartFailureSnapshot{Reason: f.Reason, Count: f.Count, LastAt: f.LastAt, LastExitCode: f.LastExitCode, LastSignal: f.LastSignal, Blocked: f.Blocked}, true +} + +func (r *Runtime) ProviderErrorBackoffRemaining(name string) (time.Duration, string, string, bool) { + s, ok := r.supervision(name) + if !ok || s.ProviderBackoff == nil { + return 0, "", "", false + } + b := s.ProviderBackoff + return time.Duration(b.RemainingMS) * time.Millisecond, b.Provider, b.Reason, true +} + +func (r *Runtime) ArchiveAllKickLogs(reason string) { + if err := r.call(http.MethodPost, api.PathKickLogsArchive, api.Reason{Reason: reason}, nil); err != nil { + r.drop("ArchiveAllKickLogs", err) + } +} + +func (r *Runtime) CountAgentsWithModel() int { + info, err := r.fleet() + if err != nil { + r.drop("CountAgentsWithModel", err) + } + return info.AgentsWithModel +} + +func (r *Runtime) CrashOOMSuspected(name string) bool { + s, _ := r.supervision(name) + return s.CrashOOMSuspected +} + +func (r *Runtime) BackendAuthAvailable(backend string) (bool, bool) { + var a api.BackendAuth + if err := r.call(http.MethodGet, api.BackendAuthPath(esc(backend)), nil, &a); err != nil { + r.drop("BackendAuthAvailable", err, "backend", backend) + return false, false + } + return a.Available, a.Known +} + +func (r *Runtime) AgentAuthAvailable(agentName string) (bool, bool) { + s, ok := r.supervision(agentName) + return s.AuthAvailable, ok && s.AuthKnown +} + +func (r *Runtime) SendGitHubAPIBudgetNudge(name, message string) { + if err := r.call(http.MethodPost, api.NudgePath(esc(name)), api.NudgeRequest{Message: message}, nil); err != nil { + r.drop("SendGitHubAPIBudgetNudge", err, "agent", name) + } +} + +func (r *Runtime) RecordAudit(action, agentName string, fields map[string]any) { + if err := r.call(http.MethodPost, api.PathAudit, api.AuditRequest{Action: action, Agent: agentName, Fields: fields}, nil); err != nil { + r.drop("RecordAudit", err, "action", action) + } +} + +func (r *Runtime) AgentHasValidCredential(agentName string) bool { + s, _ := r.supervision(agentName) + return s.HasValidCredential +} + +func (r *Runtime) RefreshAgentTokenFor(ctx context.Context, name string) error { + return r.callCtx(ctx, http.MethodPost, api.AgentTokenRefreshPath(esc(name)), nil, nil) +} + +func (r *Runtime) TrajectoryAgents() []trajectory.AgentView { + var list api.TrajectoryList + if err := r.call(http.MethodGet, api.PathTrajectory, nil, &list); err != nil { + r.drop("TrajectoryAgents", err) + return nil + } + out := make([]trajectory.AgentView, 0, len(list.Agents)) + for _, a := range list.Agents { + out = append(out, trajectory.AgentView{Name: a.Name, Running: a.Running, Intent: a.Intent, Transcript: a.Transcript, Idle: a.Idle, LastOutputAt: a.LastOutputAt}) + } + return out +} + +// ---- Restorer ---- + +func (r *Runtime) seed(method, name string, req api.SeedRequest) { + if err := r.call(http.MethodPost, api.SeedPath(esc(name)), req, nil); err != nil { + r.drop(method, err, "agent", name) + } +} + +func (r *Runtime) SeedPauseState(name string, pausedAt time.Time, trigger, reason, by string) { + r.seed("SeedPauseState", name, api.SeedRequest{Pause: &api.PauseSeed{PausedAt: pausedAt, Trigger: trigger, Reason: reason, By: by}}) +} + +func (r *Runtime) SeedRestartTelemetry(name string, count int, events []crux.RestartEvent, lastReason string) { + wire := make([]api.RestartEvent, 0, len(events)) + for _, e := range events { + wire = append(wire, api.RestartEvent{At: e.At, Reason: e.Reason}) + } + r.seed("SeedRestartTelemetry", name, api.SeedRequest{Restarts: &api.RestartSeed{Count: count, Events: wire, LastReason: lastReason}}) +} + +func (r *Runtime) SeedTurnLoss(name string, loss crux.TurnLoss) { + raw, err := json.Marshal(loss) + if err != nil { + r.drop("SeedTurnLoss", err, "agent", name) + return + } + r.seed("SeedTurnLoss", name, api.SeedRequest{TurnLoss: raw}) +} + +func (r *Runtime) SeedLastKick(name string, t time.Time) { + r.seed("SeedLastKick", name, api.SeedRequest{LastKick: &t}) +} + +func (r *Runtime) SeedKickHistory(name string, records []crux.KickRecord) { + wire := make([]api.KickRecord, 0, len(records)) + for _, k := range records { + wire = append(wire, api.KickRecord{Timestamp: k.Timestamp, Agent: k.Agent, Snippet: k.Snippet}) + } + r.seed("SeedKickHistory", name, api.SeedRequest{KickHistory: &wire}) +} + +func (r *Runtime) RestoreBreaker(engaged bool, paused []string) { + if err := r.call(http.MethodPut, api.PathBreaker, api.Breaker{Engaged: engaged, Paused: paused}, nil); err != nil { + r.drop("RestoreBreaker", err) + } +} + +// ---- Background (server-side; no-ops here) ---- + +func (r *Runtime) StartAgentTokenRefresh(context.Context) {} +func (r *Runtime) StartCredentialWatchdog(context.Context) {} +func (r *Runtime) StartCopilotSessionRefresh(context.Context) {} +func (r *Runtime) StartTranscriptActivityRefresh(context.Context) {} + +// ---- Authorizer (fail closed) ---- + +// authorize returns nil only for a 204. A 403 carries the runtime's reason +// as a plain error, the way the in-process Authorizer reports it; any other +// failure, including the server being unreachable, is a denial too. +func (r *Runtime) authorize(req api.AuthorizeRequest) error { + err := r.call(http.MethodPost, api.PathAuthorize, req, nil) + if err == nil { + return nil + } + var se *api.StatusError + if errors.As(err, &se) && se.Status == http.StatusForbidden && se.Body.Error != "" { + return errors.New(se.Body.Error) + } + return fmt.Errorf("authorize %s for %s: denied: %w", req.Action, req.Agent, err) +} + +func (r *Runtime) AuthorizePROpen(agentName string, fileUID int) error { + return r.authorize(api.AuthorizeRequest{Action: api.AuthorizePROpen, Agent: agentName, FileUID: fileUID}) +} + +func (r *Runtime) AuthorizePushBranch(agentName string, fileUID int) error { + return r.authorize(api.AuthorizeRequest{Action: api.AuthorizePushBranch, Agent: agentName, FileUID: fileUID}) +} + +func (r *Runtime) AuthorizeIssueOpen(agentName string, fileUID int, kind string) error { + return r.authorize(api.AuthorizeRequest{Action: api.AuthorizeIssueOpen, Agent: agentName, FileUID: fileUID, Kind: kind}) +} + +func (r *Runtime) AuthorizeMerge(agentName string, fileUID int, repo string) error { + return r.authorize(api.AuthorizeRequest{Action: api.AuthorizeMerge, Agent: agentName, FileUID: fileUID, Repo: repo}) +} + +func (r *Runtime) AuthorizeReviewRequest(agentName string, fileUID int) error { + return r.authorize(api.AuthorizeRequest{Action: api.AuthorizeReviewRequest, Agent: agentName, FileUID: fileUID}) +} + +// ---- watchdog.Source over the wire ---- + +// WatchdogSource returns the server's watchdog seam, the way +// *agent.Manager.WatchdogSource does in-process, so crux/runtime's +// NewWatchdogFleet can compose a fleet for a sidecar. +func (r *Runtime) WatchdogSource() watchdog.Source { return source{r} } + +type source struct{ r *Runtime } + +func (s source) AgentNames() []string { + ctx, cancel := s.r.ctx() + defer cancel() + return s.r.names(ctx, "WatchdogSource.AgentNames", http.MethodGet, api.PathWatchdogAgents) +} + +func (s source) observe(name string) (api.WatchdogObservation, error) { + var o api.WatchdogObservation + err := s.r.call(http.MethodGet, api.WatchdogPath(esc(name)), nil, &o) + return o, err +} + +func (s source) Observe(name string) (watchdog.Observation, error) { + o, err := s.observe(name) + if err != nil { + return watchdog.Observation{}, err + } + return watchdog.Observation{ + Backend: o.Backend, SessionExists: o.SessionExists, Pane: o.Pane, HasCLIMarker: o.HasCLIMarker, + ShowsLoginPrompt: o.ShowsLoginPrompt, LastChange: o.LastChange, StartedAt: o.StartedAt, + AuthAvailable: o.AuthAvailable, AuthKnown: o.AuthKnown, CredentialProven: o.CredentialProven, + ProviderErrorClass: o.ProviderErrorClass, ProviderErrorLine: o.ProviderErrorLine, + }, nil +} + +func (s source) LastProduction(name string) (time.Time, bool) { + o, err := s.observe(name) + if err != nil || o.LastProduction == nil { + return time.Time{}, false + } + return *o.LastProduction, true +} + +func (s source) SetConditions(name string, conds []crux.Condition) { + wire := make([]api.Condition, 0, len(conds)) + for _, c := range conds { + wire = append(wire, api.Condition{Type: string(c.Type), Status: string(c.Status), Reason: c.Reason, Message: c.Message, LastTransitionTime: c.LastTransitionTime}) + } + if err := s.r.call(http.MethodPut, api.WatchdogConditionsPath(esc(name)), api.Conditions{Conditions: wire}, nil); err != nil { + s.r.drop("WatchdogSource.SetConditions", err, "agent", name) + } +} + +var _ crux.Runtime = (*Runtime)(nil) diff --git a/src/crux/api/client/client_test.go b/src/crux/api/client/client_test.go new file mode 100644 index 0000000000..d6be6497be --- /dev/null +++ b/src/crux/api/client/client_test.go @@ -0,0 +1,773 @@ +package client + +import ( + "context" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "reflect" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/cruxtest" + "github.com/hivecommons/hive/crux/serve" + "github.com/hivecommons/hive/crux/trajectory" + "github.com/hivecommons/hive/crux/watchdog" +) + +const testToken = "client-token" + +// servedBy names the runtime method the server calls for a client method +// whose route is shared: IsPaused reads the agent view the GET agent route +// builds from GetStatus. +var servedBy = map[string]string{"IsPaused": "GetStatus"} + +// localOnly is the set of crux.Runtime methods that never cross the wire: +// the Background loops run in the serving process, and SetKickObserver +// subscribes to the event stream rather than calling a route. +var localOnly = map[string]bool{ + "StartAgentTokenRefresh": true, + "StartCredentialWatchdog": true, + "StartCopilotSessionRefresh": true, + "StartTranscriptActivityRefresh": true, + "SetKickObserver": true, +} + +func runtimeMethods(t *testing.T) []string { + t.Helper() + typ := reflect.TypeOf((*crux.Runtime)(nil)).Elem() + names := make([]string, 0, typ.NumMethod()) + for i := 0; i < typ.NumMethod(); i++ { + names = append(names, typ.Method(i).Name) + } + return names +} + +// TestParityInterfaceVersusRouteTable holds crux.Runtime and +// api.RuntimeRoutes to each other in both directions. +func TestParityInterfaceVersusRouteTable(t *testing.T) { + methods := runtimeMethods(t) + for _, m := range methods { + if _, ok := api.RuntimeRoutes[m]; !ok { + t.Errorf("crux.Runtime.%s has no entry in api.RuntimeRoutes", m) + } + } + set := map[string]bool{} + for _, m := range methods { + set[m] = true + } + for m := range api.RuntimeRoutes { + if !set[m] { + t.Errorf("api.RuntimeRoutes[%q] names no crux.Runtime method", m) + } + } + // Runtime's method set is exactly the interface: no extra wire methods + // hide behind the type. + rtType := reflect.TypeOf((*Runtime)(nil)) + for _, m := range methods { + if _, ok := rtType.MethodByName(m); !ok { + t.Errorf("*client.Runtime lacks %s", m) + } + } +} + +// TestParityHostVersusSetterTable holds crux.Host and api.HostSetters to +// each other: every setter has a disposition (route | callback | event | +// config | in-process-only) and every row names a setter. Callback rows +// must name a callback endpoint route the callback package registers, +// event rows an event type the runtime emits. +func TestParityHostVersusSetterTable(t *testing.T) { + typ := reflect.TypeOf((*crux.Host)(nil)).Elem() + setters := map[string]bool{} + for i := 0; i < typ.NumMethod(); i++ { + name := typ.Method(i).Name + if !strings.HasPrefix(name, "Set") { + continue + } + setters[name] = true + if _, ok := api.HostSetters[name]; !ok { + t.Errorf("crux.Host.%s has no entry in api.HostSetters: say how a sidecar runtime is served", name) + } + } + callbackPaths := map[string]bool{ + callback.PathTokens: true, callback.PathPolicy: true, callback.PathRepoAutoMerge: true, + callback.PathLinearCredential: true, callback.PathPulls: true, callback.PathConditions: true, + callback.PathInferenceRoutes + "/{agent}": true, callback.PathEffects: true, callback.PathEffects + "/{id}/end": true, + callback.PathAgentMint: true, callback.PathBobCredential: true, + } + events := map[string]bool{api.EventKickOutcome: true, api.EventPauseTransition: true, api.EventRelaunch: true, api.EventPausePersist: true, api.EventPrompt: true, api.EventSandboxAudit: true, api.EventAudit: true, api.EventCondition: true} + for name, row := range api.HostSetters { + if !setters[name] { + t.Errorf("api.HostSetters[%q] names no crux.Host setter", name) + } + if row.Via == "" { + t.Errorf("api.HostSetters[%q] has no Via", name) + } + switch row.Kind { + case api.HostSetterCallback: + for _, part := range strings.Split(row.Via, ", ") { + fields := strings.Fields(part) + if len(fields) != 2 || !callbackPaths[fields[1]] { + t.Errorf("api.HostSetters[%q] names unknown callback route %q", name, part) + } + } + case api.HostSetterEvent: + if !events[strings.Split(row.Via, ",")[0]] { + t.Errorf("api.HostSetters[%q] names unknown event %q", name, row.Via) + } + case api.HostSetterRoute: + found := false + for _, r := range api.RuntimeRoutes { + if r.Method+" "+r.Path == row.Via { + found = true + } + } + if !found { + t.Errorf("api.HostSetters[%q] names unknown runtime route %q", name, row.Via) + } + case api.HostSetterConfig, api.HostSetterInProcessOnly: + default: + t.Errorf("api.HostSetters[%q] has unknown kind %q", name, row.Kind) + } + } + if api.HostSetters["SetSpanStarter"].Kind != api.HostSetterInProcessOnly { + t.Error("SetSpanStarter is the one in-process-only setter") + } +} + +type harness struct { + rec *cruxtest.Recorder + srv *serve.Server + ts *httptest.Server + rt *Runtime +} + +func newHarness(t *testing.T) *harness { + t.Helper() + rec := cruxtest.NewRecorder() + srv := serve.New(rec) + srv.Token = testToken + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(ts.Close) + rt, err := New(ts.URL, testToken) + if err != nil { + t.Fatal(err) + } + rt.Logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) + return &harness{rec: rec, srv: srv, ts: ts, rt: rt} +} + +// TestEveryMethodReachesTheRuntime drives each crux.Runtime method through +// the client into a serve.Server over a Recorder and checks the same method +// was recorded on the far side. +func TestEveryMethodReachesTheRuntime(t *testing.T) { + h := newHarness(t) + rec, rt := h.rec, h.rt + p := rec.Add("alpha", "a1", crux.StateRunning) + p.Paused = true + rec.Pane["alpha"] = []string{"l1", "l2", "l3"} + rec.Buffer["alpha"] = []string{"b1"} + rec.FullLog["alpha"] = "full" + rec.Header["alpha"] = "hdr" + rec.KickStarted = true + rec.Dispatch = map[string]crux.KickDispatch{"alpha": {Agent: "alpha", Phase: "queued"}} + rec.KickLogs = map[string][]crux.KickLogInfo{"alpha": {{ID: "k1", Reason: "r"}}} + rec.KickLogContent = map[string]map[string]string{"alpha": {"k1": "content"}} + rec.Reconciled = []string{"alpha"} + rec.BreakerEngaged = true + rec.BreakerPaused = []string{"alpha"} + rec.Resumed = []string{"alpha"} + rec.ACMM = 3 + rec.Shared = true + rec.Relaunched = []string{"alpha"} + rec.Crashed = []string{"alpha"} + rec.Wedged = []string{"alpha"} + rec.WithModel = 4 + rec.Copilot = "ghu" + rec.CopilotSource = "env" + rec.ClaudePaths = []string{"/a", "/b"} + rec.OmpHomes = []string{"/h/a"} + rec.Handles = map[string]crux.ResumeHandle{"alpha": {Backend: "claude", SessionID: "s1"}} + rec.Sessions = map[string]string{"alpha": "s1"} + rec.Missing = map[string]bool{"alpha": true} + rec.MayStart = true + rec.Refreshable, rec.RefreshKnown = true, true + rec.Valid = true + rec.Caps = [4]bool{true, false, true, true} + rec.Backend, rec.BackendOK = "codex", true + rec.Invocation, rec.InvocationOK = [3]string{"codex", "gpt", "high"}, true + rec.Restarts.Total, rec.Restarts.Last24h, rec.Restarts.LastReason, rec.Restarts.OK = 5, 2, "crash", true + rec.StartFailure, rec.StartFailureOK = crux.StartFailureSnapshot{Reason: "boom", Count: 2, Blocked: true}, true + rec.Backoff.Remaining, rec.Backoff.Provider, rec.Backoff.Reason, rec.Backoff.OK = 3*time.Second, "anthropic", "429", true + rec.OOM = true + rec.AuthAvail, rec.AuthKnown = true, true + rec.BackendAuth = map[string][2]bool{"claude": {true, true}} + rec.Views = []trajectory.AgentView{{Name: "alpha", Running: true}} + + ctx := context.Background() + check := func(name string, got, want any) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: got %#v want %#v", name, got, want) + } + } + must := func(name string, err error) { + t.Helper() + if err != nil { + t.Errorf("%s: %v", name, err) + } + } + + // Lifecycle + must("Start", rt.Start(ctx, "alpha")) + must("Stop", rt.Stop("alpha")) + must("Restart", rt.Restart(ctx, "alpha")) + must("RestartWithBootstrap", rt.RestartWithBootstrap(ctx, "alpha", "hello")) + must("Pause", rt.Pause("alpha", "op", "why")) + must("PauseBy", rt.PauseBy("alpha", "op", "why", "me")) + must("PauseByCause", rt.PauseByCause("alpha", "op", "why", "me", crux.PauseCausation{Depth: 1, HookName: "h"})) + check("IsPaused", rt.IsPaused("alpha"), true) + must("Resume", rt.Resume(ctx, "alpha", "op", "why")) + check("IsPaused.resumed", rt.IsPaused("alpha"), false) + must("ResetRestartCount", rt.ResetRestartCount("alpha")) + + // Observer + st, err := rt.GetStatus("alpha") + must("GetStatus", err) + check("GetStatus.Name", st.Name, "alpha") + check("GetStatus.Pane", st.PaneLines(2), []string{"l2", "l3"}) + stf, err := rt.GetStatusFast("alpha") + must("GetStatusFast", err) + check("GetStatusFast.ID", stf.ID, "a1") + all := rt.AllStatuses() + check("AllStatuses.len", len(all), 1) + check("ResolveAgent", rt.ResolveAgent("a1"), "alpha") + check("ResolveAgent.unknown", rt.ResolveAgent("nope"), "nope") + lines, err := rt.GetOutput("alpha", 2) + must("GetOutput", err) + check("GetOutput", lines, []string{"l2", "l3"}) + buf, err := rt.GetBufferOutput("alpha", 0) + must("GetBufferOutput", err) + check("GetBufferOutput", buf, []string{"b1"}) + full, err := rt.CaptureFullLog("alpha") + must("CaptureFullLog", err) + check("CaptureFullLog", full, "full") + check("FullLogHeader", rt.FullLogHeader("alpha"), "hdr") + + // Kicker + must("SendKick", rt.SendKick("alpha", "go")) + must("SendKickWithSource", rt.SendKickWithSource("alpha", "go", "governor-cadence")) + started, err := rt.SendKickAsync("alpha", "go") + must("SendKickAsync", err) + check("SendKickAsync", started, true) + must("ValidateKick", rt.ValidateKick("alpha")) + d, ok := rt.KickDispatchState("alpha") + check("KickDispatchState.ok", ok, true) + check("KickDispatchState.phase", d.Phase, "queued") + kl, err := rt.ListKickLogs("alpha") + must("ListKickLogs", err) + check("ListKickLogs", kl, []crux.KickLogInfo{{ID: "k1", Reason: "r"}}) + kc, err := rt.ReadKickLog("alpha", "k1") + must("ReadKickLog", err) + check("ReadKickLog", kc, "content") + + // Definitions + rt.AddAgent("beta", config.AgentConfig{ID: "b1", Backend: "claude", Enabled: true}) + rt.RemoveAgent("beta") + rt.RemoveAgent("never-there") + must("UpdateConfig", rt.UpdateConfig("alpha", config.AgentConfig{ID: "a1", Backend: "claude", Enabled: true})) + check("ReconcileAgents", rt.ReconcileAgents(map[string]config.AgentConfig{"alpha": {ID: "a1", Backend: "claude", Enabled: true}}), []string{"alpha"}) + must("SetModelOverride", rt.SetModelOverride("alpha", "opus")) + must("SetBackendOverride", rt.SetBackendOverride("alpha", "codex")) + must("PinCLI", rt.PinCLI("alpha", "1.2.3")) + must("PinModel", rt.PinModel("alpha", "opus")) + must("UnpinCLI", rt.UnpinCLI("alpha")) + must("UnpinModel", rt.UnpinModel("alpha")) + rt.ClearModeOverrides([]string{"alpha"}) + rt.SyncModeFiles(2) + rt.RefreshInferenceRoutes("claude") + + // Fleet + eng, paused := rt.BreakerState() + check("BreakerState", []any{eng, paused}, []any{true, []string{"alpha"}}) + check("EngageBreaker", rt.EngageBreaker(), []string{"alpha"}) + check("ReleaseBreaker", rt.ReleaseBreaker(ctx), []string{"alpha"}) + rt.SetACMMLevel(2) + check("GetACMMLevel", rt.GetACMMLevel(), 2) + check("RelaunchBobAgentsAwaitingKey", rt.RelaunchBobAgentsAwaitingKey(ctx), []string{"alpha"}) + check("SharedDevUID", rt.SharedDevUID(), true) + + // Credentials + must("SubmitLoginCode", rt.SubmitLoginCode("alpha", "1234")) + check("CopilotToken", rt.CopilotToken(), "ghu") + check("CopilotTokenSource", rt.CopilotTokenSource(), "env") + rt.SetCopilotToken("ghu2") + must("ActivateCopilotToken", rt.ActivateCopilotToken("ghu3")) + rt.ReloadClaudeToken() + check("ClaudeCredentialCandidatePaths", rt.ClaudeCredentialCandidatePaths(), []string{"/a", "/b"}) + check("OmpAgentHomes", rt.OmpAgentHomes(), []string{"/h/a"}) + + // Resumer + hd, ok := rt.ResumeHandle("alpha") + check("ResumeHandle.ok", ok, true) + check("ResumeHandle", hd.SessionID, "s1") + must("SendResumeKick", rt.SendResumeKick("alpha", "msg", "s1")) + sid, ok := rt.SessionID("alpha") + check("SessionID", []any{sid, ok}, []any{"s1", true}) + check("SessionMissing", rt.SessionMissing("alpha"), true) + + // Supervisor + check("GovernorMayStart", rt.GovernorMayStart("alpha"), true) + rt.MarkStartupLaunchQueued([]string{"alpha"}) + check("CheckAndRestartCrashedAgents", rt.CheckAndRestartCrashedAgents(ctx), []string{"alpha"}) + check("ConsentWedgedAgents", rt.ConsentWedgedAgents(), []string{"alpha"}) + rt.RefreshAgentTokens(ctx) + rf, known := rt.AgentCredentialRefreshable("alpha") + check("AgentCredentialRefreshable", []bool{rf, known}, []bool{true, true}) + c1, c2, c3, cok := rt.AgentCapabilities("alpha") + check("AgentCapabilities", []bool{c1, c2, c3, cok}, []bool{true, false, true, true}) + be, ok := rt.EffectiveBackend("alpha") + check("EffectiveBackend", []any{be, ok}, []any{"codex", true}) + ib, im, ie, iok := rt.InvocationMetadata("alpha") + check("InvocationMetadata", []any{ib, im, ie, iok}, []any{"codex", "gpt", "high", true}) + tot, last, _, reason, rok := rt.RestartTelemetry("alpha") + check("RestartTelemetry", []any{tot, last, reason, rok}, []any{5, 2, "crash", true}) + sf, ok := rt.StartFailureState("alpha") + check("StartFailureState", []any{sf.Reason, sf.Count, sf.Blocked, ok}, []any{"boom", 2, true, true}) + rem, prov, breason, bok := rt.ProviderErrorBackoffRemaining("alpha") + check("ProviderErrorBackoffRemaining", []any{rem, prov, breason, bok}, []any{3 * time.Second, "anthropic", "429", true}) + rt.ArchiveAllKickLogs("rotate") + check("CountAgentsWithModel", rt.CountAgentsWithModel(), 4) + check("CrashOOMSuspected", rt.CrashOOMSuspected("alpha"), true) + av, ak := rt.BackendAuthAvailable("claude") + check("BackendAuthAvailable", []bool{av, ak}, []bool{true, true}) + aav, aak := rt.AgentAuthAvailable("alpha") + check("AgentAuthAvailable", []bool{aav, aak}, []bool{true, true}) + rt.SendGitHubAPIBudgetNudge("alpha", "slow down") + rt.RecordAudit("act", "alpha", map[string]any{"k": "v"}) + check("AgentHasValidCredential", rt.AgentHasValidCredential("alpha"), true) + must("RefreshAgentTokenFor", rt.RefreshAgentTokenFor(ctx, "alpha")) + check("TrajectoryAgents", rt.TrajectoryAgents(), []trajectory.AgentView{{Name: "alpha", Running: true}}) + + // Restorer + rt.SeedPauseState("alpha", time.Now(), "op", "why", "me") + rt.SeedRestartTelemetry("alpha", 1, []crux.RestartEvent{{At: time.Now(), Reason: "r"}}, "r") + rt.SeedTurnLoss("alpha", crux.TurnLoss{}) + rt.SeedLastKick("alpha", time.Now()) + rt.SeedKickHistory("alpha", []crux.KickRecord{{Agent: "alpha", Snippet: "s"}}) + rt.RestoreBreaker(true, []string{"alpha"}) + + // Background: local no-ops. + rt.StartAgentTokenRefresh(ctx) + rt.StartCredentialWatchdog(ctx) + rt.StartCopilotSessionRefresh(ctx) + rt.StartTranscriptActivityRefresh(ctx) + + // Authorizer + must("AuthorizePROpen", rt.AuthorizePROpen("alpha", 1000)) + must("AuthorizePushBranch", rt.AuthorizePushBranch("alpha", 1000)) + must("AuthorizeIssueOpen", rt.AuthorizeIssueOpen("alpha", 1000, "bug")) + must("AuthorizeMerge", rt.AuthorizeMerge("alpha", 1000, "o/r")) + must("AuthorizeReviewRequest", rt.AuthorizeReviewRequest("alpha", 1000)) + + seen := map[string]bool{} + for _, m := range rec.Methods() { + seen[m] = true + } + var missing []string + for _, m := range runtimeMethods(t) { + if localOnly[m] { + continue + } + if alias := servedBy[m]; alias != "" { + m = alias + } + if !seen[m] { + missing = append(missing, m) + } + } + sort.Strings(missing) + if len(missing) > 0 { + t.Errorf("methods never reached the runtime: %s", strings.Join(missing, ", ")) + } + for _, m := range []string{"StartAgentTokenRefresh", "StartCredentialWatchdog", "StartCopilotSessionRefresh", "StartTranscriptActivityRefresh"} { + if seen[m] { + t.Errorf("%s crossed the wire; Background loops are server-side", m) + } + } +} + +func TestResumeErrorsMapToSentinels(t *testing.T) { + h := newHarness(t) + h.rec.Add("alpha", "a1", crux.StateRunning) + for _, tc := range []struct{ in, want error }{ + {crux.ErrResumeBusy, crux.ErrResumeBusy}, + {crux.ErrResumeSessionGone, crux.ErrResumeSessionGone}, + {crux.ErrResumeUnsupported, crux.ErrResumeUnsupported}, + } { + h.rec.ResumeErr = tc.in + if err := h.rt.SendResumeKick("alpha", "m", "s"); !errors.Is(err, tc.want) { + t.Errorf("%v: got %v", tc.in, err) + } + } + h.rec.ResumeErr = errors.New("other") + err := h.rt.SendResumeKick("alpha", "m", "s") + var se *api.StatusError + if !errors.As(err, &se) || se.Status != http.StatusInternalServerError { + t.Errorf("plain error: got %v", err) + } + h.rec.ResumeErr = nil + if _, ok := h.rt.ResumeHandle("ghost"); ok { + t.Error("ResumeHandle for unknown agent reported ok") + } + if _, ok := h.rt.SessionID("ghost"); ok { + t.Error("SessionID for unknown agent reported ok") + } +} + +func TestAuthorizerFailsClosed(t *testing.T) { + h := newHarness(t) + h.rec.AuthorizeErr = errors.New("tier too low") + err := h.rt.AuthorizePROpen("alpha", 1000) + if err == nil || err.Error() != "tier too low" { + t.Errorf("403 reason not surfaced: %v", err) + } + h.ts.Close() + if err := h.rt.AuthorizeMerge("alpha", 1000, "o/r"); err == nil { + t.Error("unreachable server must deny") + } +} + +// TestUnreachableServerAnswersZero: every no-error read answers its zero +// value and unknown rather than panicking or guessing. +func TestUnreachableServerAnswersZero(t *testing.T) { + rt, err := New("127.0.0.1:1", "tok") + if err != nil { + t.Fatal(err) + } + rt.Timeout = 2 * time.Second + // Writes with an error return refuse with crux.ErrRuntimeUnavailable + // (hivecommons/hive#14138 stage 4: kicks are not queued) and the + // client's Status says the runtime is gone. + if err := rt.SendKick("a", "m"); !errors.Is(err, crux.ErrRuntimeUnavailable) || !errors.Is(err, api.ErrUnreachable) { + t.Fatalf("SendKick on dead runtime = %v", err) + } + if st := rt.Status(); st.Reachable || st.FailedCalls == 0 || st.LastError == "" { + t.Fatalf("Status after failures = %+v", st) + } + if err := rt.Stop("a"); !errors.Is(err, crux.ErrRuntimeUnavailable) { + t.Fatalf("Stop = %v", err) + } + if err := rt.Resume(context.Background(), "a", "t", "r"); !errors.Is(err, crux.ErrRuntimeUnavailable) { + t.Fatalf("Resume = %v", err) + } + if rt.IsPaused("a") || rt.AllStatuses() != nil || rt.ResolveAgent("a") != "a" || rt.FullLogHeader("a") != "" { + t.Error("observer reads must be zero") + } + if _, ok := rt.KickDispatchState("a"); ok { + t.Error("dispatch ok") + } + if rt.ReconcileAgents(map[string]config.AgentConfig{"a": {}}) != nil { + t.Error("reconcile") + } + if e, p := rt.BreakerState(); e || p != nil { + t.Error("breaker") + } + if rt.EngageBreaker() != nil || rt.GetACMMLevel() != 0 || rt.SharedDevUID() || rt.CopilotToken() != "" || rt.ClaudeCredentialCandidatePaths() != nil { + t.Error("fleet/credential reads") + } + if rt.GovernorMayStart("a") || rt.ConsentWedgedAgents() != nil || rt.CountAgentsWithModel() != 0 || rt.CrashOOMSuspected("a") || rt.AgentHasValidCredential("a") || rt.TrajectoryAgents() != nil { + t.Error("supervisor reads") + } + if _, known := rt.AgentCredentialRefreshable("a"); known { + t.Error("refreshable known") + } + if _, _, _, ok := rt.AgentCapabilities("a"); ok { + t.Error("caps ok") + } + if _, ok := rt.EffectiveBackend("a"); ok { + t.Error("backend ok") + } + if _, _, _, ok := rt.InvocationMetadata("a"); ok { + t.Error("invocation ok") + } + if _, _, _, _, ok := rt.RestartTelemetry("a"); ok { + t.Error("restarts ok") + } + if _, ok := rt.StartFailureState("a"); ok { + t.Error("start failure ok") + } + if _, _, _, ok := rt.ProviderErrorBackoffRemaining("a"); ok { + t.Error("backoff ok") + } + if _, known := rt.BackendAuthAvailable("claude"); known { + t.Error("backend auth known") + } + if _, known := rt.AgentAuthAvailable("a"); known { + t.Error("agent auth known") + } + if rt.SessionMissing("a") { + t.Error("session missing") + } + // Writes are dropped with a log line, never a panic. + rt.AddAgent("a", config.AgentConfig{}) + rt.RemoveAgent("a") + rt.ClearModeOverrides(nil) + rt.SyncModeFiles(1) + rt.RefreshInferenceRoutes("x") + rt.SetACMMLevel(1) + rt.SetCopilotToken("t") + rt.ReloadClaudeToken() + rt.MarkStartupLaunchQueued(nil) + rt.RefreshAgentTokens(context.Background()) + rt.ArchiveAllKickLogs("r") + rt.SendGitHubAPIBudgetNudge("a", "m") + rt.RecordAudit("x", "a", nil) + rt.SeedLastKick("a", time.Now()) + rt.RestoreBreaker(false, nil) + src := rt.WatchdogSource() + if src.AgentNames() != nil { + t.Error("watchdog names") + } + if _, err := src.Observe("a"); err == nil { + t.Error("observe must error") + } + if _, ok := src.LastProduction("a"); ok { + t.Error("production ok") + } + src.SetConditions("a", nil) +} + +func TestWatchdogSourceOverTheWire(t *testing.T) { + h := newHarness(t) + h.rec.Add("alpha", "a1", crux.StateRunning) + h.rec.WatchdogNames = []string{"alpha"} + when := time.Now().Truncate(time.Second) + h.rec.Observations = map[string]watchdog.Observation{"alpha": {Backend: "claude", SessionExists: true, Pane: "x", AuthKnown: true, LastChange: when}} + h.rec.Production = map[string]time.Time{"alpha": when} + src := h.rt.WatchdogSource() + if got := src.AgentNames(); !reflect.DeepEqual(got, []string{"alpha"}) { + t.Errorf("AgentNames %v", got) + } + o, err := src.Observe("alpha") + if err != nil || o.Backend != "claude" || !o.SessionExists || !o.AuthKnown || !o.LastChange.Equal(when) { + t.Errorf("Observe %+v %v", o, err) + } + if _, err := src.Observe("ghost"); err == nil { + t.Error("Observe unknown must error") + } + lp, ok := src.LastProduction("alpha") + if !ok || !lp.Equal(when) { + t.Errorf("LastProduction %v %v", lp, ok) + } + src.SetConditions("alpha", []crux.Condition{{Type: "Ready", Status: "True", Reason: "ok"}}) + if got := h.rec.SetConds["alpha"]; len(got) != 1 || got[0].Type != "Ready" { + t.Errorf("SetConditions %+v", got) + } + var _ watchdog.Source = src +} + +// TestKickObserverRidesTheEventStream runs a real Serve so the kick relay +// is installed, then checks a kick event reaches both the kick observer +// and the general event observer, and that clearing both stops the stream. +func TestKickObserverRidesTheEventStream(t *testing.T) { + rec := cruxtest.NewRecorder() + rec.Add("alpha", "a1", crux.StateRunning) + srv := serve.New(rec) + srv.PollInterval = 10 * time.Millisecond + dir, err := os.MkdirTemp("", "crux") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + socket := dir + "/crux.sock" + ls, err := serve.Listen(config.ServeConfig{Socket: socket}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- srv.Serve(ctx, ls) }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + t.Error(err) + } + }) + rt, err := New("unix://"+socket, "") + if err != nil { + t.Fatal(err) + } + rt.retry = 10 * time.Millisecond + if err := rt.Ready(ctx, 10*time.Millisecond); err != nil { + t.Fatal(err) + } + + var mu sync.Mutex + var kicks []string + var events []string + rt.SetKickObserver(func(agent, event, detail string) { + mu.Lock() + kicks = append(kicks, agent+"/"+event+"/"+detail) + mu.Unlock() + }) + rt.SetEventObserver(func(ev api.Event) { + mu.Lock() + events = append(events, string(ev.Type)) + mu.Unlock() + }) + // Fire until the subscriber is attached (the stream connects async). + tick := time.NewTicker(20 * time.Millisecond) + defer tick.Stop() + deadline := time.After(5 * time.Second) +poll: + for { + rec.EmitKick("alpha", "sent", "ok") + mu.Lock() + n := len(kicks) + mu.Unlock() + if n > 0 { + break + } + select { + case <-tick.C: + case <-deadline: + break poll + } + } + mu.Lock() + gotKick := len(kicks) > 0 && kicks[0] == "alpha/sent/ok" + gotEvent := false + for _, e := range events { + if e == string(api.EventKick) { + gotEvent = true + } + } + mu.Unlock() + if !gotKick { + t.Errorf("kick observer never fired: %v", kicks) + } + if !gotEvent { + t.Errorf("event observer never saw a kick: %v", events) + } + rt.SetKickObserver(nil) + rt.mu.Lock() + stillRunning := rt.cancelEvents != nil + rt.mu.Unlock() + if !stillRunning { + t.Error("stream stopped while the event observer is still set") + } + rt.SetEventObserver(nil) + rt.mu.Lock() + stopped := rt.cancelEvents == nil + rt.mu.Unlock() + if !stopped { + t.Error("stream still running after both observers cleared") + } + // Re-subscribing restarts it. + rt.SetKickObserver(func(string, string, string) {}) + rt.mu.Lock() + restarted := rt.cancelEvents != nil + rt.mu.Unlock() + if !restarted { + t.Error("stream not restarted") + } + rt.SetKickObserver(nil) +} + +func TestStreamReconnectsAfterDrop(t *testing.T) { + var mu sync.Mutex + connects := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + connects++ + mu.Unlock() + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("data: {\"type\":\"agent.kick\",\"agent\":\"a\",\"state\":\"sent\"}\n\n")) + })) + t.Cleanup(ts.Close) + rt, err := New(ts.URL, "") + if err != nil { + t.Fatal(err) + } + rt.retry = 5 * time.Millisecond + got := make(chan string, 16) + rt.SetKickObserver(func(agent, event, _ string) { got <- agent + "/" + event }) + for i := 0; i < 2; i++ { + select { + case s := <-got: + if s != "a/sent" { + t.Errorf("got %s", s) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for relayed kick") + } + } + rt.SetKickObserver(nil) + mu.Lock() + n := connects + mu.Unlock() + if n < 2 { + t.Errorf("expected a reconnect, got %d connects", n) + } +} + +func TestConstructors(t *testing.T) { + if _, err := New("://bad", ""); err == nil { + t.Error("bad address accepted") + } + c, err := api.NewClient("127.0.0.1:1", "t") + if err != nil { + t.Fatal(err) + } + rt := Wrap(c) + if rt.Client() != c { + t.Error("Client() mismatch") + } + rt.Logger = nil + if rt.logger() == nil { + t.Error("nil Logger must yield a discard logger") + } + rt.Timeout = 0 + ctx, cancel := rt.ctx() + defer cancel() + if dl, ok := ctx.Deadline(); !ok || time.Until(dl) > DefaultTimeout { + t.Error("default timeout not applied") + } + if got := tail([]string{"a", "b", "c"}, 0); len(got) != 3 { + t.Error("tail 0 must keep all") + } +} + +func TestSnapshotDecodeError(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case api.PathSnapshots: + _, _ = w.Write([]byte(`{"snapshots":[{"agent":"a","process":"notanobject"}]}`)) + default: + _, _ = w.Write([]byte(`{"agent":"a","process":"notanobject"}`)) + } + })) + t.Cleanup(ts.Close) + rt, err := New(ts.URL, "") + if err != nil { + t.Fatal(err) + } + if _, err := rt.GetStatus("a"); err == nil { + t.Error("bad snapshot must error") + } + if got := rt.AllStatuses(); len(got) != 0 { + t.Errorf("bad snapshots must be skipped, got %d", len(got)) + } +} diff --git a/src/crux/api/client_test.go b/src/crux/api/client_test.go new file mode 100644 index 0000000000..1b3357f1b8 --- /dev/null +++ b/src/crux/api/client_test.go @@ -0,0 +1,486 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// fakeServer is a hand-rolled handler speaking the wire contract, so the +// client is tested against the contract and not against crux/serve. +type fakeServer struct { + mu sync.Mutex + token string + requests []string + bodies []string + // streamHold, when set, keeps SSE handlers open until closed. Guarded + // by mu: a handler from an earlier stream may still be running when a + // test sets it. + streamHold chan struct{} +} + +func (f *fakeServer) handler() http.Handler { + mux := http.NewServeMux() + auth := func(h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + f.requests = append(f.requests, r.Method+" "+r.URL.RequestURI()) + f.mu.Unlock() + if f.token != "" && r.Header.Get("Authorization") != "Bearer "+f.token { + w.WriteHeader(http.StatusUnauthorized) + _ = json.NewEncoder(w).Encode(Error{Error: "token required", Code: CodeUnauthorized}) + return + } + h(w, r) + } + } + writeJSON := func(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", ContentTypeJSON) + _ = json.NewEncoder(w).Encode(v) + } + mux.HandleFunc("GET "+PathHealth, auth(func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, Health{Version: Version, Agents: 2}) + })) + mux.HandleFunc("GET "+PathAgents, auth(func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, ProcessList{Processes: []Process{{Name: "a"}, {Name: "b"}}}) + })) + mux.HandleFunc("POST "+PathAgents, auth(func(w http.ResponseWriter, r *http.Request) { + var req RunRequest + _ = json.NewDecoder(r.Body).Decode(&req) + f.mu.Lock() + f.bodies = append(f.bodies, string(req.Definition)) + f.mu.Unlock() + if req.Name == "boom" { + w.WriteHeader(http.StatusInternalServerError) + writeJSON(w, Error{Error: "start boom: tmux is down", Code: CodeRuntime}) + return + } + writeJSON(w, AgentResponse{Agent: req.Name, Process: &Process{Name: req.Name, State: "running"}}) + })) + mux.HandleFunc("GET "+PathAgents+"/{name}", auth(func(w http.ResponseWriter, r *http.Request) { + if r.PathValue("name") == "ghost" { + w.WriteHeader(http.StatusNotFound) + writeJSON(w, Error{Error: "agent ghost not found", Code: CodeNotFound}) + return + } + writeJSON(w, Process{Name: r.PathValue("name"), State: "running"}) + })) + mux.HandleFunc("GET "+PathAgents+"/{name}/logs", auth(func(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if r.URL.Query().Get(QueryFollow) == "" { + writeJSON(w, Logs{Agent: name, Source: r.URL.Query().Get(QuerySource), Lines: []string{"n=" + r.URL.Query().Get(QueryLines)}}) + return + } + w.Header().Set("Content-Type", ContentTypeEvent) + fl := w.(http.Flusher) + _ = WriteEvent(w, EventLog, LogChunk{Agent: name, Lines: []string{"one", "two"}}) + _ = WriteEvent(w, "future.type", Event{Type: "future.type"}) // ignored by the client + fl.Flush() + if name == "lost" { + _ = WriteEvent(w, EventState, Event{Type: EventState, Agent: name, Detail: "agent lost not found"}) + fl.Flush() + return + } + if name == "garbage" { + _, _ = w.Write([]byte("event: " + EventLog + "\ndata: {not json\n\n")) + fl.Flush() + return + } + if hold := f.hold(); hold != nil { + select { + case <-hold: + case <-r.Context().Done(): + } + } + })) + mux.HandleFunc("POST "+PathAgents+"/{name}/kick", auth(func(w http.ResponseWriter, r *http.Request) { + var req KickRequest + _ = json.NewDecoder(r.Body).Decode(&req) + f.mu.Lock() + f.bodies = append(f.bodies, req.Message) + f.mu.Unlock() + writeJSON(w, AgentResponse{Agent: r.PathValue("name")}) + })) + mux.HandleFunc("POST "+PathAgents+"/{name}/stop", auth(func(w http.ResponseWriter, r *http.Request) { + if r.PathValue("name") == "plain" { + // A non-JSON error body: the client must still surface the status. + http.Error(w, "nope", http.StatusBadGateway) + return + } + writeJSON(w, AgentResponse{Agent: r.PathValue("name")}) + })) + mux.HandleFunc("GET "+PathEvents, auth(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", ContentTypeEvent) + fl := w.(http.Flusher) + _ = WriteEvent(w, EventStarted, Event{Type: EventStarted, Agent: "a"}) + // A record typed only by the event: line. + _, _ = w.Write([]byte("event: " + EventKick + "\ndata: {\"agent\":\"a\"}\n\n")) + if r.URL.Query().Get("garbage") != "" { + _, _ = w.Write([]byte("event: x\ndata: {\n\n")) + } + fl.Flush() + if hold := f.hold(); hold != nil { + select { + case <-hold: + case <-r.Context().Done(): + } + } + })) + return mux +} + +func (f *fakeServer) hold() chan struct{} { + f.mu.Lock() + defer f.mu.Unlock() + return f.streamHold +} + +func (f *fakeServer) seen() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.requests...) +} + +// startUnix serves f on a unix socket and returns a client for it. +func startUnix(t *testing.T, f *fakeServer, token string) *Client { + t.Helper() + dir, err := os.MkdirTemp("", "cruxapi") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + path := filepath.Join(dir, "s.sock") + ln, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewUnstartedServer(f.handler()) + srv.Listener = ln + srv.Start() + t.Cleanup(srv.Close) + c, err := NewClient(path, token) + if err != nil { + t.Fatal(err) + } + return c +} + +func TestClientVerbsOverUnixSocket(t *testing.T) { + f := &fakeServer{token: "s3cret"} + c := startUnix(t, f, "s3cret") + ctx := context.Background() + + if c.Address().Network != "unix" { + t.Fatalf("address %+v", c.Address()) + } + h, err := c.Health(ctx) + if err != nil || h.Agents != 2 || h.Version != Version { + t.Fatalf("health %+v %v", h, err) + } + procs, err := c.List(ctx) + if err != nil || len(procs) != 2 { + t.Fatalf("list %+v %v", procs, err) + } + p, err := c.Get(ctx, "a b") + if err != nil || p.Name != "a b" { + t.Fatalf("get %+v %v", p, err) + } + resp, err := c.Run(ctx, RunRequest{Name: "fixer", Definition: json.RawMessage(`{"backend":"claude"}`)}) + if err != nil || resp.Agent != "fixer" || resp.Process == nil { + t.Fatalf("run %+v %v", resp, err) + } + logs, err := c.Logs(ctx, "fixer", LogSourceBuffer, 7) + if err != nil || logs.Source != LogSourceBuffer || logs.Lines[0] != "n=7" { + t.Fatalf("logs %+v %v", logs, err) + } + if logs, err = c.Logs(ctx, "fixer", "", 0); err != nil || logs.Lines[0] != "n=" { + t.Fatalf("default logs %+v %v", logs, err) + } + if resp, err = c.Kick(ctx, "fixer", "go"); err != nil || resp.Agent != "fixer" { + t.Fatalf("kick %+v %v", resp, err) + } + if resp, err = c.Stop(ctx, "fixer"); err != nil || resp.Agent != "fixer" { + t.Fatalf("stop %+v %v", resp, err) + } + + seen := strings.Join(f.seen(), "\n") + for _, want := range []string{"GET /v1/health", "GET /v1/agents\n", "GET /v1/agents/a%20b", "POST /v1/agents\n", + "GET /v1/agents/fixer/logs?n=7&source=buffer", "GET /v1/agents/fixer/logs?", "POST /v1/agents/fixer/kick", "POST /v1/agents/fixer/stop"} { + if !strings.Contains(seen+"\n", want) { + t.Errorf("missing %q in\n%s", want, seen) + } + } + f.mu.Lock() + bodies := strings.Join(f.bodies, "|") + f.mu.Unlock() + if bodies != `{"backend":"claude"}|go` { + t.Fatalf("bodies %q", bodies) + } +} + +func TestClientErrors(t *testing.T) { + f := &fakeServer{token: "s3cret"} + ctx := context.Background() + + // Wrong token: 401 with the server's api.Error. + c := startUnix(t, f, "wrong") + _, err := c.Health(ctx) + if !IsUnauthorized(err) || IsNotFound(err) || err.Error() != "token required" { + t.Fatalf("unauthorized: %v", err) + } + var se *StatusError + if !errors.As(err, &se) || se.Status != http.StatusUnauthorized || se.Body.Code != CodeUnauthorized { + t.Fatalf("status error %+v", se) + } + if err := c.FollowLogs(ctx, "x", func(LogChunk) error { return nil }); !IsUnauthorized(err) { + t.Fatalf("follow unauthorized: %v", err) + } + + c = startUnix(t, f, "s3cret") + if _, err := c.Get(ctx, "ghost"); !IsNotFound(err) || err.Error() != "agent ghost not found" { + t.Fatalf("not found: %v", err) + } + if _, err := c.Run(ctx, RunRequest{Name: "boom"}); err == nil || !strings.Contains(err.Error(), "tmux is down") { + t.Fatalf("runtime error: %v", err) + } + // Non-JSON error body falls back to the status. + if _, err := c.Stop(ctx, "plain"); err == nil || !strings.Contains(err.Error(), "HTTP 502") { + t.Fatalf("plain error: %v", err) + } + if IsNotFound(errors.New("x")) || IsUnauthorized(nil) { + t.Fatal("classifiers on non-status errors") + } + + // Unreachable socket. + dead, err := NewClient(filepath.Join(os.TempDir(), "crux-no-such.sock"), "") + if err != nil { + t.Fatal(err) + } + if _, err := dead.List(ctx); err == nil || !strings.Contains(err.Error(), "unix://") { + t.Fatalf("dead socket: %v", err) + } + if err := dead.Events(ctx, func(Event) error { return nil }); err == nil { + t.Fatal("dead socket events should fail") + } + + // Bad address and bad request construction. + if _, err := NewClient("", ""); err == nil { + t.Fatal("empty address should fail") + } + bad, _ := NewClient("127.0.0.1:1", "") + bad.addr.BaseURL = "http://[::1" + if _, err := bad.Health(ctx); err == nil { + t.Fatal("unparseable base URL should fail") + } + if err := bad.Events(ctx, func(Event) error { return nil }); err == nil { + t.Fatal("unparseable base URL should fail for streams") + } + if _, err := bad.Run(ctx, RunRequest{Name: "x", Definition: json.RawMessage(`{`)}); err == nil { + t.Fatal("unmarshalable body should fail") + } + + // A JSON body that is not the expected shape. + srvBad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("[not an object")) + })) + t.Cleanup(srvBad.Close) + c, err = NewClient(srvBad.URL, "") + if err != nil { + t.Fatal(err) + } + if _, err := c.Health(ctx); err == nil || !strings.Contains(err.Error(), "decoding GET") { + t.Fatalf("decode error: %v", err) + } +} + +func TestClientStreams(t *testing.T) { + f := &fakeServer{} + c := startUnix(t, f, "") + ctx := context.Background() + + var lines []string + if err := c.FollowLogs(ctx, "fixer", func(ch LogChunk) error { + lines = append(lines, ch.Lines...) + return nil + }); err != nil { + t.Fatal(err) + } + if strings.Join(lines, ",") != "one,two" { + t.Fatalf("lines %v", lines) + } + + // The server reports the agent gone: the stream ends with an error. + err := c.FollowLogs(ctx, "lost", func(LogChunk) error { return nil }) + if err == nil || !strings.Contains(err.Error(), "agent lost not found") { + t.Fatalf("lost: %v", err) + } + if err := c.FollowLogs(ctx, "garbage", func(LogChunk) error { return nil }); err == nil || !strings.Contains(err.Error(), "log stream") { + t.Fatalf("garbage: %v", err) + } + // fn's error is returned. + stop := errors.New("enough") + if err := c.FollowLogs(ctx, "fixer", func(LogChunk) error { return stop }); !errors.Is(err, stop) { + t.Fatalf("fn error: %v", err) + } + + var evs []Event + if err := c.Events(ctx, func(ev Event) error { + evs = append(evs, ev) + return nil + }); err != nil { + t.Fatal(err) + } + if len(evs) != 2 || evs[0].Type != EventStarted || evs[1].Type != EventKick || evs[1].Agent != "a" { + t.Fatalf("events %+v", evs) + } + // Garbage on a stream read through the generic reader surfaces fn's error. + err = c.stream(ctx, PathEvents+"?garbage=1", func(_ string, data []byte) error { + var ev Event + return json.Unmarshal(data, &ev) + }) + if err == nil { + t.Fatal("garbage via stream should fail") + } + + // Cancelling the context ends a held stream without an error. + hold := make(chan struct{}) + f.mu.Lock() + f.streamHold = hold + f.mu.Unlock() + t.Cleanup(func() { close(hold) }) + cctx, cancel := context.WithCancel(ctx) + got := make(chan struct{}, 1) + done := make(chan error, 1) + go func() { + done <- c.FollowLogs(cctx, "fixer", func(LogChunk) error { + select { + case got <- struct{}{}: + default: + } + return nil + }) + }() + select { + case <-got: + case <-time.After(5 * time.Second): + t.Fatal("no chunk before cancel") + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("cancelled follow: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("follow did not end on cancel") + } +} + +func TestClientEventsDecodeError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", ContentTypeEvent) + _, _ = w.Write([]byte("event: x\ndata: {\n\n")) + })) + t.Cleanup(srv.Close) + c, err := NewClient(srv.URL, "tok") + if err != nil { + t.Fatal(err) + } + if c.Address().Network != "tcp" { + t.Fatalf("address %+v", c.Address()) + } + if err := c.Events(context.Background(), func(Event) error { return nil }); err == nil || !strings.Contains(err.Error(), "event stream") { + t.Fatalf("decode error: %v", err) + } + // TCP error target is the base URL. + deadTCP, _ := NewClient("127.0.0.1:1", "") + if _, err := deadTCP.Health(context.Background()); err == nil || !strings.Contains(err.Error(), "http://127.0.0.1:1") { + t.Fatalf("dead tcp: %v", err) + } +} + +func TestClientSetHTTPClientAndWaitReady(t *testing.T) { + f := &fakeServer{} + c := startUnix(t, f, "") + // A replacement client without a transport keeps the socket dialer. + c.SetHTTPClient(&http.Client{Timeout: 5 * time.Second}) + if _, err := c.Health(context.Background()); err != nil { + t.Fatal(err) + } + // One with its own transport is used as given. + c.SetHTTPClient(&http.Client{Transport: http.DefaultTransport}) + if _, err := c.Health(context.Background()); err == nil { + t.Fatal("default transport cannot reach a unix base URL") + } + + c = startUnix(t, f, "") + if err := c.WaitReady(context.Background(), 0); err != nil { + t.Fatal(err) + } + dead, _ := NewClient(filepath.Join(os.TempDir(), "crux-no-such.sock"), "") + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + if err := dead.WaitReady(ctx, 20*time.Millisecond); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("dead wait: %v", err) + } + // Unauthorized is final: polling cannot fix it. + wrong := startUnix(t, &fakeServer{token: "t"}, "bad") + if err := wrong.WaitReady(context.Background(), time.Hour); !IsUnauthorized(err) { + t.Fatalf("wrong token wait: %v", err) + } +} + +func TestClientStatusTracksReachability(t *testing.T) { + f := &fakeServer{} + c := startUnix(t, f, "") + if st := c.Status(); !st.Reachable || st.FailedCalls != 0 { + t.Fatalf("fresh client status = %+v, want reachable", st) + } + dead, _ := NewClient(filepath.Join(os.TempDir(), "crux-no-such.sock"), "") + if _, err := dead.Health(context.Background()); !errors.Is(err, ErrUnreachable) { + t.Fatalf("dead Health = %v, want ErrUnreachable", err) + } + if _, err := dead.Health(context.Background()); !errors.Is(err, ErrUnreachable) { + t.Fatalf("dead Health again = %v", err) + } + st := dead.Status() + if st.Reachable || st.FailedCalls != 2 || st.Since.IsZero() || !strings.Contains(st.LastError, "crux-no-such.sock") { + t.Fatalf("dead status = %+v", st) + } + // Streams record the same way. + if err := dead.Events(context.Background(), func(Event) error { return nil }); !errors.Is(err, ErrUnreachable) { + t.Fatalf("dead Events = %v", err) + } + if st := dead.Status(); st.FailedCalls != 3 { + t.Fatalf("stream failure not counted: %+v", st) + } + // A caller cancelling its own context is not a reachability signal. + cctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := c.Health(cctx); err == nil { + t.Fatal("cancelled ctx must fail") + } + if st := c.Status(); !st.Reachable { + t.Fatalf("cancelled call flipped status: %+v", st) + } + // Protocol errors from a live server are not transport failures. + if _, err := startUnix(t, &fakeServer{token: "t"}, "bad").Health(context.Background()); !IsUnauthorized(err) { + t.Fatalf("want unauthorized, got %v", err) + } + // Success resets. + if _, err := c.Health(context.Background()); err != nil { + t.Fatal(err) + } + if st := c.Status(); !st.Reachable || st.LastSuccess.IsZero() || st.LastError != "" { + t.Fatalf("status after success = %+v", st) + } +} diff --git a/src/crux/api/runtime.go b/src/crux/api/runtime.go new file mode 100644 index 0000000000..50442d46f5 --- /dev/null +++ b/src/crux/api/runtime.go @@ -0,0 +1,848 @@ +package api + +import ( + "encoding/json" + "time" +) + +// This file is the wire contract for the rest of crux.Runtime +// (hivecommons/hive#14138, stage 3): every family the CLI verbs do not cover +// (Lifecycle beyond run/stop, the full Observer snapshot, Kicker's async and +// archive reads, Definitions, Fleet, Credentials, Resumer, Supervisor, +// Restorer, Background, Authorizer) plus the watchdog's observe/restart seam. +// It stays standard-library only like the rest of the package: payloads the +// runtime owns the shape of (a crux.Process snapshot, a config.AgentConfig +// definition, a crux.TurnLoss record) cross as json.RawMessage in the +// runtime's own JSON form, the way RunRequest.Definition already does; small +// values are mirrored here field for field so the wire shape is pinned +// independently of the runtime's types. +// +// RuntimeRoutes is the parity table: every crux.Runtime method names the +// route that serves it. crux/api/client's tests assert the table and the +// interface agree in both directions; crux/serve's tests assert every route +// in the table is registered. + +// Routes beyond the CLI-shaped ones in api.go. Agent routes take the agent's +// name or ID in place of {name}. +const ( + PathSnapshots = "/" + Version + "/snapshots" + PathDefinitions = "/" + Version + "/definitions" + PathReconcile = PathDefinitions + "/reconcile" + PathFleet = "/" + Version + "/fleet" + PathBreaker = PathFleet + "/breaker" + PathBreakerOn = PathBreaker + "/engage" + PathBreakerOff = PathBreaker + "/release" + PathACMM = PathFleet + "/acmm" + PathModesSync = PathFleet + "/modes/sync" + PathModesClear = PathFleet + "/modes/clear" + PathInference = PathFleet + "/inference/refresh" + PathRelaunchBob = PathFleet + "/relaunch-bob" + // PathStartupQueued records the host's staggered boot queue. + PathStartupQueued = PathFleet + "/startup-queued" + // PathRestartCrashed relaunches crashed agents. + PathRestartCrashed = PathFleet + "/restart-crashed" + // PathTokensRefresh rewrites every agent's scoped token. + PathTokensRefresh = PathFleet + "/tokens/refresh" + // PathKickLogsArchive rotates every agent's kick log. + PathKickLogsArchive = PathFleet + "/kicklogs/archive" + PathCopilotCred = "/" + Version + "/credentials/copilot" + PathClaudeCred = "/" + Version + "/credentials/claude" + PathClaudeReload = PathClaudeCred + "/reload" + PathOmpHomes = "/" + Version + "/credentials/omp/homes" + PathBackends = "/" + Version + "/backends" + PathAudit = "/" + Version + "/audit" + PathTrajectory = "/" + Version + "/trajectory" + PathAuthorize = "/" + Version + "/authorize" + PathWatchdogAgents = "/" + Version + "/watchdog/agents" +) + +// Per-agent routes. +func StartPath(name string) string { return AgentPath(name) + "/start" } +func RestartPath(name string) string { return AgentPath(name) + "/restart" } +func PausePath(name string) string { return AgentPath(name) + "/pause" } +func ResumePath(name string) string { return AgentPath(name) + "/resume" } +func RestartCountPath(name string) string { return AgentPath(name) + "/restart-count" } +func SnapshotPath(name string) string { return AgentPath(name) + "/snapshot" } +func KickValidatePath(name string) string { return KickPath(name) + "/validate" } +func KickLogsPath(name string) string { return AgentPath(name) + "/kicklogs" } +func KickLogPath(name, id string) string { return KickLogsPath(name) + "/" + id } +func DefinitionPath(name string) string { return AgentPath(name) + "/definition" } +func OverridesPath(name string) string { return AgentPath(name) + "/overrides" } +func PinPath(name, kind string) string { return AgentPath(name) + "/pins/" + kind } +func LoginCodePath(name string) string { return AgentPath(name) + "/login-code" } +func SessionPath(name string) string { return AgentPath(name) + "/session" } +func SessionKickPath(name string) string { return SessionPath(name) + "/kick" } +func SupervisionPath(name string) string { return AgentPath(name) + "/supervision" } +func AgentTokenRefreshPath(name string) string { return AgentPath(name) + "/tokens/refresh" } +func NudgePath(name string) string { return AgentPath(name) + "/nudge" } +func SeedPath(name string) string { return AgentPath(name) + "/seed" } +func WatchdogPath(name string) string { return AgentPath(name) + "/watchdog" } +func WatchdogConditionsPath(name string) string { return WatchdogPath(name) + "/conditions" } + +// BackendAuthPath is the route reporting whether a backend has credentials. +func BackendAuthPath(backend string) string { return PathBackends + "/" + backend + "/auth" } + +// Pin kinds, the {kind} of the pins route. +const ( + PinCLI = "cli" + PinModel = "model" +) + +// Query parameters of the snapshot route. +const ( + // QueryFast asks for the snapshot without a pane re-read + // (Observer.GetStatusFast). + QueryFast = "fast" +) + +// Snapshot is the body of GET /v1/agents/{name}/snapshot: the runtime's full +// crux.Process for one agent in its own JSON form (the compact api.Process +// is what GET /v1/agents/{name} returns), plus the pane capture the snapshot +// was taken with, which crux.Process keeps unexported. Process never carries +// the output ring buffer; the logs route serves it. +type Snapshot struct { + Agent string `json:"agent"` + Process json.RawMessage `json:"process"` + Pane []string `json:"pane,omitempty"` +} + +// SnapshotList is the body of GET /v1/snapshots (Observer.AllStatuses), +// sorted by agent. +type SnapshotList struct { + Snapshots []Snapshot `json:"snapshots"` +} + +// RestartRequest is the body of POST /v1/agents/{name}/restart. A non-empty +// Bootstrap selects Lifecycle.RestartWithBootstrap. +type RestartRequest struct { + Bootstrap string `json:"bootstrap,omitempty"` +} + +// PauseRequest is the body of POST /v1/agents/{name}/pause. By selects +// Lifecycle.PauseBy; Cause selects Supervisor.PauseByCause (By may then be +// empty, as the runtime allows). +type PauseRequest struct { + Trigger string `json:"trigger"` + Reason string `json:"reason"` + By string `json:"by,omitempty"` + Cause *PauseCausation `json:"cause,omitempty"` +} + +// PauseCausation mirrors crux.PauseCausation. +type PauseCausation struct { + Depth int `json:"depth"` + HookName string `json:"hook_name,omitempty"` + OriginTransition string `json:"origin_transition,omitempty"` +} + +// ResumeRequest is the body of POST /v1/agents/{name}/resume. +type ResumeRequest struct { + Trigger string `json:"trigger"` + Reason string `json:"reason"` +} + +// KickDispatch mirrors crux.KickDispatch. +type KickDispatch struct { + Agent string `json:"agent"` + Phase string `json:"phase"` + Error string `json:"error,omitempty"` + QueuedAt time.Time `json:"queuedAt"` + SettledAt time.Time `json:"settledAt,omitempty"` +} + +// KickDispatchResponse is the body of GET /v1/agents/{name}/kick +// (Kicker.KickDispatchState); Dispatch is nil when the agent has none. +type KickDispatchResponse struct { + Agent string `json:"agent"` + Dispatch *KickDispatch `json:"dispatch,omitempty"` +} + +// KickLogInfo mirrors crux.KickLogInfo. +type KickLogInfo struct { + ID string `json:"id"` + Timestamp time.Time `json:"timestamp"` + SizeBytes int64 `json:"size_bytes"` + Reason string `json:"reason"` + Source string `json:"source,omitempty"` +} + +// KickLogList is the body of GET /v1/agents/{name}/kicklogs. +type KickLogList struct { + Agent string `json:"agent"` + Logs []KickLogInfo `json:"logs"` +} + +// KickLog is the body of GET /v1/agents/{name}/kicklogs/{id}. +type KickLog struct { + Agent string `json:"agent"` + ID string `json:"id"` + Content string `json:"content"` +} + +// DefinitionRequest is the body of POST /v1/definitions +// (Definitions.AddAgent). Definition is the JSON form of config.AgentConfig +// and is registered verbatim: unlike POST /v1/agents, no defaults are filled, +// because the caller is a host handing over a complete definition. The +// response is 201 when the name was new and 200 when the runtime already +// held it (AddAgent is a no-op then). PUT /v1/agents/{name}/definition +// (Definitions.UpdateConfig) takes the same body with Name ignored. +type DefinitionRequest struct { + Name string `json:"name"` + Definition json.RawMessage `json:"definition"` +} + +// ReconcileRequest is the body of POST /v1/definitions/reconcile +// (Definitions.ReconcileAgents): the host's full definition set, each in +// the JSON form of config.AgentConfig. +type ReconcileRequest struct { + Definitions map[string]json.RawMessage `json:"definitions"` +} + +// Names is the body of responses that list agent names (reconcile, breaker +// release, relaunch, restart-crashed, watchdog agents) and of requests that +// take one (modes clear, startup queued). +type Names struct { + Names []string `json:"names"` +} + +// OverridesRequest is the body of PUT /v1/agents/{name}/overrides. Each +// pointer that is set is applied (Definitions.SetModelOverride / +// SetBackendOverride); nil leaves the override alone. An empty string clears +// it, as the runtime does. +type OverridesRequest struct { + Model *string `json:"model,omitempty"` + Backend *string `json:"backend,omitempty"` +} + +// PinRequest is the body of PUT /v1/agents/{name}/pins/{kind} +// (Definitions.PinCLI / PinModel); DELETE on the same route unpins. +type PinRequest struct { + Value string `json:"value"` +} + +// Level is the body of PUT /v1/fleet/acmm (Fleet.SetACMMLevel) and POST +// /v1/fleet/modes/sync (Definitions.SyncModeFiles). +type Level struct { + Level int `json:"level"` +} + +// Backend is the body of POST /v1/fleet/inference/refresh +// (Definitions.RefreshInferenceRoutes). +type Backend struct { + Backend string `json:"backend"` +} + +// Breaker is the fleet breaker as GET /v1/fleet/breaker reports it +// (Fleet.BreakerState), as POST /v1/fleet/breaker/engage returns it, and as +// PUT /v1/fleet/breaker restores it (Restorer.RestoreBreaker). +type Breaker struct { + Engaged bool `json:"engaged"` + Paused []string `json:"paused"` +} + +// FleetInfo is the body of GET /v1/fleet: the fleet-wide reads that have no +// agent (Fleet.GetACMMLevel, Fleet.SharedDevUID, +// Supervisor.ConsentWedgedAgents, Supervisor.CountAgentsWithModel). +type FleetInfo struct { + ACMMLevel int `json:"acmm_level"` + SharedDevUID bool `json:"shared_dev_uid"` + AgentsWithModel int `json:"agents_with_model"` + ConsentWedged []string `json:"consent_wedged"` +} + +// Reason is the body of POST /v1/fleet/kicklogs/archive +// (Supervisor.ArchiveAllKickLogs). +type Reason struct { + Reason string `json:"reason"` +} + +// LoginCodeRequest is the body of POST /v1/agents/{name}/login-code +// (Credentials.SubmitLoginCode). +type LoginCodeRequest struct { + Code string `json:"code"` +} + +// CopilotCredential is the body of GET /v1/credentials/copilot +// (Credentials.CopilotToken, CopilotTokenSource) and of PUT on the same +// route: Activate false is Credentials.SetCopilotToken, true is +// ActivateCopilotToken. Source is read-only. +type CopilotCredential struct { + Token string `json:"token"` + Source string `json:"source,omitempty"` + Activate bool `json:"activate,omitempty"` +} + +// ClaudeCredential is the body of GET /v1/credentials/claude +// (Credentials.ClaudeCredentialCandidatePaths). +type ClaudeCredential struct { + CandidatePaths []string `json:"candidate_paths"` +} + +// OmpHomes is the body of GET /v1/credentials/omp/homes +// (Credentials.OmpAgentHomes). +type OmpHomes struct { + Homes []string `json:"homes"` +} + +// ResumeHandle mirrors crux.ResumeHandle. +type ResumeHandle struct { + Backend string `json:"backend"` + SessionID string `json:"session_id"` + Transcript string `json:"transcript,omitempty"` + Command string `json:"command,omitempty"` + LastActive time.Time `json:"last_active"` + CapturedAt time.Time `json:"captured_at"` +} + +// Session is the body of GET /v1/agents/{name}/session: the Resumer reads +// (SessionID, SessionMissing, ResumeHandle) in one object. Handle is nil +// when the agent has no resumable conversation. +type Session struct { + Agent string `json:"agent"` + SessionID string `json:"session_id,omitempty"` + HasSession bool `json:"has_session"` + Missing bool `json:"missing"` + Handle *ResumeHandle `json:"handle,omitempty"` +} + +// ResumeKickRequest is the body of POST /v1/agents/{name}/session/kick +// (Resumer.SendResumeKick). The response distinguishes the runtime's resume +// sentinels by status: 409 CodeResumeBusy, 410 CodeResumeSessionGone, 422 +// CodeResumeUnsupported. +type ResumeKickRequest struct { + Message string `json:"message"` + SessionID string `json:"session_id"` +} + +// StartFailureSnapshot mirrors crux.StartFailureSnapshot. +type StartFailureSnapshot struct { + Reason string `json:"reason"` + Count int `json:"count"` + LastAt time.Time `json:"last_at"` + LastExitCode *int `json:"last_exit_code,omitempty"` + LastSignal string `json:"last_signal,omitempty"` + Blocked bool `json:"blocked"` +} + +// RestartTelemetry mirrors Supervisor.RestartTelemetry's results. +type RestartTelemetry struct { + Total int `json:"total"` + Last24h int `json:"last_24h"` + LastRestartAt time.Time `json:"last_restart_at"` + LastReason string `json:"last_reason,omitempty"` +} + +// ProviderBackoff mirrors Supervisor.ProviderErrorBackoffRemaining's +// results; RemainingMS is milliseconds. +type ProviderBackoff struct { + RemainingMS int64 `json:"remaining_ms"` + Provider string `json:"provider,omitempty"` + Reason string `json:"reason,omitempty"` +} + +// Supervision is the body of GET /v1/agents/{name}/supervision: every +// per-agent Supervisor read in one object, so the host's heartbeat takes one +// round trip per agent. Each optional field is nil when the runtime's ok +// result was false. +type Supervision struct { + Agent string `json:"agent"` + // MayStart is Supervisor.GovernorMayStart. + MayStart bool `json:"may_start"` + // CredentialRefreshable / CredentialKnown are AgentCredentialRefreshable. + CredentialRefreshable bool `json:"credential_refreshable"` + CredentialKnown bool `json:"credential_known"` + // HasValidCredential is AgentHasValidCredential. + HasValidCredential bool `json:"has_valid_credential"` + // Capabilities is AgentCapabilities; nil when ok was false. + Capabilities *Capabilities `json:"capabilities,omitempty"` + // Backend is EffectiveBackend; nil when ok was false. + Backend *string `json:"backend,omitempty"` + // Invocation is InvocationMetadata; nil when ok was false. + Invocation *Invocation `json:"invocation,omitempty"` + // Restarts is RestartTelemetry; nil when ok was false. + Restarts *RestartTelemetry `json:"restarts,omitempty"` + // StartFailure is StartFailureState; nil when none. + StartFailure *StartFailureSnapshot `json:"start_failure,omitempty"` + // ProviderBackoff is ProviderErrorBackoffRemaining; nil when none. + ProviderBackoff *ProviderBackoff `json:"provider_backoff,omitempty"` + // CrashOOMSuspected is CrashOOMSuspected. + CrashOOMSuspected bool `json:"crash_oom_suspected"` + // AuthAvailable / AuthKnown are AgentAuthAvailable. + AuthAvailable bool `json:"auth_available"` + AuthKnown bool `json:"auth_known"` +} + +// Capabilities mirrors Supervisor.AgentCapabilities' bits. +type Capabilities struct { + CanOpenIssue bool `json:"can_open_issue"` + CanOpenPR bool `json:"can_open_pr"` + CanMerge bool `json:"can_merge"` +} + +// Invocation mirrors Supervisor.InvocationMetadata's results. +type Invocation struct { + Backend string `json:"backend"` + Model string `json:"model,omitempty"` + Effort string `json:"effort,omitempty"` +} + +// BackendAuth is the body of GET /v1/backends/{backend}/auth +// (Supervisor.BackendAuthAvailable). +type BackendAuth struct { + Backend string `json:"backend"` + Available bool `json:"available"` + Known bool `json:"known"` +} + +// NudgeRequest is the body of POST /v1/agents/{name}/nudge +// (Supervisor.SendGitHubAPIBudgetNudge). +type NudgeRequest struct { + Message string `json:"message"` +} + +// AuditRequest is the body of POST /v1/audit (Supervisor.RecordAudit). +type AuditRequest struct { + Action string `json:"action"` + Agent string `json:"agent,omitempty"` + Fields map[string]any `json:"fields,omitempty"` +} + +// TrajectoryAgent mirrors trajectory.AgentView. +type TrajectoryAgent struct { + Name string `json:"name"` + Running bool `json:"running"` + Intent string `json:"intent,omitempty"` + Transcript string `json:"transcript,omitempty"` + Idle bool `json:"idle"` + LastOutputAt time.Time `json:"last_output_at"` +} + +// TrajectoryList is the body of GET /v1/trajectory +// (Supervisor.TrajectoryAgents). +type TrajectoryList struct { + Agents []TrajectoryAgent `json:"agents"` +} + +// RestartEvent mirrors crux.RestartEvent. +type RestartEvent struct { + At time.Time `json:"at"` + Reason string `json:"reason"` +} + +// KickRecord mirrors crux.KickRecord. +type KickRecord struct { + Timestamp time.Time `json:"timestamp"` + Agent string `json:"agent"` + Snippet string `json:"snippet"` +} + +// PauseSeed is Restorer.SeedPauseState's arguments. +type PauseSeed struct { + PausedAt time.Time `json:"paused_at"` + Trigger string `json:"trigger"` + Reason string `json:"reason"` + By string `json:"by,omitempty"` +} + +// RestartSeed is Restorer.SeedRestartTelemetry's arguments. +type RestartSeed struct { + Count int `json:"count"` + Events []RestartEvent `json:"events"` + LastReason string `json:"last_reason,omitempty"` +} + +// SeedRequest is the body of POST /v1/agents/{name}/seed: the Restorer's +// per-agent seeds, each applied when present. TurnLoss is the JSON form of +// crux.TurnLoss, the runtime's own persisted shape. KickHistory being +// present but empty replaces the history with nothing. +type SeedRequest struct { + Pause *PauseSeed `json:"pause,omitempty"` + Restarts *RestartSeed `json:"restarts,omitempty"` + TurnLoss json.RawMessage `json:"turn_loss,omitempty"` + LastKick *time.Time `json:"last_kick,omitempty"` + KickHistory *[]KickRecord `json:"kick_history,omitempty"` +} + +// Authorize actions, the Action of AuthorizeRequest; one per Authorizer +// method. +const ( + AuthorizePROpen = "pr_open" + AuthorizePushBranch = "push_branch" + AuthorizeIssueOpen = "issue_open" + AuthorizeMerge = "merge" + AuthorizeReviewRequest = "review_request" +) + +// AuthorizeRequest is the body of POST /v1/authorize, the synchronous gate +// on the host's relay request path. Kind is for AuthorizeIssueOpen, Repo for +// AuthorizeMerge. 204 means permitted; 403 CodeForbidden carries the +// runtime's refusal; any other status is also a refusal to the caller. +type AuthorizeRequest struct { + Action string `json:"action"` + Agent string `json:"agent"` + FileUID int `json:"file_uid"` + Kind string `json:"kind,omitempty"` + Repo string `json:"repo,omitempty"` +} + +// WatchdogObservation is the body of GET /v1/agents/{name}/watchdog: the +// fleet watchdog's observed truth for one agent (watchdog.Observation) and +// the newest production evidence (watchdog.Fleet.LastProduction), nil when +// no evidence source was readable. +type WatchdogObservation struct { + Agent string `json:"agent"` + Backend string `json:"backend"` + SessionExists bool `json:"session_exists"` + Pane string `json:"pane,omitempty"` + HasCLIMarker bool `json:"has_cli_marker"` + ShowsLoginPrompt bool `json:"shows_login_prompt"` + LastChange time.Time `json:"last_change"` + StartedAt time.Time `json:"started_at"` + AuthAvailable bool `json:"auth_available"` + AuthKnown bool `json:"auth_known"` + CredentialProven bool `json:"credential_proven"` + ProviderErrorClass string `json:"provider_error_class,omitempty"` + ProviderErrorLine string `json:"provider_error_line,omitempty"` + LastProduction *time.Time `json:"last_production,omitempty"` +} + +// Condition mirrors crux.Condition. +type Condition struct { + Type string `json:"type"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` + LastTransitionTime time.Time `json:"lastTransitionTime"` +} + +// Conditions is the body of PUT /v1/agents/{name}/watchdog/conditions +// (watchdog.Fleet.SetConditions). +type Conditions struct { + Conditions []Condition `json:"conditions"` +} + +// Additional error codes. +const ( + // CodeForbidden is an Authorizer refusal. + CodeForbidden = "forbidden" + // CodeKickRefused is Kicker.ValidateKick's refusal (409). + CodeKickRefused = "kick_refused" + // CodeResumeBusy, CodeResumeSessionGone and CodeResumeUnsupported are + // the Resumer sentinels (409, 410, 422). + CodeResumeBusy = "resume_busy" + CodeResumeSessionGone = "resume_session_gone" + CodeResumeUnsupported = "resume_unsupported" + // CodeNotSupported is a route the server's runtime cannot serve (the + // watchdog routes on a runtime without a watchdog source). + CodeNotSupported = "not_supported" +) + +// Event types for the observer half of crux.Host (hivecommons/hive#14138 +// stage 3): the server relays each observer the runtime offers as an event, +// with the observer's payload in Event.Data. +const ( + // EventKickOutcome carries a KickOutcome (Host.SetKickOutcomeObserver). + EventKickOutcome = "agent.kick_outcome" + // EventPauseTransition carries a PauseTransition + // (Host.SetPauseTransitionObserver). + EventPauseTransition = "agent.pause_transition" + // EventRelaunch has no payload (Host.SetRelaunchObserver). + EventRelaunch = "agent.relaunch" + // EventPausePersist carries a PausePersist (Host.SetPersistPauseCallback). + EventPausePersist = "agent.pause_persist" + // EventPrompt carries a Prompt (Host.SetRecordPromptCallback). + EventPrompt = "agent.prompt" + // EventSandboxAudit carries a SandboxAudit (Host.SetSandboxAuditCallback). + EventSandboxAudit = "agent.sandbox_audit" + // EventAudit carries an Audit (Host.SetAuditSink). + EventAudit = "runtime.audit" + // EventCondition carries a ConditionEvent (Host.SetConditionReporter). + EventCondition = "runtime.condition" +) + +// KickOutcome mirrors crux.KickOutcome. +type KickOutcome struct { + Kind string `json:"kind"` + Reason string `json:"reason,omitempty"` + At time.Time `json:"at"` + KickAt time.Time `json:"kickAt"` +} + +// PauseTransition mirrors crux.PauseTransitionEvent. +type PauseTransition struct { + Paused bool `json:"paused"` + Trigger string `json:"trigger,omitempty"` + Reason string `json:"reason,omitempty"` + By string `json:"by,omitempty"` + Causation PauseCausation `json:"causation"` + At time.Time `json:"at"` +} + +// PausePersist is the payload of EventPausePersist. +type PausePersist struct { + Paused bool `json:"paused"` +} + +// Prompt is the payload of EventPrompt. +type Prompt struct { + Trigger string `json:"trigger"` + Prompt string `json:"prompt"` +} + +// SandboxAudit is the payload of EventSandboxAudit. +type SandboxAudit struct { + Action string `json:"action"` + Detail string `json:"detail,omitempty"` +} + +// Audit is the payload of EventAudit. +type Audit struct { + Actor string `json:"actor"` + Action string `json:"action"` + Fields map[string]any `json:"fields,omitempty"` +} + +// ConditionEvent is the payload of EventCondition; Action is "observe" or +// "clear". +type ConditionEvent struct { + Action string `json:"action"` + Class string `json:"class"` + Key string `json:"key"` + Message string `json:"message,omitempty"` + Diagnostics string `json:"diagnostics,omitempty"` +} + +// Condition actions. +const ( + ConditionObserve = "observe" + ConditionClear = "clear" +) + +// Route is one HTTP route: the method and the path pattern as the server +// registers it (with {name}, {kind}, {id}, {backend} placeholders). +type Route struct { + Method string + Path string +} + +// RuntimeRoutes maps every crux.Runtime method to the route that serves it. +// Several methods share a route (one GET answers a family of reads; one POST +// takes a request whose fields select the variant). The Background family +// maps to the health route: the loops run inside the serving process, which +// reports them, and a client's Background methods are no-ops. Observers +// (SetKickObserver) map to the event stream. crux/api/client's parity test +// holds this table and the interface to each other. +var RuntimeRoutes = map[string]Route{ + // Lifecycle + "Start": {"POST", PathAgents + "/{name}/start"}, + "Stop": {"POST", PathAgents + "/{name}/stop"}, + "Restart": {"POST", PathAgents + "/{name}/restart"}, + "RestartWithBootstrap": {"POST", PathAgents + "/{name}/restart"}, + "Pause": {"POST", PathAgents + "/{name}/pause"}, + "PauseBy": {"POST", PathAgents + "/{name}/pause"}, + "Resume": {"POST", PathAgents + "/{name}/resume"}, + "IsPaused": {"GET", PathAgents + "/{name}"}, + "ResetRestartCount": {"DELETE", PathAgents + "/{name}/restart-count"}, + // Observer + "GetStatus": {"GET", PathAgents + "/{name}/snapshot"}, + "GetStatusFast": {"GET", PathAgents + "/{name}/snapshot"}, + "AllStatuses": {"GET", PathSnapshots}, + "ResolveAgent": {"GET", PathAgents + "/{name}"}, + "GetOutput": {"GET", PathAgents + "/{name}/logs"}, + "GetBufferOutput": {"GET", PathAgents + "/{name}/logs"}, + "CaptureFullLog": {"GET", PathAgents + "/{name}/logs"}, + "FullLogHeader": {"GET", PathAgents + "/{name}/logs"}, + // Kicker + "SendKick": {"POST", PathAgents + "/{name}/kick"}, + "SendKickAsync": {"POST", PathAgents + "/{name}/kick"}, + "SendKickWithSource": {"POST", PathAgents + "/{name}/kick"}, + "ValidateKick": {"POST", PathAgents + "/{name}/kick/validate"}, + "KickDispatchState": {"GET", PathAgents + "/{name}/kick"}, + "ListKickLogs": {"GET", PathAgents + "/{name}/kicklogs"}, + "ReadKickLog": {"GET", PathAgents + "/{name}/kicklogs/{id}"}, + "SetKickObserver": {"GET", PathEvents}, + // Definitions + "AddAgent": {"POST", PathDefinitions}, + "RemoveAgent": {"DELETE", PathAgents + "/{name}"}, + "UpdateConfig": {"PUT", PathAgents + "/{name}/definition"}, + "ReconcileAgents": {"POST", PathReconcile}, + "SetModelOverride": {"PUT", PathAgents + "/{name}/overrides"}, + "SetBackendOverride": {"PUT", PathAgents + "/{name}/overrides"}, + "PinCLI": {"PUT", PathAgents + "/{name}/pins/{kind}"}, + "PinModel": {"PUT", PathAgents + "/{name}/pins/{kind}"}, + "UnpinCLI": {"DELETE", PathAgents + "/{name}/pins/{kind}"}, + "UnpinModel": {"DELETE", PathAgents + "/{name}/pins/{kind}"}, + "ClearModeOverrides": {"POST", PathModesClear}, + "SyncModeFiles": {"POST", PathModesSync}, + "RefreshInferenceRoutes": {"POST", PathInference}, + // Fleet + "BreakerState": {"GET", PathBreaker}, + "EngageBreaker": {"POST", PathBreakerOn}, + "ReleaseBreaker": {"POST", PathBreakerOff}, + "SetACMMLevel": {"PUT", PathACMM}, + "GetACMMLevel": {"GET", PathFleet}, + "RelaunchBobAgentsAwaitingKey": {"POST", PathRelaunchBob}, + "SharedDevUID": {"GET", PathFleet}, + // Credentials + "SubmitLoginCode": {"POST", PathAgents + "/{name}/login-code"}, + "CopilotToken": {"GET", PathCopilotCred}, + "CopilotTokenSource": {"GET", PathCopilotCred}, + "SetCopilotToken": {"PUT", PathCopilotCred}, + "ActivateCopilotToken": {"PUT", PathCopilotCred}, + "ReloadClaudeToken": {"POST", PathClaudeReload}, + "ClaudeCredentialCandidatePaths": {"GET", PathClaudeCred}, + "OmpAgentHomes": {"GET", PathOmpHomes}, + // Resumer + "ResumeHandle": {"GET", PathAgents + "/{name}/session"}, + "SendResumeKick": {"POST", PathAgents + "/{name}/session/kick"}, + "SessionID": {"GET", PathAgents + "/{name}/session"}, + "SessionMissing": {"GET", PathAgents + "/{name}/session"}, + // Supervisor + "GovernorMayStart": {"GET", PathAgents + "/{name}/supervision"}, + "MarkStartupLaunchQueued": {"POST", PathStartupQueued}, + "CheckAndRestartCrashedAgents": {"POST", PathRestartCrashed}, + "ConsentWedgedAgents": {"GET", PathFleet}, + "RefreshAgentTokens": {"POST", PathTokensRefresh}, + "RefreshAgentTokenFor": {"POST", PathAgents + "/{name}/tokens/refresh"}, + "AgentCredentialRefreshable": {"GET", PathAgents + "/{name}/supervision"}, + "AgentHasValidCredential": {"GET", PathAgents + "/{name}/supervision"}, + "AgentCapabilities": {"GET", PathAgents + "/{name}/supervision"}, + "EffectiveBackend": {"GET", PathAgents + "/{name}/supervision"}, + "InvocationMetadata": {"GET", PathAgents + "/{name}/supervision"}, + "RestartTelemetry": {"GET", PathAgents + "/{name}/supervision"}, + "StartFailureState": {"GET", PathAgents + "/{name}/supervision"}, + "ProviderErrorBackoffRemaining": {"GET", PathAgents + "/{name}/supervision"}, + "PauseByCause": {"POST", PathAgents + "/{name}/pause"}, + "ArchiveAllKickLogs": {"POST", PathKickLogsArchive}, + "CountAgentsWithModel": {"GET", PathFleet}, + "CrashOOMSuspected": {"GET", PathAgents + "/{name}/supervision"}, + "BackendAuthAvailable": {"GET", PathBackends + "/{backend}/auth"}, + "AgentAuthAvailable": {"GET", PathAgents + "/{name}/supervision"}, + "SendGitHubAPIBudgetNudge": {"POST", PathAgents + "/{name}/nudge"}, + "RecordAudit": {"POST", PathAudit}, + "TrajectoryAgents": {"GET", PathTrajectory}, + // Restorer + "SeedPauseState": {"POST", PathAgents + "/{name}/seed"}, + "SeedRestartTelemetry": {"POST", PathAgents + "/{name}/seed"}, + "SeedTurnLoss": {"POST", PathAgents + "/{name}/seed"}, + "SeedLastKick": {"POST", PathAgents + "/{name}/seed"}, + "SeedKickHistory": {"POST", PathAgents + "/{name}/seed"}, + "RestoreBreaker": {"PUT", PathBreaker}, + // Background (server-side loops; the health body lists them) + "StartAgentTokenRefresh": {"GET", PathHealth}, + "StartCredentialWatchdog": {"GET", PathHealth}, + "StartCopilotSessionRefresh": {"GET", PathHealth}, + "StartTranscriptActivityRefresh": {"GET", PathHealth}, + // Authorizer + "AuthorizePROpen": {"POST", PathAuthorize}, + "AuthorizePushBranch": {"POST", PathAuthorize}, + "AuthorizeIssueOpen": {"POST", PathAuthorize}, + "AuthorizeMerge": {"POST", PathAuthorize}, + "AuthorizeReviewRequest": {"POST", PathAuthorize}, +} + +// WatchdogRoutes are the watchdog seam's routes (crux/runtime.NewWatchdogFleet +// over a sidecar). They are not crux.Runtime methods, so they sit outside +// RuntimeRoutes; crux/serve's tests assert they are registered. +var WatchdogRoutes = map[string]Route{ + "AgentNames": {"GET", PathWatchdogAgents}, + "Observe": {"GET", PathAgents + "/{name}/watchdog"}, + "LastProduction": {"GET", PathAgents + "/{name}/watchdog"}, + "SetConditions": {"PUT", PathAgents + "/{name}/watchdog/conditions"}, +} + +// HostSetterKind says how a crux.Host setter reaches a sidecar runtime +// (hivecommons/hive#14138 stage 4). +type HostSetterKind string + +const ( + // HostSetterEvent: the setter installs an observer; the sidecar's + // runtime emits the observation on the event stream and hive's host + // shim invokes the observer from it. Via is the event type. + HostSetterEvent HostSetterKind = "event" + // HostSetterCallback: the setter hands the runtime a hive-provided + // value or call; the sidecar's runtime reaches the equivalent on the + // hive callback endpoint (crux/api/callback). Via is the route. + HostSetterCallback HostSetterKind = "callback" + // HostSetterRoute: hive pushes the value to the runtime over the + // runtime API. Via is the route. + HostSetterRoute HostSetterKind = "route" + // HostSetterConfig: the sidecar reads the value from its own + // configuration. Via is the key. + HostSetterConfig HostSetterKind = "config" + // HostSetterInProcessOnly: the setter has no serialisable form and the + // sidecar runtime uses its own in-process equivalent. Via says what. + HostSetterInProcessOnly HostSetterKind = "inprocess" +) + +// HostSetter is one row of HostSetters. +type HostSetter struct { + Kind HostSetterKind + Via string +} + +// Callback endpoint routes the table refers to, spelled here because +// crux/api/callback imports this package; its tests hold the two in step. +const ( + hostViaPolicy = "GET /v1/policy" + hostViaRepoAutoMerge = "POST /v1/policy/repo-auto-merge" + hostViaTokens = "POST /v1/tokens" + hostViaMint = "POST /v1/tokens/mint" + hostViaBobCredential = "GET /v1/bob/credential" + hostViaLinear = "GET /v1/linear/credential" + hostViaPulls = "POST /v1/pulls" + hostViaInferenceRoutes = "PUT|DELETE /v1/inference/routes/{agent}" + hostViaEffects = "POST /v1/effects, POST /v1/effects/{id}/end" + hostViaConditions = "POST /v1/conditions" +) + +// HostSetters maps every crux.Host setter to how a sidecar runtime gets +// what an in-process runtime is handed through it. crux/api/client's parity +// test holds this table and the interface to each other, so a new setter +// cannot be added without saying how the sidecar is served. +var HostSetters = map[string]HostSetter{ + // Credentials -> hive callback endpoint (stage 2; the mint issuer and + // Bob key in stage 4). + "SetAppAuth": {HostSetterCallback, hostViaTokens}, + "SetAgentMint": {HostSetterCallback, hostViaMint}, + "SetLinearCredentialResolver": {HostSetterCallback, hostViaLinear}, + "SetBobAPIKeyResolver": {HostSetterCallback, hostViaBobCredential}, + "SetBobKeySourceResolver": {HostSetterCallback, hostViaBobCredential}, + "SetSandboxPushMinter": {HostSetterCallback, hostViaTokens}, + "SetSandboxPRClient": {HostSetterCallback, hostViaPulls}, + + // Policy -> the hive callback's policy snapshot (stage 4). + "SetBuildInfo": {HostSetterCallback, hostViaPolicy}, + "SetGatewayBackendChecker": {HostSetterCallback, hostViaPolicy}, + "SetExplainModeDefaultResolver": {HostSetterCallback, hostViaPolicy}, + "SetRepoAutoMergeEnabledResolver": {HostSetterCallback, hostViaRepoAutoMerge}, + "SetKickLogSourceResolver": {HostSetterCallback, hostViaPolicy}, + "SetSandboxConfig": {HostSetterCallback, hostViaPolicy}, + "SetDeadSessionRecoveryOwner": {HostSetterCallback, hostViaPolicy}, + + // Calls into hive -> their own callback families (stage 4). + "SetInferenceCallbacks": {HostSetterCallback, hostViaInferenceRoutes}, + "SetSandboxMutationBoundary": {HostSetterCallback, hostViaEffects}, + + // Observers -> the event stream (stage 3). Conditions also go back + // over the callback so hive's fleet report sees them. + "SetKickOutcomeObserver": {HostSetterEvent, EventKickOutcome}, + "SetPauseTransitionObserver": {HostSetterEvent, EventPauseTransition}, + "SetRelaunchObserver": {HostSetterEvent, EventRelaunch}, + "SetPersistPauseCallback": {HostSetterEvent, EventPausePersist}, + "SetRecordPromptCallback": {HostSetterEvent, EventPrompt}, + "SetSandboxAuditCallback": {HostSetterEvent, EventSandboxAudit}, + "SetAuditSink": {HostSetterEvent, EventAudit}, + "SetConditionReporter": {HostSetterEvent, EventCondition + ", " + hostViaConditions}, + + // In-process only. + "SetSpanStarter": {HostSetterInProcessOnly, "OpenTelemetry global tracer in the runtime process; trajectory events are independent of spans"}, + // Advisor lane (v6, #9722): the sidecar runtime does not run the advisor + // lane yet; resolvers are installed only in-process until the lane's + // policy rides the hive callback. + "SetAdvisorEnabledResolver": {HostSetterInProcessOnly, "advisor lane is in-process only; sidecar follow-up"}, + "SetAdvisorRolesResolver": {HostSetterInProcessOnly, "advisor lane is in-process only; sidecar follow-up"}, + // Repeat-failure BASH_ENV route (v6, #14574): the enabled switch is read + // in-process until it rides the hive callback policy. + "SetRepeatFailureEnabledResolver": {HostSetterInProcessOnly, "repeat-failure capture switch is in-process only; sidecar follow-up"}, +} diff --git a/src/crux/api/runtime_test.go b/src/crux/api/runtime_test.go new file mode 100644 index 0000000000..4a169a0d09 --- /dev/null +++ b/src/crux/api/runtime_test.go @@ -0,0 +1,63 @@ +package api + +import ( + "net/http" + "strings" + "testing" +) + +func TestAgentPathHelpers(t *testing.T) { + for got, want := range map[string]string{ + StartPath("a"): PathAgents + "/a/start", + RestartPath("a"): PathAgents + "/a/restart", + PausePath("a"): PathAgents + "/a/pause", + ResumePath("a"): PathAgents + "/a/resume", + RestartCountPath("a"): PathAgents + "/a/restart-count", + SnapshotPath("a"): PathAgents + "/a/snapshot", + KickValidatePath("a"): PathAgents + "/a/kick/validate", + KickLogsPath("a"): PathAgents + "/a/kicklogs", + KickLogPath("a", "k"): PathAgents + "/a/kicklogs/k", + DefinitionPath("a"): PathAgents + "/a/definition", + OverridesPath("a"): PathAgents + "/a/overrides", + PinPath("a", PinCLI): PathAgents + "/a/pins/cli", + LoginCodePath("a"): PathAgents + "/a/login-code", + SessionPath("a"): PathAgents + "/a/session", + SessionKickPath("a"): PathAgents + "/a/session/kick", + SupervisionPath("a"): PathAgents + "/a/supervision", + AgentTokenRefreshPath("a"): PathAgents + "/a/tokens/refresh", + NudgePath("a"): PathAgents + "/a/nudge", + SeedPath("a"): PathAgents + "/a/seed", + WatchdogPath("a"): PathAgents + "/a/watchdog", + WatchdogConditionsPath("a"): PathAgents + "/a/watchdog/conditions", + BackendAuthPath("claude"): PathBackends + "/claude/auth", + } { + if got != want { + t.Errorf("got %s want %s", got, want) + } + } +} + +// TestRouteTablesAreWellFormed checks every route uses a known method and a +// versioned path, and that each path helper's shape matches a table entry +// once the placeholders are filled. +func TestRouteTablesAreWellFormed(t *testing.T) { + methods := map[string]bool{http.MethodGet: true, http.MethodPost: true, http.MethodPut: true, http.MethodDelete: true} + fill := strings.NewReplacer("{name}", "a", "{id}", "k", "{kind}", PinCLI, "{backend}", "claude") + paths := map[string]bool{} + for _, table := range []map[string]Route{RuntimeRoutes, WatchdogRoutes} { + for method, r := range table { + if !methods[r.Method] { + t.Errorf("%s: method %q", method, r.Method) + } + if !strings.HasPrefix(r.Path, "/"+Version+"/") { + t.Errorf("%s: path %q is not versioned", method, r.Path) + } + paths[fill.Replace(r.Path)] = true + } + } + for _, p := range []string{StartPath("a"), KickLogPath("a", "k"), PinPath("a", PinCLI), SessionKickPath("a"), WatchdogConditionsPath("a"), BackendAuthPath("claude"), PathSnapshots, PathReconcile, PathBreakerOn, PathAuthorize} { + if !paths[p] { + t.Errorf("%s has no route table entry", p) + } + } +} diff --git a/src/crux/api/wire.go b/src/crux/api/wire.go new file mode 100644 index 0000000000..57a7293f4d --- /dev/null +++ b/src/crux/api/wire.go @@ -0,0 +1,185 @@ +package api + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/netip" + "net/url" + "path/filepath" + "strings" +) + +// Address is a parsed server address: where to dial and what URL to put in +// front of the routes. For a unix socket BaseURL is a fixed http://crux and +// the transport dials Path; for TCP BaseURL is the scheme and host and Path +// is empty. +type Address struct { + // Network is "unix" or "tcp". + Network string + // Path is the socket path (unix only). + Path string + // Host is host:port (tcp only). + Host string + // BaseURL is the URL prefix requests are built on. + BaseURL string +} + +// unixBaseURL is the host clients use in URLs sent over a unix socket; the +// server never reads it. +const unixBaseURL = "http://crux" + +// ParseAddress accepts the forms --server and CRUX_SERVER take: +// +// unix:///run/user/1000/crux/crux.sock a unix socket by URL +// /run/user/1000/crux/crux.sock a unix socket by absolute path +// ./crux.sock, ../x/crux.sock a unix socket by relative path +// http://127.0.0.1:7777 a TCP listener by URL +// https://crux.example:7777 a TLS listener by URL +// 127.0.0.1:7777, [::1]:7777 a TCP listener, http implied +// +// Plain http (explicit or implied) is accepted only for a loopback host: +// the bearer token travels in the clear, so a remote server must be reached +// over https (a TLS-terminating proxy in front of `crux serve`). An empty +// string is an error so a caller cannot fall through to a default by +// accident. +func ParseAddress(s string) (Address, error) { + s = strings.TrimSpace(s) + if s == "" { + return Address{}, errors.New("empty server address") + } + switch { + case strings.HasPrefix(s, "unix://"): + p := strings.TrimPrefix(s, "unix://") + if p == "" { + return Address{}, fmt.Errorf("server address %q has no socket path", s) + } + return Address{Network: "unix", Path: p, BaseURL: unixBaseURL}, nil + case strings.HasPrefix(s, "/"), strings.HasPrefix(s, "./"), strings.HasPrefix(s, "../"): + return Address{Network: "unix", Path: filepath.Clean(s), BaseURL: unixBaseURL}, nil + case strings.HasPrefix(s, "http://"), strings.HasPrefix(s, "https://"): + u, err := url.Parse(s) + if err != nil { + return Address{}, fmt.Errorf("server address %q: %w", s, err) + } + if u.Host == "" { + return Address{}, fmt.Errorf("server address %q has no host", s) + } + if u.Scheme == "http" && !LoopbackHost(u.Host) { + return Address{}, fmt.Errorf("server address %q: http is allowed only for loopback hosts (the bearer token would cross the network in cleartext); use https:// for a remote server", s) + } + return Address{Network: "tcp", Host: u.Host, BaseURL: u.Scheme + "://" + u.Host}, nil + } + if strings.Contains(s, "://") { + return Address{}, fmt.Errorf("server address %q: unsupported scheme (use unix://, http:// or https://)", s) + } + // host:port with no scheme. + if _, _, ok := splitHostPort(s); !ok { + return Address{}, fmt.Errorf("server address %q is neither a socket path nor host:port", s) + } + if !LoopbackHost(s) { + return Address{}, fmt.Errorf("server address %q: plain host:port implies http, which is allowed only for loopback hosts (the bearer token would cross the network in cleartext); use https:// for a remote server", s) + } + return Address{Network: "tcp", Host: s, BaseURL: "http://" + s}, nil +} + +// LoopbackHost reports whether hostport (host:port, or a bare host) names +// the local machine only: "localhost" or a loopback IP (127.0.0.0/8, ::1, +// and their zoned or IPv4-mapped spellings). An empty host, a wildcard +// (0.0.0.0, ::) and anything that needs a resolver is not loopback, so a +// caller using this as a gate fails closed. It is the one rule the client +// (ParseAddress) and the server (config.ServeConfig.Validate) share about +// where a bearer token may travel over plain http. +func LoopbackHost(hostport string) bool { + if ip, err := netip.ParseAddr(hostport); err == nil { + return ip.Unmap().IsLoopback() + } + host := hostport + if h, _, ok := splitHostPort(hostport); ok { + host = h + } + host = strings.TrimSuffix(strings.TrimPrefix(host, "["), "]") + if host == "" { + return false + } + if strings.EqualFold(host, "localhost") { + return true + } + ip, err := netip.ParseAddr(host) + if err != nil { + return false + } + return ip.Unmap().IsLoopback() +} + +// splitHostPort is net.SplitHostPort's shape without importing net, so this +// package stays free of anything a client might not want. +func splitHostPort(s string) (host, port string, ok bool) { + i := strings.LastIndex(s, ":") + if i < 0 || i == len(s)-1 { + return "", "", false + } + host, port = s[:i], s[i+1:] + for _, r := range port { + if r < '0' || r > '9' { + return "", "", false + } + } + if strings.HasPrefix(host, "[") != strings.HasSuffix(host, "]") { + return "", "", false + } + return host, port, true +} + +// WriteEvent writes one server-sent event: `event: ` then the JSON of v +// on a `data:` line, then the blank line that ends the record. The caller +// flushes. +func WriteEvent(w io.Writer, typ string, v any) error { + data, err := json.Marshal(v) + if err != nil { + return err + } + _, err = fmt.Fprintf(w, "event: %s\ndata: %s\n\n", typ, data) + return err +} + +// ReadEvents decodes a text/event-stream body, calling fn with each record's +// type and raw data until r ends or fn returns an error, which is returned. +// Comment lines (`:` prefix, used as keep-alives) and unknown fields are +// skipped. A record with no data line is skipped. +func ReadEvents(r io.Reader, fn func(typ string, data []byte) error) error { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + var typ string + var data [][]byte + flush := func() error { + if len(data) == 0 { + typ = "" + return nil + } + err := fn(typ, bytes.Join(data, []byte("\n"))) + typ, data = "", nil + return err + } + for sc.Scan() { + line := sc.Text() + switch { + case line == "": + if err := flush(); err != nil { + return err + } + case strings.HasPrefix(line, ":"): + case strings.HasPrefix(line, "event:"): + typ = strings.TrimSpace(strings.TrimPrefix(line, "event:")) + case strings.HasPrefix(line, "data:"): + data = append(data, []byte(strings.TrimPrefix(strings.TrimPrefix(line, "data:"), " "))) + } + } + if err := sc.Err(); err != nil { + return err + } + return flush() +} diff --git a/src/pkg/crux/types/audit.go b/src/crux/audit.go similarity index 97% rename from src/pkg/crux/types/audit.go rename to src/crux/audit.go index a2646da8f6..350cfa09e4 100644 --- a/src/pkg/crux/types/audit.go +++ b/src/crux/audit.go @@ -1,4 +1,4 @@ -package types +package crux // Audit action names the runtime itself records: agent lifecycle events, the // tool-approval verdict, and the credential watchdog's missing-token signals. @@ -7,7 +7,7 @@ package types // treats them uniformly. // // The governor-side vocabulary (lease stages, relay events, jev decisions, -// commit trailers) is pkg/agentaudit's alone; this leaf holds only what the +// commit trailers) is pkg/agentaudit's alone; this package holds only what the // runtime emits, so the runtime does not depend on the governor's package // (hivecommons/hive#13536). pkg/agentaudit aliases these so the recorder // and every other emitter keep their existing spelling. diff --git a/src/pkg/crux/types/backend_auth.go b/src/crux/backend_auth.go similarity index 66% rename from src/pkg/crux/types/backend_auth.go rename to src/crux/backend_auth.go index 1472129fee..0a6ffd9351 100644 --- a/src/pkg/crux/types/backend_auth.go +++ b/src/crux/backend_auth.go @@ -1,6 +1,10 @@ -package types +package crux -import "time" +import ( + "time" + + "github.com/hivecommons/hive/crux/config" +) // BackendAuth status values. These are the coarse states an operator or the // hub's fleet aggregator needs to distinguish; the runtime's finer provider @@ -38,3 +42,27 @@ type BackendAuthState struct { Since time.Time `json:"since,omitempty"` LastError string `json:"lastError,omitempty"` } + +// interactiveAuthBackends are the CLI backends whose credentials come from an +// INTERACTIVE login flow (OAuth / device flow) that a human must complete. Only +// these can ever legitimately show a login badge. +var interactiveAuthBackends = map[string]bool{ + "claude": true, + "copilot": true, + "codex": true, + "gemini": true, + "omp": true, +} + +// BackendRequiresInteractiveAuth reports whether a backend authenticates via an +// interactive login the operator must perform. Model-gateway backends +// (config.InferenceBackends: litellm, vllm, llm-d, watsonx) authenticate with +// an API key supplied by config — watsonx with an IAM bearer minted from that +// key — and therefore NEVER require a login. Showing them a 🔑 badge is always +// a false alarm. +func BackendRequiresInteractiveAuth(backend string) bool { + if config.IsInferenceBackend(backend) { + return false + } + return interactiveAuthBackends[backend] +} diff --git a/src/crux/canary.go b/src/crux/canary.go new file mode 100644 index 0000000000..35050b32c5 --- /dev/null +++ b/src/crux/canary.go @@ -0,0 +1,13 @@ +package crux + +// CanaryLeak is one detection of a planted canary token in text an agent is +// about to send off-host: which agent the token was planted for, the token +// itself and the surface it was seen on (for example "github:POST +// /repos/o/r/issues"). The registry that plants tokens and recognises their +// encodings is host-side (pkg/ioscan aliases this type); the proxy only +// carries the value to the host's leak handler (hivecommons/hive#13536). +type CanaryLeak struct { + Agent string + Token string + Source string +} diff --git a/src/pkg/claude/oauth.go b/src/crux/claude/oauth.go similarity index 100% rename from src/pkg/claude/oauth.go rename to src/crux/claude/oauth.go diff --git a/src/pkg/claude/oauth_test.go b/src/crux/claude/oauth_test.go similarity index 100% rename from src/pkg/claude/oauth_test.go rename to src/crux/claude/oauth_test.go diff --git a/src/pkg/claude/oauth_write_error_branches_test.go b/src/crux/claude/oauth_write_error_branches_test.go similarity index 100% rename from src/pkg/claude/oauth_write_error_branches_test.go rename to src/crux/claude/oauth_write_error_branches_test.go diff --git a/src/pkg/claude/usable_token_test.go b/src/crux/claude/usable_token_test.go similarity index 100% rename from src/pkg/claude/usable_token_test.go rename to src/crux/claude/usable_token_test.go diff --git a/src/crux/cli/agy_turn_test.go b/src/crux/cli/agy_turn_test.go new file mode 100644 index 0000000000..ef5a29d676 --- /dev/null +++ b/src/crux/cli/agy_turn_test.go @@ -0,0 +1,43 @@ +package cli + +import ( + "context" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/config" +) + +// `crux agy-turn` is the pane-side subprocess a `crux serve` manager types +// into headless agy panes (it spawns its own executable). It dispatches to +// agent.RunAgyTurn, runs locally even when CRUX_SERVER points at a server, +// and is refused over the control socket. +func TestAgyTurnSubcommand(t *testing.T) { + h := newHarness(t) + if code := h.run(agent.AgyTurnSubcommand, "--help"); code != ExitOK { + t.Fatalf("agy-turn --help: code %d, stderr %q", code, h.stderr) + } + if !strings.Contains(h.stderr.String(), "-prompt-file") { + t.Fatalf("agy-turn --help did not print agy-turn's flags:\n%s", h.stderr) + } + + h = newHarness(t) + if code := h.run(agent.AgyTurnSubcommand, "--no-such-flag"); code != 2 { + t.Fatalf("agy-turn bad flag: code %d, stderr %q", code, h.stderr) + } + + // A CRUX_SERVER in the environment must not send the subprocess over + // the API: nothing listens there for it, and the pane would hang. + t.Setenv(config.ServerEnvVar, "127.0.0.1:1") + h = newHarness(t) + if code := h.run(agent.AgyTurnSubcommand, "--help"); code != ExitOK { + t.Fatalf("agy-turn with CRUX_SERVER: code %d, stderr %q", code, h.stderr) + } + + h = newHarness(t) + if code := h.env.dispatch(context.Background(), globalOptions{}, agent.AgyTurnSubcommand, nil, true); code != ExitUsage || + !strings.Contains(h.stderr.String(), "not available over the control socket") { + t.Fatalf("agy-turn over the control socket: code %d, stderr %q", code, h.stderr) + } +} diff --git a/src/pkg/crux/cli/cli.go b/src/crux/cli/cli.go similarity index 64% rename from src/pkg/crux/cli/cli.go rename to src/crux/cli/cli.go index b7235ff74f..6ff4770db1 100644 --- a/src/pkg/crux/cli/cli.go +++ b/src/crux/cli/cli.go @@ -1,6 +1,7 @@ // Package cli is the `crux` command (hivecommons/hive#13536, step 5 of the // Stage 1 roadmap in src/docs/design/crux.md): run, ps, logs, kick and stop, -// each a thin mapping onto one or two crux.Runtime methods. +// each a thin mapping onto one or two crux.Runtime methods, and serve, which +// exposes the Runtime over the Crux API (crux/api, crux/serve; Stage 3). // // The runtime is in-process. `crux run` is therefore the long-lived command: // it builds a Runtime, starts the agent, streams its output and stops it when @@ -22,12 +23,16 @@ import ( "fmt" "io" "os" + "os/signal" "path/filepath" "strings" + "syscall" "time" - "github.com/hivecommons/hive/pkg/crux" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" ) // Exit codes. Usage errors are distinct from runtime failures so wrappers can @@ -52,6 +57,14 @@ const ( streamWindow = 200 // defaultLogLines is the default -n for `logs`. defaultLogLines = 100 + // DefsEnvVar is the environment default for --defs / --config: the + // definition source the host's DefinitionLoader reads when neither flag + // is given (a directory or file of agent definitions for the module's + // own loader, a hive.yaml for the Hive-backed one). + DefsEnvVar = "CRUX_DEFS" + // DefaultVersion is what --version reports when the host did not set + // Env.Version (a `go build` without -ldflags "-X main.version=..."). + DefaultVersion = "dev" ) // globalOptions are the flags that precede the subcommand. @@ -60,6 +73,11 @@ type globalOptions struct { config string socket string local bool + version bool + // server and token select remote mode: drive a `crux serve` over the + // Crux API instead of an in-process runtime. + server string + token string } // runtimeFactory builds the Runtime a command operates on. Production uses @@ -79,11 +97,16 @@ type Env struct { // Loader reads --config and definition files. Nil means this build cannot // read them (see noLoader); cmd/crux wires the Hive-backed one. Loader DefinitionLoader + // Version is what --version prints; empty means DefaultVersion. Hosts set + // it from their ldflags-injected main.version. + Version string newRuntime runtimeFactory ticker tickerFactory // dial connects to a control socket; nil uses the unix dialer. dial dialFunc + // newClient builds the API client for --server; nil uses api.NewClient. + newClient clientFactory } // Main runs the crux command line and returns its exit code. ctx ending @@ -102,9 +125,13 @@ func (e *Env) Run(ctx context.Context, args []string) int { fs.SetOutput(e.Stderr) var opts globalOptions fs.StringVar(&opts.workDir, "workdir", "", "agent work dir (default $HIVE_WORK_DIR, then $HOME/"+defaultWorkDirName+")") - fs.StringVar(&opts.config, "config", "", "hive.yaml whose agents define the runtime") + fs.StringVar(&opts.config, "config", "", "definition source the loader reads (a hive.yaml for the hive-backed crux; default $"+DefsEnvVar+")") + fs.StringVar(&opts.config, "defs", "", "synonym of -config: a directory or file of agent definitions for this build's loader (default $"+DefsEnvVar+")") fs.StringVar(&opts.socket, "socket", "", "control socket path (default /"+controlSocketName+")") fs.BoolVar(&opts.local, "local", false, "never use a running `crux run` control socket; build a fresh runtime") + fs.StringVar(&opts.server, "server", "", "drive a `crux serve` at this address (unix socket path, unix:///path, host:port or http://…) instead of an in-process runtime (default $"+config.ServerEnvVar+")") + fs.StringVar(&opts.token, "token", "", "bearer token for --server (default $"+config.ServeTokenEnvVar+")") + fs.BoolVar(&opts.version, "version", false, "print the crux version and exit") fs.Usage = func() { e.usage(fs) } if err := fs.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { @@ -112,6 +139,10 @@ func (e *Env) Run(ctx context.Context, args []string) int { } return ExitUsage } + if opts.version { + fmt.Fprintln(e.Stdout, "crux", e.version()) + return ExitOK + } rest := fs.Args() if len(rest) == 0 { e.usage(fs) @@ -121,6 +152,16 @@ func (e *Env) Run(ctx context.Context, args []string) int { fmt.Fprintln(e.Stderr, "crux:", err) return ExitError } + if opts.remoteSelected() { + switch rest[0] { + case "help", "-h", "--help", "serve", agent.AgyTurnSubcommand: + // serve is where the runtime lives; a CRUX_SERVER left in a shell + // profile must not stop a local server from starting. agy-turn is + // the pane-side subprocess the serving runtime itself spawns. + default: + return e.remote(ctx, opts, rest[0], rest[1:]) + } + } return e.dispatch(ctx, opts, rest[0], rest[1:], false) } @@ -135,6 +176,20 @@ func (e *Env) dispatch(ctx context.Context, opts globalOptions, name string, arg return ExitUsage } return e.cmdRun(ctx, opts, args) + case "serve": + if remote { + fmt.Fprintln(e.Stderr, "crux: serve is not available over the control socket") + return ExitUsage + } + return e.cmdServe(ctx, opts, args) + case agent.AgyTurnSubcommand: + if remote { + fmt.Fprintf(e.Stderr, "crux: %s is not available over the control socket\n", name) + return ExitUsage + } + // The manager types ` agy-turn …` into a headless agy + // pane; under `crux serve` that executable is crux. + return agent.RunAgyTurn(args, e.Stdout, e.Stderr) case "ps", "logs", "kick", "stop": if !remote && !opts.local { if code, ok := e.forward(ctx, opts, append([]string{name}, args...)); ok { @@ -210,11 +265,32 @@ func (e *Env) defaults() { if e.dial == nil { e.dial = dialControl } + if e.newClient == nil { + e.newClient = api.NewClient + } +} + +func (e *Env) version() string { + if e.Version == "" { + return DefaultVersion + } + return e.Version } -// resolve fills the work dir and socket defaults and exports the work dir to -// the manager through HIVE_WORK_DIR. +// SignalContext returns a context that ends on SIGINT or SIGTERM, so a +// foreground `crux run` or `crux serve` stops its agents before exiting. Both +// hosts (cmd/crux in the root module, crux/cmd/crux in this one) hand it to +// Env.Run. +func SignalContext() (context.Context, context.CancelFunc) { + return signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) +} + +// resolve fills the work dir and socket defaults, the definition source from +// CRUX_DEFS, and exports the work dir to the manager through HIVE_WORK_DIR. func (o *globalOptions) resolve() error { + if o.config == "" { + o.config = os.Getenv(DefsEnvVar) + } if o.workDir == "" { o.workDir = os.Getenv(envWorkDir) } @@ -245,9 +321,22 @@ Commands (each maps onto crux.Runtime): --full: CaptureFullLog) kick Kicker.SendKick stop Lifecycle.Stop + serve expose the runtime over the Crux API (HTTP+JSON + on a unix socket; --listen adds TCP with --token) + agy-turn one headless agy turn; typed into a pane by the + runtime that spawned it, not run by hand While 'crux run' is in the foreground it serves /crux.sock; ps, logs, kick and stop use that socket when present so they act on the live runtime. +'crux serve' is the API server other harnesses drive (crux/api, crux/serve). +With --server ADDR (or CRUX_SERVER set) run, ps, logs, kick and stop drive that +server over the API instead of an in-process runtime; --token / CRUX_TOKEN +carries its bearer token, and 'logs -f' follows the agent's output. 'serve' +itself always runs locally, whatever CRUX_SERVER says. + +--config and --defs name the same thing: where this build's definition loader +reads agents from (default $CRUX_DEFS). The hive-backed crux reads a hive.yaml; +the module's own crux reads a directory or file of agent definition YAML. Global flags: `) diff --git a/src/pkg/crux/cli/cli_test.go b/src/crux/cli/cli_test.go similarity index 90% rename from src/pkg/crux/cli/cli_test.go rename to src/crux/cli/cli_test.go index bbd990fcd0..2ca8750769 100644 --- a/src/pkg/crux/cli/cli_test.go +++ b/src/crux/cli/cli_test.go @@ -12,8 +12,8 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" ) // fakeRuntime is a crux.Runtime that records the calls the CLI makes and @@ -123,6 +123,17 @@ func (f *fakeRuntime) AllStatuses() map[string]*crux.Process { return out } +func (f *fakeRuntime) GetStatus(name string) (*crux.Process, error) { + f.record("GetStatus(%s)", name) + f.mu.Lock() + defer f.mu.Unlock() + p, ok := f.procs[name] + if !ok { + return nil, fmt.Errorf("agent %s not found", name) + } + return p, nil +} + func (f *fakeRuntime) ResolveAgent(nameOrID string) string { f.record("ResolveAgent(%s)", nameOrID) f.mu.Lock() @@ -194,6 +205,12 @@ func (f *fakeRuntime) SendKick(name, message string) error { return nil } +// SetKickObserver is what `crux serve` installs so the API can relay kick +// events; the fake only records that it was asked. +func (f *fakeRuntime) SetKickObserver(fn func(agentName, event, detail string)) { + f.record("SetKickObserver(%t)", fn != nil) +} + // harness wires an Env to a fake runtime, captured streams and a hand-driven // ticker. type harness struct { @@ -676,3 +693,51 @@ func TestJSONAndTableWriteErrors(t *testing.T) { t.Fatalf("table: code %d", code) } } + +func TestVersionFlag(t *testing.T) { + h := newHarness(t) + if code := h.run("--version"); code != ExitOK { + t.Fatalf("--version: code %d, stderr %q", code, h.stderr) + } + if got := h.stdout.String(); got != "crux "+DefaultVersion+"\n" { + t.Fatalf("stdout = %q", got) + } + h = newHarness(t) + h.env.Version = "v1.2.3" + if code := h.run("--version", "ps"); code != ExitOK || h.stdout.String() != "crux v1.2.3\n" { + t.Fatalf("--version with a command: code %d, stdout %q", code, h.stdout) + } +} + +func TestDefsFlagAndEnvSelectTheDefinitionSource(t *testing.T) { + cfg := writeFile(t, filepath.Join(t.TempDir(), "hive.yaml"), "agents:\n fixer:\n backend: codex\n") + + h := newHarness(t) + if code := h.run("--local", "--defs", cfg, "ps"); code != ExitOK { + t.Fatalf("--defs: code %d, stderr %q", code, h.stderr) + } + if _, ok := h.defs["fixer"]; !ok { + t.Fatalf("--defs did not reach the loader: defs = %v", h.defs) + } + + t.Setenv(DefsEnvVar, cfg) + h = newHarness(t) + if code := h.run("--local", "ps"); code != ExitOK { + t.Fatalf("%s: code %d, stderr %q", DefsEnvVar, code, h.stderr) + } + if _, ok := h.defs["fixer"]; !ok { + t.Fatalf("%s did not reach the loader: defs = %v", DefsEnvVar, h.defs) + } + + // An explicit flag wins over the environment. + h = newHarness(t) + if code := h.run("--local", "--config", filepath.Join(t.TempDir(), "missing.yaml"), "ps"); code != ExitError { + t.Fatalf("flag over env: code %d, stderr %q", code, h.stderr) + } +} + +func TestSignalContextIsCancellable(t *testing.T) { + ctx, stop := SignalContext() + stop() + <-ctx.Done() +} diff --git a/src/pkg/crux/cli/commands.go b/src/crux/cli/commands.go similarity index 96% rename from src/pkg/crux/cli/commands.go rename to src/crux/cli/commands.go index 81a9d9ed84..40ed25064f 100644 --- a/src/pkg/crux/cli/commands.go +++ b/src/crux/cli/commands.go @@ -12,7 +12,7 @@ import ( "text/tabwriter" "time" - "github.com/hivecommons/hive/pkg/crux" + "github.com/hivecommons/hive/crux" ) // subFlags builds a subcommand flag set that reports to stderr and never @@ -207,6 +207,11 @@ func (e *Env) cmdPS(rt crux.Runtime, args []string) int { if *asJSON { return e.writeJSON(rows) } + return e.printPS(rows) +} + +// printPS writes the ps table; it is shared with remote mode. +func (e *Env) printPS(rows []processRow) int { tw := tabwriter.NewWriter(e.Stdout, 0, 0, 2, ' ', 0) fmt.Fprintln(tw, "NAME\tID\tSTATE\tBACKEND\tMODEL\tUPTIME\tLAST KICK\tRESTARTS") now := time.Now() @@ -290,7 +295,14 @@ func (e *Env) cmdLogs(rt crux.Runtime, args []string) int { if *asJSON { return e.writeJSON(out) } - if *full { + e.printLogs(out, *full) + return ExitOK +} + +// printLogs writes raw log output: header and content for a full capture, +// otherwise one line per entry. Shared with remote mode. +func (e *Env) printLogs(out logsOutput, full bool) { + if full { if out.Header != "" { fmt.Fprint(e.Stdout, out.Header) if !strings.HasSuffix(out.Header, "\n") { @@ -301,12 +313,11 @@ func (e *Env) cmdLogs(rt crux.Runtime, args []string) int { if out.Content != "" && !strings.HasSuffix(out.Content, "\n") { fmt.Fprintln(e.Stdout) } - return ExitOK + return } for _, line := range out.Lines { fmt.Fprintln(e.Stdout, line) } - return ExitOK } // cmdKick is `crux kick`: Kicker.SendKick. The message is the remaining diff --git a/src/pkg/crux/cli/control.go b/src/crux/cli/control.go similarity index 99% rename from src/pkg/crux/cli/control.go rename to src/crux/cli/control.go index dd58000df6..8aa5f1e79c 100644 --- a/src/pkg/crux/cli/control.go +++ b/src/crux/cli/control.go @@ -11,7 +11,7 @@ import ( "os" "sync" - "github.com/hivecommons/hive/pkg/crux" + "github.com/hivecommons/hive/crux" ) // The control socket is how ps, logs, kick and stop reach the runtime a diff --git a/src/pkg/crux/cli/control_test.go b/src/crux/cli/control_test.go similarity index 98% rename from src/pkg/crux/cli/control_test.go rename to src/crux/cli/control_test.go index f5ac67892c..91a5f19661 100644 --- a/src/pkg/crux/cli/control_test.go +++ b/src/crux/cli/control_test.go @@ -12,9 +12,9 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" - "github.com/hivecommons/hive/pkg/crux" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/config" ) // shortSocketDir returns a directory whose path is short enough for a unix diff --git a/src/pkg/crux/cli/definitions.go b/src/crux/cli/definitions.go similarity index 95% rename from src/pkg/crux/cli/definitions.go rename to src/crux/cli/definitions.go index fd53be0c49..378c94c8a8 100644 --- a/src/pkg/crux/cli/definitions.go +++ b/src/crux/cli/definitions.go @@ -5,8 +5,8 @@ import ( "path/filepath" "strings" - "github.com/hivecommons/hive/pkg/crux" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" ) // DefinitionLoader turns the files a command names into agent definitions. diff --git a/src/crux/cli/hive_callback.go b/src/crux/cli/hive_callback.go new file mode 100644 index 0000000000..f98cef8013 --- /dev/null +++ b/src/crux/cli/hive_callback.go @@ -0,0 +1,199 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "sync" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/effects" + "github.com/hivecommons/hive/crux/hivecap" + "github.com/hivecommons/hive/crux/pushbroker" + "github.com/hivecommons/hive/crux/serve" + "github.com/hivecommons/hive/crux/toolapprove" +) + +// hiveCapTarget is the subset of *agent.Manager setters the hive callback +// feeds in sidecar mode: the seams hive provides in-process through +// pkg/cruxhost (hivecommons/hive#14138 stage 2). A runtime that is not the +// manager (a test fake) is left alone. +type hiveCapTarget interface { + SetSandboxPRClient(agent.PRCreator) + SetLinearCredentialResolver(func() agent.LinearCredential) + SetSandboxPushMinter(pushbroker.TokenMinter) + SetInferenceCallbacks(setRoute func(agentName, backend, model string), clearRoute func(agentName string)) + SetSandboxMutationBoundary(effects.Boundary) + // Credential writers (hivecommons/hive#14138 stage 4). + SetAppAuth(agent.AppTokenMinter) + SetAgentMint(agent.AgentMintIssuer) + SetBobAPIKeyResolver(func() string) + SetBobKeySourceResolver(func() string) +} + +// hivePolicyTarget is the subset of crux.Host policy setters the hive +// callback's policy snapshot feeds (hivecommons/hive#14138 stage 4): the +// values an in-process hive hands the manager directly. +type hivePolicyTarget interface { + SetBuildInfo(func() (version, commit string)) + SetGatewayBackendChecker(func(backend string) bool) + SetExplainModeDefaultResolver(func() string) + SetRepoAutoMergeEnabledResolver(func(repo string) bool) + SetKickLogSourceResolver(func() string) + SetSandboxConfig(config.AgentSandboxConfig) + SetDeadSessionRecoveryOwner(ownedElsewhere bool) +} + +// policyApplier turns each new policy snapshot into setter calls. Resolvers +// are installed once, the first time hive reports the value, and then read +// the live snapshot; the pushed values (sandbox config, dead-session owner) +// are set again whenever they change. A value hive does not report leaves +// the runtime's own default in place. +type policyApplier struct { + target hivePolicyTarget + policy *hivecap.Policy + errw io.Writer + + mu sync.Mutex + installed map[string]bool + sandbox []byte + owner *bool +} + +func newPolicyApplier(target hivePolicyTarget, policy *hivecap.Policy, errw io.Writer) *policyApplier { + return &policyApplier{target: target, policy: policy, errw: errw, installed: map[string]bool{}} +} + +func (a *policyApplier) apply(pol callback.Policy) { + a.mu.Lock() + defer a.mu.Unlock() + once := func(name string, present bool, install func()) { + if present && !a.installed[name] { + a.installed[name] = true + install() + } + } + p := a.policy + once("build", pol.Build != nil, func() { a.target.SetBuildInfo(p.BuildInfo) }) + once("explain_mode_default", pol.ExplainModeDefault != nil, func() { a.target.SetExplainModeDefaultResolver(p.ExplainModeDefault) }) + once("kick_log_source", pol.KickLogSource != nil, func() { a.target.SetKickLogSourceResolver(p.KickLogSource) }) + once("gateway_backends", pol.GatewayBackends != nil, func() { a.target.SetGatewayBackendChecker(p.GatewayBackend) }) + once("repo_auto_merge", pol.RepoAutoMerge, func() { a.target.SetRepoAutoMergeEnabledResolver(p.RepoAutoMergeEnabled) }) + if v := pol.DeadSessionRecoveryOwnedElsewhere; v != nil && (a.owner == nil || *a.owner != *v) { + owned := *v + a.owner = &owned + a.target.SetDeadSessionRecoveryOwner(owned) + } + if len(pol.Sandbox) > 0 && !bytes.Equal(pol.Sandbox, a.sandbox) { + var cfg config.AgentSandboxConfig + if err := json.Unmarshal(pol.Sandbox, &cfg); err != nil { + fmt.Fprintf(a.errw, "crux: hive policy: undecodable sandbox config, keeping the previous one: %v\n", err) + return + } + a.sandbox = append([]byte(nil), pol.Sandbox...) + a.target.SetSandboxConfig(cfg) + } +} + +// hiveCallback is `crux serve --hive-callback`: the hive capability endpoint +// the runtime dials for the capabilities it does not own. +type hiveCallback struct { + client *callback.Client + set *hivecap.Set +} + +// dialHiveCallback builds the client and adapters for addr (a form +// api.ParseAddress accepts), presenting token when non-empty. Nothing is +// called yet: every adapter fails closed until hive answers, and +// hivecap.Watch reports the outage. +func dialHiveCallback(addr, token string) (*hiveCallback, error) { + c, err := callback.NewClient(addr, token) + if err != nil { + return nil, fmt.Errorf("hive callback %s: %w", addr, err) + } + return &hiveCallback{client: c, set: hivecap.New(c, 0)}, nil +} + +// install feeds the manager and process-wide seams from the adapters, +// reports the endpoint on the server's health route, and starts the +// condition watch, which ends with ctx. It reports to w which seams were +// wired so an operator can see the difference between a runtime that is a +// manager and one that is not. +func (h *hiveCallback) install(ctx context.Context, rt crux.Runtime, srv *serve.Server, w io.Writer) { + srv.HiveCallback = h.health + // Conditions go through the server's tee, not the manager setter: Serve + // installs its own reporter on a Host runtime (so conditions reach the + // event stream) and forwards to srv.ConditionReporter; setting the + // manager directly would be overwritten when Serve starts. + srv.ConditionReporter = h.set.Conditions + toolapprove.SetDefaultTextScanner(h.set.Text) + seams := "text scanner, conditions" + if m, ok := rt.(hiveCapTarget); ok { + m.SetSandboxPRClient(h.set.PullRequest) + linear := h.set.Linear + m.SetLinearCredentialResolver(func() agent.LinearCredential { + tok, key := linear.Resolve() + return agent.LinearCredential{AccessToken: tok, APIKey: key} + }) + m.SetSandboxPushMinter(pushbroker.GitHubAppMinter{Auth: h.set.TokenScoper("", 0)}) + // Routes are pushed by a worker so the manager's launch path never + // waits on hive; the boundary fails closed when hive is away. + m.SetInferenceCallbacks(h.set.Inference.Set, h.set.Inference.Clear) + go h.set.Inference.Run(ctx, 0) + m.SetSandboxMutationBoundary(h.set.Mutation) + // Credential writers: App tokens and the opt-in mint are minted by + // hive and only cached here; Bob's key is read through hive. All + // fail closed (no credential) when hive is away. + m.SetAppAuth(h.set.AppTokens) + m.SetAgentMint(h.set.Mint) + m.SetBobAPIKeyResolver(h.set.Bob.Key) + m.SetBobKeySourceResolver(h.set.Bob.Source) + seams += ", pull requests, Linear credential, push tokens, inference routes, mutation boundary, App tokens, agent mint, Bob credential" + } + fmt.Fprintf(w, "hive callback %s: %s\n", addressString(h.client.Address()), seams) + // hive_callback_unavailable describes the callback being down, so it + // goes to the stream only and never over the callback. + go hivecap.Watch(ctx, h.client.Status, srv.StreamConditions(), 0) +} + +// watchPolicy polls hive's policy snapshot (GET /v1/policy) until ctx ends +// and feeds it to rt's crux.Host policy setters (#14138 stage 4). A runtime +// without them is left alone; a hive without the policy capability answers +// not_configured and the runtime keeps its own configuration. +func (h *hiveCallback) watchPolicy(ctx context.Context, rt crux.Runtime, w, errw io.Writer) { + m, ok := rt.(hivePolicyTarget) + if !ok { + return + } + go hivecap.WatchPolicy(ctx, h.set.Policy, 0, newPolicyApplier(m, h.set.Policy, errw).apply) + fmt.Fprintf(w, "hive callback %s: host policy\n", addressString(h.client.Address())) +} + +// health is the status Health reports. +func (h *hiveCallback) health() api.HiveCallback { + st := h.client.Status() + out := api.HiveCallback{Address: addressString(h.client.Address()), Reachable: st.Reachable, LastError: st.LastError, FailedCalls: st.FailedCalls} + if !st.Since.IsZero() { + since := st.Since + out.Since = &since + } + if !st.LastSuccess.IsZero() { + last := st.LastSuccess + out.LastSuccess = &last + } + return out +} + +// addressString renders an address the way serve.Listeners.Addrs does. +func addressString(a api.Address) string { + if a.Network == "unix" { + return "unix://" + a.Path + } + return a.BaseURL +} diff --git a/src/crux/cli/hive_callback_test.go b/src/crux/cli/hive_callback_test.go new file mode 100644 index 0000000000..4b74881742 --- /dev/null +++ b/src/crux/cli/hive_callback_test.go @@ -0,0 +1,250 @@ +package cli + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/cruxtest" + "github.com/hivecommons/hive/crux/effects" + "github.com/hivecommons/hive/crux/pushbroker" + "github.com/hivecommons/hive/crux/serve" + "github.com/hivecommons/hive/crux/toolapprove" +) + +// managerLike is a fake runtime with the manager setters the hive callback +// feeds; it records what it was handed. +type managerLike struct { + *fakeRuntime + mu sync.Mutex + prs agent.PRCreator + linear func() agent.LinearCredential + minter pushbroker.TokenMinter + route func(agentName, backend, model string) + clear func(agentName string) + bound effects.Boundary + app agent.AppTokenMinter + mint agent.AgentMintIssuer + bobKey func() string + bobSrc func() string +} + +func (m *managerLike) SetAppAuth(a agent.AppTokenMinter) { + m.mu.Lock() + defer m.mu.Unlock() + m.app = a +} + +func (m *managerLike) SetAgentMint(i agent.AgentMintIssuer) { + m.mu.Lock() + defer m.mu.Unlock() + m.mint = i +} + +func (m *managerLike) SetBobAPIKeyResolver(fn func() string) { + m.mu.Lock() + defer m.mu.Unlock() + m.bobKey = fn +} + +func (m *managerLike) SetBobKeySourceResolver(fn func() string) { + m.mu.Lock() + defer m.mu.Unlock() + m.bobSrc = fn +} + +func (m *managerLike) SetInferenceCallbacks(set func(agentName, backend, model string), clear func(agentName string)) { + m.mu.Lock() + defer m.mu.Unlock() + m.route, m.clear = set, clear +} + +func (m *managerLike) SetSandboxMutationBoundary(b effects.Boundary) { + m.mu.Lock() + defer m.mu.Unlock() + m.bound = b +} + +func (m *managerLike) SetSandboxPRClient(c agent.PRCreator) { + m.mu.Lock() + defer m.mu.Unlock() + m.prs = c +} + +func (m *managerLike) SetLinearCredentialResolver(fn func() agent.LinearCredential) { + m.mu.Lock() + defer m.mu.Unlock() + m.linear = fn +} + +func (m *managerLike) SetSandboxPushMinter(t pushbroker.TokenMinter) { + m.mu.Lock() + defer m.mu.Unlock() + m.minter = t +} + +func TestServeHiveCallbackFeedsTheManagerSeams(t *testing.T) { + t.Setenv(config.ServeTokenEnvVar, "") + t.Setenv(callback.TokenEnvVar, "") + t.Cleanup(func() { toolapprove.SetDefaultTextScanner(nil) }) + h := newHarness(t) + m := &managerLike{fakeRuntime: h.rt} + h.env.newRuntime = func(map[string]config.AgentConfig, crux.ProjectContext) crux.Runtime { return m } + dir := shortSocketDir(t) + socket := filepath.Join(dir, "api.sock") + // Nothing listens on the hive socket: every capability fails closed and + // the outage is reported. + hiveSocket := filepath.Join(dir, "hive.sock") + t.Setenv(callback.SocketEnvVar, hiveSocket) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan int, 1) + go func() { + done <- h.env.Run(ctx, append(h.workDirArgs(), "serve", "--socket", socket)) + }() + waitForSocket(t, socket) + + m.mu.Lock() + linear, prs, minter, route, clear, bound := m.linear, m.prs, m.minter, m.route, m.clear, m.bound + app, mint, bobKey, bobSrc := m.app, m.mint, m.bobKey, m.bobSrc + m.mu.Unlock() + if linear == nil || prs == nil || minter == nil || route == nil || clear == nil || bound == nil { + t.Fatalf("seams not fed: linear=%v prs=%v minter=%v route=%v clear=%v boundary=%v", linear != nil, prs != nil, minter != nil, route != nil, clear != nil, bound != nil) + } + if app == nil || mint == nil || bobKey == nil || bobSrc == nil { + t.Fatalf("credential seams not fed: app=%v mint=%v bobKey=%v bobSrc=%v", app != nil, mint != nil, bobKey != nil, bobSrc != nil) + } + if toolapprove.DefaultTextScanner() == nil { + t.Fatal("text scanner not installed") + } + + health := func() api.Health { + t.Helper() + resp, err := socketClient(socket).Get("http://crux" + api.PathHealth) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out api.Health + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatal(err) + } + return out + } + if hc := health().HiveCallback; hc == nil || !hc.Reachable || hc.Address != "unix://"+hiveSocket { + t.Fatalf("health before any call: %+v", hc) + } + if cred := linear(); !cred.Empty() { + t.Fatalf("linear over a dead socket = %+v, want empty", cred) + } + if _, err := minter.MintPushToken(ctx, "repo"); err == nil { + t.Fatal("mint over a dead socket must fail") + } + // Credential writers fail closed: no token written, no mint, no Bob key. + if err := app.WriteAgentToken(ctx, "a", "t", 0); err == nil { + t.Fatal("App token over a dead socket must fail") + } + if tok, err := mint.MintAgentToken("a", "t"); tok != "" || err == nil { + t.Fatalf("agent mint over a dead socket = %q, %v", tok, err) + } + if bobKey() != "" || bobSrc() != "" { + t.Fatal("Bob credential over a dead socket must be empty") + } + // The route callbacks return at once even with hive away; the boundary + // refuses the effect rather than running it unjournaled. + route("a", "litellm", "m") + clear("a") + if _, err := bound.Execute(ctx, effects.Claim{Repo: "o/r", Kind: effects.KindBranchPush, Target: "b"}, func(context.Context) (effects.Result, error) { + t.Fatal("effect ran with hive unreachable") + return effects.Result{}, nil + }); err == nil { + t.Fatal("boundary over a dead socket must refuse") + } + if hc := health().HiveCallback; hc == nil || hc.Reachable || hc.LastError == "" || hc.FailedCalls < 2 || hc.Since == nil { + t.Fatalf("health after failed calls: %+v", hc) + } + cancel() + <-done + if !strings.Contains(h.stdout.String(), "hive callback unix://"+hiveSocket+": text scanner, conditions, pull requests, Linear credential, push tokens, inference routes, mutation boundary, App tokens, agent mint, Bob credential") { + t.Fatalf("stdout = %q", h.stdout.String()) + } +} + +func TestServeHiveCallbackOnAPlainRuntime(t *testing.T) { + t.Setenv(config.ServeTokenEnvVar, "") + t.Setenv(callback.SocketEnvVar, "") + t.Cleanup(func() { toolapprove.SetDefaultTextScanner(nil) }) + h := newHarness(t) + socket := filepath.Join(shortSocketDir(t), "api.sock") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan int, 1) + go func() { + done <- h.env.Run(ctx, append(h.workDirArgs(), "serve", "--socket", socket, "--hive-callback", "http://127.0.0.1:1")) + }() + waitForSocket(t, socket) + cancel() + if code := <-done; code != ExitOK { + t.Fatalf("exit %d stderr %q", code, h.stderr) + } + if out := h.stdout.String(); !strings.Contains(out, "hive callback http://127.0.0.1:1: text scanner, conditions\n") { + t.Fatalf("stdout = %q", out) + } +} + +func TestServeHiveCallbackBadAddress(t *testing.T) { + t.Setenv(config.ServeTokenEnvVar, "") + h := newHarness(t) + if code := h.env.Run(context.Background(), append(h.workDirArgs(), "serve", "--hive-callback", "ftp://nope")); code != ExitUsage { + t.Fatalf("exit %d stderr %q", code, h.stderr) + } + if !strings.Contains(h.stderr.String(), "hive callback ftp://nope") { + t.Fatalf("stderr = %q", h.stderr.String()) + } +} + +// Conditions reach hive through the server's tee, which Serve installs on a +// Host runtime and which forwards to srv.ConditionReporter; the manager +// setter would be overwritten when Serve starts. The watch's own +// hive_callback_unavailable goes to the stream only, never over the +// callback it describes. +func TestHiveCallbackInstallRoutesConditionsThroughTheServerTee(t *testing.T) { + t.Cleanup(func() { toolapprove.SetDefaultTextScanner(nil) }) + hiveSocket := filepath.Join(shortSocketDir(t), "hive.sock") + h, err := dialHiveCallback(hiveSocket, "") + if err != nil { + t.Fatal(err) + } + rec := cruxtest.NewRecorder() + srv := serve.New(rec) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var out strings.Builder + h.install(ctx, rec, srv, &out) + + if srv.ConditionReporter != h.set.Conditions { + t.Fatalf("srv.ConditionReporter = %T, want the hivecap reporter", srv.ConditionReporter) + } + if !strings.Contains(out.String(), "conditions") { + t.Fatalf("banner does not list conditions: %q", out.String()) + } + // Through the tee, a runtime condition is attempted over the (dead) + // callback: fails closed and counts as a failed call. + before := h.client.Status().FailedCalls + srv.Conditions().Observe("stall", "scanner", "m", "d") + if after := h.client.Status().FailedCalls; after != before+1 { + t.Fatalf("condition not forwarded over the callback: failed calls %d → %d", before, after) + } + // The stream-only reporter never touches the callback. + srv.StreamConditions().Observe(callback.ConditionClassUnavailable, "endpoint", "down", "") + if after := h.client.Status().FailedCalls; after != before+1 { + t.Fatalf("stream-only condition travelled over the callback: failed calls %d", after) + } +} diff --git a/src/crux/cli/hive_policy_test.go b/src/crux/cli/hive_policy_test.go new file mode 100644 index 0000000000..b4e51507c1 --- /dev/null +++ b/src/crux/cli/hive_policy_test.go @@ -0,0 +1,208 @@ +package cli + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/hivecap" + "github.com/hivecommons/hive/crux/internal/testutil" +) + +// policyTarget records what the policy applier hands the runtime. +type policyTarget struct { + *fakeRuntime + mu sync.Mutex + build func() (string, string) + gateway func(string) bool + explain func() string + autoMerge func(string) bool + kickLog func() string + sandbox []config.AgentSandboxConfig + owner []bool +} + +func (p *policyTarget) SetBuildInfo(fn func() (string, string)) { + p.mu.Lock() + defer p.mu.Unlock() + p.build = fn +} + +func (p *policyTarget) SetGatewayBackendChecker(fn func(string) bool) { + p.mu.Lock() + defer p.mu.Unlock() + p.gateway = fn +} + +func (p *policyTarget) SetExplainModeDefaultResolver(fn func() string) { + p.mu.Lock() + defer p.mu.Unlock() + p.explain = fn +} + +func (p *policyTarget) SetRepoAutoMergeEnabledResolver(fn func(string) bool) { + p.mu.Lock() + defer p.mu.Unlock() + p.autoMerge = fn +} + +func (p *policyTarget) SetKickLogSourceResolver(fn func() string) { + p.mu.Lock() + defer p.mu.Unlock() + p.kickLog = fn +} + +func (p *policyTarget) SetSandboxConfig(cfg config.AgentSandboxConfig) { + p.mu.Lock() + defer p.mu.Unlock() + p.sandbox = append(p.sandbox, cfg) +} + +func (p *policyTarget) SetDeadSessionRecoveryOwner(owned bool) { + p.mu.Lock() + defer p.mu.Unlock() + p.owner = append(p.owner, owned) +} + +var _ hivePolicyTarget = (*policyTarget)(nil) + +// policyHive is a hive callback that serves only the policy half. +type policyHive struct { + callback.Capabilities + mu sync.Mutex + policy callback.Policy +} + +func (h *policyHive) Configured(c string) bool { return c == callback.CapabilityPolicy } + +func (h *policyHive) Policy(context.Context) (callback.Policy, error) { + h.mu.Lock() + defer h.mu.Unlock() + return h.policy, nil +} + +func (h *policyHive) RepoAutoMergeEnabled(_ context.Context, repo string) (bool, error) { + return repo == "o/merge", nil +} + +func (h *policyHive) set(p callback.Policy) { + h.mu.Lock() + defer h.mu.Unlock() + h.policy = p +} + +// Resolvers are installed once, when hive first reports the value; pushed +// values are set again only when they change; absent values leave the +// runtime's own defaults alone. +func TestPolicyApplier(t *testing.T) { + explain := "on" + owned := true + gateways := []string{"gw"} + hive := &policyHive{policy: callback.Policy{ + ExplainModeDefault: &explain, + GatewayBackends: &gateways, + DeadSessionRecoveryOwnedElsewhere: &owned, + Sandbox: json.RawMessage(`{"enabled":true,"image":"img"}`), + }} + pol := hivecap.NewPolicy(hive, time.Second) + target := &policyTarget{fakeRuntime: &fakeRuntime{}} + var errw strings.Builder + a := newPolicyApplier(target, pol, &errw) + + first, err := pol.Refresh(context.Background()) + if err != nil { + t.Fatal(err) + } + a.apply(first) + if target.explain == nil || target.explain() != "on" || target.gateway == nil || !target.gateway("GW") { + t.Fatal("resolvers not installed from the first snapshot") + } + if target.build != nil || target.kickLog != nil || target.autoMerge != nil { + t.Fatal("absent values installed a resolver") + } + if len(target.owner) != 1 || !target.owner[0] || len(target.sandbox) != 1 || !target.sandbox[0].Enabled || target.sandbox[0].Image != "img" { + t.Fatalf("pushed values = owner %v sandbox %+v", target.owner, target.sandbox) + } + + a.apply(first) + if len(target.owner) != 1 || len(target.sandbox) != 1 { + t.Fatal("unchanged values set again") + } + + explain2, owned2 := "off", false + hive.set(callback.Policy{ + Build: &callback.BuildInfo{Version: "v5", Commit: "c"}, + ExplainModeDefault: &explain2, + RepoAutoMerge: true, + DeadSessionRecoveryOwnedElsewhere: &owned2, + Sandbox: json.RawMessage(`{"enabled":false}`), + }) + second, err := pol.Refresh(context.Background()) + if err != nil { + t.Fatal(err) + } + installedExplain := target.explain + a.apply(second) + if target.explain() != "off" || installedExplain() != "off" { + t.Fatal("explain resolver does not read the live snapshot") + } + if v, c := target.build(); v != "v5" || c != "c" { + t.Fatalf("build = %s %s", v, c) + } + if target.autoMerge == nil || !target.autoMerge("o/merge") || target.autoMerge("o/other") { + t.Fatal("auto-merge resolver wrong") + } + if len(target.owner) != 2 || target.owner[1] || len(target.sandbox) != 2 || target.sandbox[1].Enabled { + t.Fatalf("changed values = owner %v sandbox %+v", target.owner, target.sandbox) + } + + a.apply(callback.Policy{Sandbox: json.RawMessage(`"not an object"`)}) + if len(target.sandbox) != 2 || !strings.Contains(errw.String(), "undecodable sandbox config") { + t.Fatalf("bad sandbox applied or not reported: %d, %q", len(target.sandbox), errw.String()) + } +} + +// watchPolicy feeds a policy-capable runtime from the hive callback, and +// leaves a runtime without the setters alone. +func TestWatchPolicyOverTheCallback(t *testing.T) { + explain := "on" + hive := &policyHive{policy: callback.Policy{ExplainModeDefault: &explain}} + sock := filepath.Join(shortSocketDir(t), "hive.sock") + ls, err := callback.Listen(callback.ListenConfig{Socket: sock}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = callback.NewServer(hive, "").Serve(ctx, ls) }() + + h, err := dialHiveCallback(sock, "") + if err != nil { + t.Fatal(err) + } + var out, errw strings.Builder + h.watchPolicy(ctx, &fakeRuntime{}, &out, &errw) + if out.Len() != 0 { + t.Fatalf("plain runtime announced host policy: %q", out.String()) + } + + target := &policyTarget{fakeRuntime: &fakeRuntime{}} + h.watchPolicy(ctx, target, &out, &errw) + if !strings.Contains(out.String(), ": host policy") { + t.Fatalf("banner = %q", out.String()) + } + explainFn := testutil.EventuallyValue(t, 5*time.Second, func() (func() string, bool) { + target.mu.Lock() + defer target.mu.Unlock() + return target.explain, target.explain != nil + }, "policy never applied over the callback") + if got := explainFn(); got != "on" { + t.Fatalf("explain = %q", got) + } +} diff --git a/src/pkg/crux/cli/loader_test.go b/src/crux/cli/loader_test.go similarity index 97% rename from src/pkg/crux/cli/loader_test.go rename to src/crux/cli/loader_test.go index 7c4761cbdb..827afcd356 100644 --- a/src/pkg/crux/cli/loader_test.go +++ b/src/crux/cli/loader_test.go @@ -9,8 +9,8 @@ import ( "gopkg.in/yaml.v3" - "github.com/hivecommons/hive/pkg/crux" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" ) // fakePolicyDir is what fakeLoader derives as the ProjectContext policy dir, diff --git a/src/crux/cli/remote.go b/src/crux/cli/remote.go new file mode 100644 index 0000000000..68a0efab61 --- /dev/null +++ b/src/crux/cli/remote.go @@ -0,0 +1,267 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" +) + +// clientFactory builds the API client remote mode uses. Production uses +// api.NewClient; tests may substitute one bound to an httptest server. +type clientFactory func(server, token string) (*api.Client, error) + +// remoteSelected reports whether this invocation drives a `crux serve` +// instead of an in-process runtime: --server was given or CRUX_SERVER is +// set. The token comes from --token, then CRUX_TOKEN. `help` and `serve` +// ignore the selection; everything else goes through remote. +func (o *globalOptions) remoteSelected() bool { + if o.server == "" { + o.server = os.Getenv(config.ServerEnvVar) + } + if o.server != "" && o.token == "" { + o.token = os.Getenv(config.ServeTokenEnvVar) + } + return o.server != "" +} + +// remote routes a subcommand to a `crux serve` over the Crux API. Every +// verb maps onto one client call; `run` without -detach follows the log +// stream and stops the agent when ctx ends, as the in-process `run` does. +func (e *Env) remote(ctx context.Context, opts globalOptions, name string, args []string) int { + c, err := e.newClient(opts.server, opts.token) + if err != nil { + fmt.Fprintf(e.Stderr, "crux: --server %q: %v\n", opts.server, err) + return ExitUsage + } + switch name { + case "run": + return e.remoteRun(ctx, c, opts, args) + case "ps": + return e.remotePS(ctx, c, args) + case "logs": + return e.remoteLogs(ctx, c, args) + case "kick": + return e.remoteKick(ctx, c, args) + case "stop": + return e.remoteStop(ctx, c, args) + default: + fmt.Fprintf(e.Stderr, "crux: %q is not available with --server; it runs where the runtime is\n", name) + return ExitUsage + } +} + +// remoteErr prints a server-side failure. Usage-class answers (404 for an +// unknown agent, 400 for a bad request) are the caller's mistake; the rest +// are runtime errors. +func (e *Env) remoteErr(verb, name string, err error) int { + if name != "" { + fmt.Fprintf(e.Stderr, "crux: %s %s: %v\n", verb, name, err) + } else { + fmt.Fprintf(e.Stderr, "crux: %s: %v\n", verb, err) + } + if api.IsUnauthorized(err) { + fmt.Fprintf(e.Stderr, "crux: the server wants a bearer token; pass --token or set %s\n", config.ServeTokenEnvVar) + } + return ExitError +} + +// remoteRun is `crux --server … run`: a definition file is loaded here and +// sent as RunRequest.Definition; a bare name must already be defined on the +// server. Without -detach the ring buffer is followed over SSE until ctx +// ends, then the agent is stopped. +func (e *Env) remoteRun(ctx context.Context, c *api.Client, opts globalOptions, args []string) int { + fs := e.subFlags("run") + detach := fs.Bool("detach", false, "return after Start; do not follow or stop the agent") + _ = fs.Duration("interval", defaultStreamInterval, "ignored with --server; the server paces the stream") + _ = fs.Bool("no-socket", false, "ignored with --server; no control socket is served") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(e.Stderr, "usage: crux --server ADDR run [-detach] ") + return ExitUsage + } + target := fs.Arg(0) + req := api.RunRequest{Name: target} + if isDefinitionPath(target) { + defName, cfg, err := e.Loader.LoadDefinitionFile(target) + if err != nil { + fmt.Fprintln(e.Stderr, "crux:", err) + return ExitError + } + raw, err := json.Marshal(cfg) + if err != nil { + fmt.Fprintln(e.Stderr, "crux: encoding definition:", err) + return ExitError + } + req.Name, req.Definition = defName, raw + } + + resp, err := c.Run(ctx, req) + if err != nil { + return e.remoteErr("start", req.Name, err) + } + name := resp.Agent + fmt.Fprintf(e.Stdout, "started %s\n", name) + if *detach { + return ExitOK + } + + followErr := c.FollowLogs(ctx, name, func(chunk api.LogChunk) error { + for _, line := range chunk.Lines { + fmt.Fprintln(e.Stdout, line) + } + return nil + }) + if followErr != nil { + fmt.Fprintf(e.Stderr, "crux: output %s: %v\n", name, followErr) + } + + // ctx is done (or the stream broke); stop with a fresh context so the + // request still goes out. + if _, err := c.Stop(context.WithoutCancel(ctx), name); err != nil { + return e.remoteErr("stop", name, err) + } + fmt.Fprintf(e.Stdout, "stopped %s\n", name) + return ExitOK +} + +// remotePS is `crux --server … ps`: GET /v1/agents. +func (e *Env) remotePS(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("ps") + asJSON := fs.Bool("json", false, "emit a JSON array instead of a table") + if code, ok := parseSub(fs, args); !ok { + return code + } + procs, err := c.List(ctx) + if err != nil { + return e.remoteErr("ps", "", err) + } + rows := make([]processRow, 0, len(procs)) + for _, p := range procs { + rows = append(rows, processRow{ + Name: p.Name, + ID: p.ID, + State: p.State, + Backend: p.Backend, + Model: p.Model, + Paused: p.Paused, + StartedAt: p.StartedAt, + LastKick: p.LastKick, + RestartCount: p.RestartCount, + }) + } + if *asJSON { + return e.writeJSON(rows) + } + return e.printPS(rows) +} + +// remoteLogs is `crux --server … logs`: GET /v1/agents/{name}/logs, with -f +// following the ring buffer over SSE until ctx ends. +func (e *Env) remoteLogs(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("logs") + lines := fs.Int("n", defaultLogLines, "lines to print from the pane or buffer") + buffer := fs.Bool("buffer", false, "read the ring buffer instead of the live pane") + full := fs.Bool("full", false, "capture the whole scrollback") + follow := fs.Bool("f", false, "follow the ring buffer until interrupted") + asJSON := fs.Bool("json", false, "emit JSON instead of raw lines") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(e.Stderr, "usage: crux --server ADDR logs [-n N] [-buffer] [-full] [-f] [-json] ") + return ExitUsage + } + if (*buffer && *full) || (*follow && (*full || *asJSON)) { + fmt.Fprintln(e.Stderr, "crux: -buffer and -full are exclusive; -f takes neither -full nor -json") + return ExitUsage + } + name := fs.Arg(0) + if *follow { + err := c.FollowLogs(ctx, name, func(chunk api.LogChunk) error { + for _, line := range chunk.Lines { + fmt.Fprintln(e.Stdout, line) + } + return nil + }) + if err != nil { + return e.remoteErr("logs", name, err) + } + return ExitOK + } + source := api.LogSourcePane + switch { + case *full: + source = api.LogSourceFull + case *buffer: + source = api.LogSourceBuffer + } + logs, err := c.Logs(ctx, name, source, *lines) + if err != nil { + return e.remoteErr("logs", name, err) + } + out := logsOutput{Agent: logs.Agent, Source: logs.Source, Header: logs.Header, Lines: logs.Lines, Content: logs.Content} + if *asJSON { + return e.writeJSON(out) + } + e.printLogs(out, *full) + return ExitOK +} + +// remoteKick is `crux --server … kick`: POST /v1/agents/{name}/kick. +func (e *Env) remoteKick(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("kick") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() < 2 { + fmt.Fprintln(e.Stderr, "usage: crux kick ") + return ExitUsage + } + name := fs.Arg(0) + message := strings.Join(fs.Args()[1:], " ") + if message == "-" { + data, err := io.ReadAll(e.Stdin) + if err != nil { + fmt.Fprintln(e.Stderr, "crux: reading kick from stdin:", err) + return ExitError + } + message = strings.TrimRight(string(data), "\n") + } + if strings.TrimSpace(message) == "" { + fmt.Fprintln(e.Stderr, "crux: kick message is empty") + return ExitUsage + } + resp, err := c.Kick(ctx, name, message) + if err != nil { + return e.remoteErr("kick", name, err) + } + fmt.Fprintf(e.Stdout, "kicked %s\n", resp.Agent) + return ExitOK +} + +// remoteStop is `crux --server … stop`: POST /v1/agents/{name}/stop. +func (e *Env) remoteStop(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("stop") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(e.Stderr, "usage: crux stop ") + return ExitUsage + } + name := fs.Arg(0) + resp, err := c.Stop(ctx, name) + if err != nil { + return e.remoteErr("stop", name, err) + } + fmt.Fprintf(e.Stdout, "stopped %s\n", resp.Agent) + return ExitOK +} diff --git a/src/crux/cli/remote_test.go b/src/crux/cli/remote_test.go new file mode 100644 index 0000000000..b1b38ff871 --- /dev/null +++ b/src/crux/cli/remote_test.go @@ -0,0 +1,409 @@ +package cli + +import ( + "context" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" +) + +// fakeAPI is a `crux serve` stand-in speaking crux/api, so remote mode is +// tested against the wire contract with full control over timing. +type fakeAPI struct { + mu sync.Mutex + token string + requests []string + run api.RunRequest + kick string + // hold keeps a followed log stream open until closed. + hold chan struct{} +} + +func (f *fakeAPI) seen() string { + f.mu.Lock() + defer f.mu.Unlock() + return strings.Join(f.requests, "\n") +} + +func (f *fakeAPI) handler() http.Handler { + mux := http.NewServeMux() + wrap := func(h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + f.requests = append(f.requests, r.Method+" "+r.URL.RequestURI()) + f.mu.Unlock() + if f.token != "" && r.Header.Get("Authorization") != "Bearer "+f.token { + w.WriteHeader(http.StatusUnauthorized) + _ = json.NewEncoder(w).Encode(api.Error{Error: "token required", Code: api.CodeUnauthorized}) + return + } + h(w, r) + } + } + notFound := func(w http.ResponseWriter, name string) bool { + if name != "ghost" { + return false + } + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(api.Error{Error: "agent ghost not found", Code: api.CodeNotFound}) + return true + } + started := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + mux.HandleFunc("GET "+api.PathAgents, wrap(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(api.ProcessList{Processes: []api.Process{ + {Name: "fixer", ID: "f1", State: "running", Backend: "claude", Model: "opus", StartedAt: &started, RestartCount: 2}, + {Name: "scout", ID: "s1", State: "paused", Backend: "codex", Paused: true}, + }}) + })) + mux.HandleFunc("POST "+api.PathAgents, wrap(func(w http.ResponseWriter, r *http.Request) { + var req api.RunRequest + _ = json.NewDecoder(r.Body).Decode(&req) + f.mu.Lock() + f.run = req + f.mu.Unlock() + if notFound(w, req.Name) { + return + } + _ = json.NewEncoder(w).Encode(api.AgentResponse{Agent: req.Name}) + })) + mux.HandleFunc("GET "+api.PathAgents+"/{name}/logs", wrap(func(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if notFound(w, name) { + return + } + q := r.URL.Query() + if q.Get(api.QueryFollow) == "" { + out := api.Logs{Agent: name, Source: q.Get(api.QuerySource)} + if out.Source == api.LogSourceFull { + out.Header, out.Content = "# header", "all of it" + } else { + out.Lines = []string{out.Source + " n=" + q.Get(api.QueryLines)} + } + _ = json.NewEncoder(w).Encode(out) + return + } + w.Header().Set("Content-Type", api.ContentTypeEvent) + fl := w.(http.Flusher) + _ = api.WriteEvent(w, api.EventLog, api.LogChunk{Agent: name, Lines: []string{"boot", "ready"}}) + fl.Flush() + if name == "lost" { + _ = api.WriteEvent(w, api.EventState, api.Event{Type: api.EventState, Agent: name, Detail: "agent lost not found"}) + fl.Flush() + return + } + if f.hold != nil { + select { + case <-f.hold: + case <-r.Context().Done(): + } + } + })) + mux.HandleFunc("POST "+api.PathAgents+"/{name}/kick", wrap(func(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if notFound(w, name) { + return + } + var req api.KickRequest + _ = json.NewDecoder(r.Body).Decode(&req) + f.mu.Lock() + f.kick = req.Message + f.mu.Unlock() + _ = json.NewEncoder(w).Encode(api.AgentResponse{Agent: name}) + })) + mux.HandleFunc("POST "+api.PathAgents+"/{name}/stop", wrap(func(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if notFound(w, name) { + return + } + _ = json.NewEncoder(w).Encode(api.AgentResponse{Agent: name}) + })) + return mux +} + +// waitUntil polls cond on a short ticker until it holds or the deadline +// passes; the nested module cannot import testutil.Eventually. +func waitUntil(t *testing.T, what string, cond func() bool) { + t.Helper() + tick := time.NewTicker(2 * time.Millisecond) + defer tick.Stop() + deadline := time.After(5 * time.Second) + for { + if cond() { + return + } + select { + case <-tick.C: + case <-deadline: + t.Fatalf("timed out waiting for %s", what) + } + } +} + +// lockedBuffer is a bytes.Buffer safe to read while a command writes it. +type lockedBuffer struct { + mu sync.Mutex + buf strings.Builder +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// startFakeAPI serves f on a unix socket and returns the socket path. +func startFakeAPI(t *testing.T, f *fakeAPI) string { + t.Helper() + path := filepath.Join(shortSocketDir(t), "api.sock") + ln, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewUnstartedServer(f.handler()) + srv.Listener = ln + srv.Start() + t.Cleanup(srv.Close) + return path +} + +func TestRemotePSLogsKickStop(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + f := &fakeAPI{token: "tok"} + socket := startFakeAPI(t, f) + + h := newHarness(t) + if code := h.run("--server", socket, "--token", "tok", "ps"); code != ExitOK { + t.Fatalf("ps: %d %s", code, h.stderr) + } + out := h.stdout.String() + if !strings.Contains(out, "NAME") || !strings.Contains(out, "fixer f1 running") || !strings.Contains(out, "paused (paused)") || !strings.Contains(out, "opus") { + t.Fatalf("ps table:\n%s", out) + } + if len(h.rt.Calls()) != 0 { + t.Fatalf("remote ps touched the local runtime: %v", h.rt.Calls()) + } + + h = newHarness(t) + if code := h.run("--server", socket, "--token", "tok", "ps", "--json"); code != ExitOK { + t.Fatalf("ps --json: %d %s", code, h.stderr) + } + var rows []processRow + if err := json.Unmarshal(h.stdout.Bytes(), &rows); err != nil || len(rows) != 2 || rows[0].RestartCount != 2 { + t.Fatalf("ps json %v: %s", err, h.stdout) + } + + for _, c := range []struct { + args []string + want string + }{ + {[]string{"logs", "fixer"}, "pane n=100\n"}, + {[]string{"logs", "-n", "5", "-buffer", "fixer"}, "buffer n=5\n"}, + {[]string{"logs", "-full", "fixer"}, "# header\nall of it\n"}, + } { + h = newHarness(t) + if code := h.run(append([]string{"--server", socket, "--token", "tok"}, c.args...)...); code != ExitOK || h.stdout.String() != c.want { + t.Fatalf("%v: code %d stdout %q stderr %q", c.args, code, h.stdout, h.stderr) + } + } + h = newHarness(t) + if code := h.run("--server", socket, "--token", "tok", "logs", "-json", "-buffer", "fixer"); code != ExitOK { + t.Fatalf("logs -json: %d %s", code, h.stderr) + } + var lo logsOutput + if err := json.Unmarshal(h.stdout.Bytes(), &lo); err != nil || lo.Source != "buffer" || lo.Agent != "fixer" { + t.Fatalf("logs json %v: %s", err, h.stdout) + } + + h = newHarness(t) + h.env.Stdin = strings.NewReader("from stdin\n") + if code := h.run("--server", socket, "--token", "tok", "kick", "fixer", "-"); code != ExitOK || h.stdout.String() != "kicked fixer\n" { + t.Fatalf("kick: %d %q %q", code, h.stdout, h.stderr) + } + if f.kick != "from stdin" { + t.Fatalf("kick message %q", f.kick) + } + h = newHarness(t) + if code := h.run("--server", socket, "--token", "tok", "kick", "fixer", "do", "it"); code != ExitOK || f.kick != "do it" { + t.Fatalf("kick words: %d %q", code, f.kick) + } + + h = newHarness(t) + if code := h.run("--server", socket, "--token", "tok", "stop", "fixer"); code != ExitOK || h.stdout.String() != "stopped fixer\n" { + t.Fatalf("stop: %d %q %q", code, h.stdout, h.stderr) + } + + // The token and server can come from the environment. + t.Setenv(config.ServerEnvVar, socket) + t.Setenv(config.ServeTokenEnvVar, "tok") + h = newHarness(t) + if code := h.run("stop", "fixer"); code != ExitOK { + t.Fatalf("env stop: %d %s", code, h.stderr) + } + if !strings.Contains(f.seen(), "POST /v1/agents/fixer/stop") { + t.Fatalf("requests:\n%s", f.seen()) + } +} + +func TestRemoteRunFollowsAndStops(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + f := &fakeAPI{hold: make(chan struct{})} + t.Cleanup(func() { close(f.hold) }) + socket := startFakeAPI(t, f) + def := writeFile(t, filepath.Join(t.TempDir(), "scout.yaml"), portableDefinition) + + h := newHarness(t) + out := &lockedBuffer{} + h.env.Stdout = out + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan int, 1) + go func() { + done <- h.env.Run(ctx, append(h.workDirArgs(), "--server", socket, "run", def)) + }() + waitUntil(t, "first chunk printed", func() bool { return strings.Contains(out.String(), "ready\n") }) + cancel() + select { + case code := <-done: + if code != ExitOK { + t.Fatalf("code %d stderr %q", code, h.stderr) + } + case <-time.After(5 * time.Second): + t.Fatal("remote run did not stop") + } + if out.String() != "started scout-agent\nboot\nready\nstopped scout-agent\n" { + t.Fatalf("stdout %q", out.String()) + } + if f.run.Name != "scout-agent" || !strings.Contains(string(f.run.Definition), `"role":"scout"`) { + t.Fatalf("run request %+v %s", f.run, f.run.Definition) + } + if !strings.Contains(f.seen(), "POST /v1/agents/scout-agent/stop") { + t.Fatalf("requests:\n%s", f.seen()) + } + if len(h.rt.Calls()) != 0 { + t.Fatalf("remote run touched the local runtime: %v", h.rt.Calls()) + } + + // A bare name is sent without a definition; -detach returns after start. + h = newHarness(t) + if code := h.run("--server", socket, "run", "-detach", "fixer"); code != ExitOK || h.stdout.String() != "started fixer\n" { + t.Fatalf("detach: %d %q %q", code, h.stdout, h.stderr) + } + if f.run.Name != "fixer" || len(f.run.Definition) != 0 { + t.Fatalf("bare run request %+v", f.run) + } + + // The server losing the agent ends the follow with a message, then stops. + h = newHarness(t) + if code := h.run("--server", socket, "run", "lost"); code != ExitOK || !strings.Contains(h.stderr.String(), "agent lost not found") || !strings.Contains(h.stdout.String(), "stopped lost") { + t.Fatalf("lost: %d %q %q", code, h.stdout, h.stderr) + } + + // logs -f follows the same stream. + h = newHarness(t) + if code := h.run("--server", socket, "logs", "-f", "lost"); code != ExitError || h.stdout.String() != "boot\nready\n" { + t.Fatalf("logs -f: %d %q %q", code, h.stdout, h.stderr) + } +} + +func TestRemoteErrors(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + f := &fakeAPI{token: "tok"} + socket := startFakeAPI(t, f) + + cases := []struct { + args []string + code int + stderr string + }{ + {[]string{"--server", "", "ps"}, ExitOK, ""}, // empty --server is not remote mode + {[]string{"--server", "::bad::", "ps"}, ExitUsage, "--server"}, + {[]string{"--server", socket, "ps"}, ExitError, "bearer token"}, + {[]string{"--server", socket, "--token", "tok", "serve", "-h"}, ExitOK, ""}, // serve always runs locally + {[]string{"--server", socket, "--token", "tok", "stop", "ghost"}, ExitError, "agent ghost not found"}, + {[]string{"--server", socket, "--token", "tok", "kick", "ghost", "hi"}, ExitError, "agent ghost not found"}, + {[]string{"--server", socket, "--token", "tok", "logs", "ghost"}, ExitError, "agent ghost not found"}, + {[]string{"--server", socket, "--token", "tok", "run", "ghost"}, ExitError, "agent ghost not found"}, + {[]string{"--server", socket, "--token", "tok", "run", filepath.Join(t.TempDir(), "missing.yaml")}, ExitError, "reading definition"}, + {[]string{"--server", socket, "--token", "tok", "run"}, ExitUsage, "usage: crux --server"}, + {[]string{"--server", socket, "--token", "tok", "run", "-h"}, ExitOK, ""}, + {[]string{"--server", socket, "--token", "tok", "ps", "-h"}, ExitOK, ""}, + {[]string{"--server", socket, "--token", "tok", "logs"}, ExitUsage, "usage: crux --server"}, + {[]string{"--server", socket, "--token", "tok", "logs", "-h"}, ExitOK, ""}, + {[]string{"--server", socket, "--token", "tok", "logs", "-buffer", "-full", "fixer"}, ExitUsage, "exclusive"}, + {[]string{"--server", socket, "--token", "tok", "logs", "-f", "-json", "fixer"}, ExitUsage, "exclusive"}, + {[]string{"--server", socket, "--token", "tok", "kick", "fixer"}, ExitUsage, "usage: crux kick"}, + {[]string{"--server", socket, "--token", "tok", "kick", "-h"}, ExitOK, ""}, + {[]string{"--server", socket, "--token", "tok", "kick", "fixer", " "}, ExitUsage, "empty"}, + {[]string{"--server", socket, "--token", "tok", "stop"}, ExitUsage, "usage: crux stop"}, + {[]string{"--server", socket, "--token", "tok", "stop", "-h"}, ExitOK, ""}, + {[]string{"--server", socket, "--token", "tok", "help"}, ExitOK, ""}, + {[]string{"--server", socket, "--token", "tok", "bogus"}, ExitUsage, "not available with --server"}, + } + for _, c := range cases { + h := newHarness(t) + code := h.run(c.args...) + if code != c.code || !strings.Contains(h.stderr.String(), c.stderr) { + t.Errorf("%v: code %d (want %d) stderr %q (want %q)", c.args, code, c.code, h.stderr, c.stderr) + } + } + + // Stdin failure and a dead stop after a run follow. + h := newHarness(t) + h.env.Stdin = failingReader{} + if code := h.run("--server", socket, "--token", "tok", "kick", "fixer", "-"); code != ExitError || !strings.Contains(h.stderr.String(), "broken stdin") { + t.Fatalf("stdin: %d %q", code, h.stderr) + } +} + +// TestRemoteAgainstRealServe round-trips `crux --server` against a real +// `crux serve` on a socket: the two halves agree on the wire. +func TestRemoteAgainstRealServe(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + server := newHarness(t) + cfgPath := writeFile(t, filepath.Join(t.TempDir(), "hive.yaml"), "agents: {fixer: {}, scout: {}}\n") + socket := filepath.Join(shortSocketDir(t), "api.sock") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + server.env.Run(ctx, append(server.workDirArgs(), "--config", cfgPath, "serve", "--socket", socket, "--token", "tok")) + }() + waitForSocket(t, socket) + + client := newHarness(t) + if code := client.run("--server", "unix://"+socket, "--token", "tok", "ps", "--json"); code != ExitOK { + t.Fatalf("ps: %d %s", code, client.stderr) + } + var rows []processRow + if err := json.Unmarshal(client.stdout.Bytes(), &rows); err != nil || len(rows) != 2 || rows[0].Name != "fixer" { + t.Fatalf("rows %v %s", err, client.stdout) + } + client = newHarness(t) + if code := client.run("--server", socket, "--token", "tok", "kick", "scout", "hello"); code != ExitOK || client.stdout.String() != "kicked scout\n" { + t.Fatalf("kick: %d %q %q", code, client.stdout, client.stderr) + } + if kicks := server.rt.Kicks(); len(kicks) != 1 || kicks[0] != "hello" { + t.Fatalf("server kicks %v", kicks) + } + client = newHarness(t) + if code := client.run("--server", socket, "ps"); code != ExitError || !strings.Contains(client.stderr.String(), "bearer token") { + t.Fatalf("no token: %d %q", code, client.stderr) + } +} diff --git a/src/pkg/crux/cli/runtime.go b/src/crux/cli/runtime.go similarity index 79% rename from src/pkg/crux/cli/runtime.go rename to src/crux/cli/runtime.go index 2339d70950..63aa05fe45 100644 --- a/src/pkg/crux/cli/runtime.go +++ b/src/crux/cli/runtime.go @@ -5,9 +5,9 @@ import ( "os" "path/filepath" - "github.com/hivecommons/hive/pkg/agent" - "github.com/hivecommons/hive/pkg/crux" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/config" ) // uidMapName is the per-work-dir UID map. cmd/hive uses the entrypoint's @@ -23,5 +23,5 @@ const uidMapName = "uid-map.json" func newManagerRuntime(defs map[string]config.AgentConfig, project crux.ProjectContext) crux.Runtime { logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) uidMap := filepath.Join(os.Getenv(envWorkDir), uidMapName) - return crux.FromManager(agent.NewManagerWithOptions(defs, logger, project, agent.WithUIDMapPath(uidMap))) + return agent.AsRuntime(agent.NewManagerWithOptions(defs, logger, project, agent.WithUIDMapPath(uidMap))) } diff --git a/src/crux/cli/serve.go b/src/crux/cli/serve.go new file mode 100644 index 0000000000..70ee837561 --- /dev/null +++ b/src/crux/cli/serve.go @@ -0,0 +1,86 @@ +package cli + +import ( + "context" + "fmt" + "log/slog" + "os" + "strings" + + "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/serve" +) + +// cmdServe is `crux serve`: build the runtime from --config and expose it +// over the Crux API (crux/api, served by crux/serve) until ctx ends. It is +// the process boundary of Stage 3: a harness other than hive drives agents +// through the socket or, with a token, over TCP; the other `crux` +// subcommands reach it with --server / CRUX_SERVER. +// +// The socket defaults to config.DefaultServeSocket (not the per-work-dir +// control socket `crux run` serves); --listen adds TCP and is refused +// without --token / CRUX_TOKEN, and refused off loopback (plain http would +// carry the token in the clear; remote access goes through a TLS proxy). +func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) int { + fs := e.subFlags("serve") + var cfg config.ServeConfig + fs.StringVar(&cfg.Socket, "socket", "", "unix socket to serve on (default $"+config.ServeSocketEnvVar+", then "+config.DefaultServeSocket()+")") + fs.StringVar(&cfg.Listen, "listen", "", "also serve on this loopback TCP host:port (plain http); requires a token") + fs.StringVar(&cfg.Token, "token", "", "bearer token every request must carry (default $"+config.ServeTokenEnvVar+")") + var hiveAddr string + fs.StringVar(&hiveAddr, "hive-callback", "", "hive capability endpoint to dial (default $"+callback.SocketEnvVar+"; none when unset)") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 0 { + fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR]") + return ExitUsage + } + if cfg.Token == "" { + cfg.Token = os.Getenv(config.ServeTokenEnvVar) + } + if err := cfg.Validate(); err != nil { + fmt.Fprintln(e.Stderr, "crux:", err) + return ExitUsage + } + if hiveAddr == "" { + hiveAddr = os.Getenv(callback.SocketEnvVar) + } + var hive *hiveCallback + if hiveAddr != "" { + var err error + if hive, err = dialHiveCallback(hiveAddr, os.Getenv(callback.TokenEnvVar)); err != nil { + fmt.Fprintln(e.Stderr, "crux:", err) + return ExitUsage + } + } + + defs, project, err := e.Loader.LoadConfig(opts.config) + if err != nil { + fmt.Fprintln(e.Stderr, "crux:", err) + return ExitError + } + rt := e.newRuntime(defs, project) + + srv := serve.New(rt) + srv.Token = cfg.Token + srv.Logger = slog.New(slog.NewTextHandler(e.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) + if hive != nil { + hive.install(ctx, rt, srv, e.Stdout) + hive.watchPolicy(ctx, rt, e.Stdout, e.Stderr) + } + + ls, err := serve.Listen(cfg) + if err != nil { + fmt.Fprintln(e.Stderr, "crux: serve:", err) + return ExitError + } + fmt.Fprintf(e.Stdout, "serving %d agent definition(s) on %s\n", len(defs), strings.Join(ls.Addrs(), " and ")) + if err := srv.Serve(ctx, ls); err != nil { + fmt.Fprintln(e.Stderr, "crux: serve:", err) + return ExitError + } + fmt.Fprintln(e.Stdout, "serve stopped") + return ExitOK +} diff --git a/src/crux/cli/serve_test.go b/src/crux/cli/serve_test.go new file mode 100644 index 0000000000..249cc1bcdd --- /dev/null +++ b/src/crux/cli/serve_test.go @@ -0,0 +1,158 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" +) + +// socketClient is an http.Client that dials the unix socket at path. +func socketClient(path string) *http.Client { + return &http.Client{Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", path) + }, + }} +} + +func TestServeExposesTheRuntimeOverTheAPI(t *testing.T) { + t.Setenv(config.ServeTokenEnvVar, "") + h := newHarness(t) + cfgPath := writeFile(t, filepath.Join(t.TempDir(), "hive.yaml"), "agents: {fixer: {}, scout: {}}\n") + socket := filepath.Join(shortSocketDir(t), "api.sock") + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan int, 1) + go func() { + done <- h.env.Run(ctx, append(h.workDirArgs(), "--config", cfgPath, "serve", "--socket", socket)) + }() + waitForSocket(t, socket) + + fi, err := os.Stat(socket) + if err != nil || fi.Mode().Perm() != config.ServeSocketMode { + t.Fatalf("socket stat %v mode %o type %v path %s", err, fi.Mode().Perm(), fi.Mode().Type(), socket) + } + client := socketClient(socket) + resp, err := client.Get("http://crux" + api.PathAgents) + if err != nil { + t.Fatal(err) + } + var list api.ProcessList + if err := json.NewDecoder(resp.Body).Decode(&list); err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if len(list.Processes) != 2 || list.Processes[0].Name != "fixer" || list.Processes[0].State != string(crux.StateStopped) { + t.Fatalf("list = %+v", list) + } + if calls := h.rt.Calls(); calls[0] != "SetKickObserver(true)" || calls[1] != "AllStatuses()" { + t.Fatalf("calls = %v", calls) + } + + cancel() + select { + case code := <-done: + if code != ExitOK { + t.Fatalf("serve exit %d stderr %q", code, h.stderr) + } + case <-time.After(5 * time.Second): + t.Fatal("serve did not stop") + } + if out := h.stdout.String(); !strings.Contains(out, "serving 2 agent definition(s) on unix://"+socket) || !strings.Contains(out, "serve stopped") { + t.Fatalf("stdout = %q", out) + } + if _, err := os.Stat(socket); !os.IsNotExist(err) { + t.Fatalf("socket should be removed on stop, stat err = %v", err) + } +} + +func TestServeTokenFromEnvironmentGatesTCP(t *testing.T) { + t.Setenv(config.ServeTokenEnvVar, "from-env") + h := newHarness(t) + socket := filepath.Join(shortSocketDir(t), "api.sock") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan int, 1) + go func() { + done <- h.env.Run(ctx, append(h.workDirArgs(), "serve", "--socket", socket, "--listen", "127.0.0.1:0")) + }() + waitForSocket(t, socket) + + // The token in the environment is required on the socket too. + resp, err := socketClient(socket).Get("http://crux" + api.PathHealth) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("no token: %d", resp.StatusCode) + } + req, _ := http.NewRequest(http.MethodGet, "http://crux"+api.PathHealth, nil) + req.Header.Set("Authorization", "Bearer from-env") + resp, err = socketClient(socket).Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("with token: %d", resp.StatusCode) + } + // The TCP address was bound and announced; the banner is printed before + // Serve starts, so it is complete once the socket answers. + if !strings.Contains(h.stdout.String(), "http://127.0.0.1:") { + t.Fatalf("stdout = %q", h.stdout.String()) + } + cancel() + <-done +} + +func TestServeUsageAndErrors(t *testing.T) { + t.Setenv(config.ServeTokenEnvVar, "") + h := newHarness(t) + socket := filepath.Join(shortSocketDir(t), "api.sock") + + if code := h.run("serve", "extra"); code != ExitUsage || !strings.Contains(h.stderr.String(), "usage: crux serve") { + t.Fatalf("extra arg: %d %q", code, h.stderr) + } + h.stderr.Reset() + if code := h.run("serve", "--socket", socket, "--listen", "127.0.0.1:0"); code != ExitUsage || !strings.Contains(h.stderr.String(), "bearer token") { + t.Fatalf("TCP without token: %d %q", code, h.stderr) + } + h.stderr.Reset() + if code := h.run("serve", "-bogus"); code != ExitUsage { + t.Fatalf("bad flag: %d", code) + } + h.stderr.Reset() + if code := h.run("serve", "-h"); code != ExitOK { + t.Fatalf("-h: %d", code) + } + h.stderr.Reset() + if code := h.run("--config", filepath.Join(t.TempDir(), "missing.yaml"), "serve", "--socket", socket); code != ExitError { + t.Fatalf("missing config: %d %q", code, h.stderr) + } + h.stderr.Reset() + // A socket path that cannot be bound: its parent is a regular file. + writeFile(t, socket, "") + if code := h.run("serve", "--socket", filepath.Join(socket, "under-a-file.sock")); code != ExitError || !strings.Contains(h.stderr.String(), "serve:") { + t.Fatalf("bad socket: %d %q", code, h.stderr) + } + + // serve is refused over the control socket, like run. + served := &Env{Stdout: &bytes.Buffer{}, Stderr: &bytes.Buffer{}} + served.defaults() + if code := served.dispatch(context.Background(), globalOptions{}, "serve", nil, true); code != ExitUsage || !strings.Contains(served.Stderr.(*bytes.Buffer).String(), "not available over the control socket") { + t.Fatalf("remote serve: %d %q", code, served.Stderr) + } +} diff --git a/src/crux/cli/verbs.go b/src/crux/cli/verbs.go new file mode 100644 index 0000000000..2afa2814da --- /dev/null +++ b/src/crux/cli/verbs.go @@ -0,0 +1,17 @@ +package cli + +// RuntimeVerbs maps each crux subcommand that drives an agent runtime to the +// crux.Runtime methods it calls, in call order. It is the CLI's half of the +// parity contract in hivecommons/hive#14138: TestVerbsHaveRuntimeMethods +// asserts every method named here is on crux.Runtime and every verb dispatch +// routes to a runtime is listed here, so a verb cannot grow a dependency the +// seam does not expose (and, in stage 3, that the API does not route). +// `serve` is absent on purpose: it exposes the runtime rather than driving +// it, and its routes are covered by crux/serve. +var RuntimeVerbs = map[string][]string{ + "run": {"AddAgent", "Start", "GetBufferOutput", "Stop"}, + "ps": {"AllStatuses"}, + "logs": {"ResolveAgent", "GetOutput", "GetBufferOutput", "CaptureFullLog", "FullLogHeader"}, + "kick": {"ResolveAgent", "SendKick"}, + "stop": {"ResolveAgent", "Stop"}, +} diff --git a/src/crux/cli/verbs_test.go b/src/crux/cli/verbs_test.go new file mode 100644 index 0000000000..0d7c784c1b --- /dev/null +++ b/src/crux/cli/verbs_test.go @@ -0,0 +1,163 @@ +package cli + +import ( + "go/ast" + "go/parser" + "go/token" + "reflect" + "sort" + "strconv" + "strings" + "testing" + + "github.com/hivecommons/hive/crux" +) + +// nonRuntimeVerbs are dispatch cases that do not operate on a runtime: serve +// exposes one, help prints usage. +var nonRuntimeVerbs = map[string]bool{"serve": true, "help": true, "-h": true, "--help": true} + +func TestVerbsHaveRuntimeMethods(t *testing.T) { + rt := reflect.TypeOf((*crux.Runtime)(nil)).Elem() + for verb, methods := range RuntimeVerbs { + if len(methods) == 0 { + t.Errorf("verb %q lists no runtime methods", verb) + } + for _, m := range methods { + if _, ok := rt.MethodByName(m); !ok { + t.Errorf("verb %q calls %s, which is not a crux.Runtime method", verb, m) + } + } + } +} + +func TestEveryDispatchedVerbIsListed(t *testing.T) { + verbs := dispatchVerbs(t) + for _, v := range verbs { + if nonRuntimeVerbs[v] { + continue + } + if _, ok := RuntimeVerbs[v]; !ok { + t.Errorf("dispatch handles %q but RuntimeVerbs does not list it", v) + } + } + seen := map[string]bool{} + for _, v := range verbs { + seen[v] = true + } + for v := range RuntimeVerbs { + if !seen[v] { + t.Errorf("RuntimeVerbs lists %q but dispatch does not handle it", v) + } + } +} + +// TestVerbsMatchCommandBodies checks the table against the code: the rt. +// selectors in each cmd function are exactly the methods listed. +func TestVerbsMatchCommandBodies(t *testing.T) { + fset := token.NewFileSet() + // Every (*Env) method body, so a cmd that delegates to a helper + // (run -> stream) is credited with the helper's runtime calls. + bodies := map[string]*ast.BlockStmt{} + for _, file := range []string{"commands.go", "control.go"} { + f, err := parser.ParseFile(fset, file, nil, 0) + if err != nil { + t.Fatal(err) + } + for _, decl := range f.Decls { + if fn, ok := decl.(*ast.FuncDecl); ok && fn.Recv != nil && fn.Body != nil { + bodies[fn.Name.Name] = fn.Body + } + } + } + var collect func(body *ast.BlockStmt, into map[string]bool, seen map[string]bool) + collect = func(body *ast.BlockStmt, into map[string]bool, seen map[string]bool) { + ast.Inspect(body, func(n ast.Node) bool { + sel, ok := n.(*ast.SelectorExpr) + if !ok { + return true + } + id, ok := sel.X.(*ast.Ident) + if !ok { + return true + } + switch { + case id.Name == "rt": + into[sel.Sel.Name] = true + case id.Name == "e" && bodies[sel.Sel.Name] != nil && !seen[sel.Sel.Name]: + seen[sel.Sel.Name] = true + collect(bodies[sel.Sel.Name], into, seen) + } + return true + }) + } + calls := map[string]map[string]bool{} + for name, body := range bodies { + if !strings.HasPrefix(name, "cmd") { + continue + } + verb := strings.ToLower(strings.TrimPrefix(name, "cmd")) + calls[verb] = map[string]bool{} + collect(body, calls[verb], map[string]bool{name: true}) + } + for verb, methods := range RuntimeVerbs { + got := sortedKeys(calls[verb]) + want := append([]string(nil), methods...) + sort.Strings(want) + if !reflect.DeepEqual(got, want) { + t.Errorf("verb %q: code calls rt.%v, RuntimeVerbs lists %v", verb, got, want) + } + } +} + +func sortedKeys(m map[string]bool) []string { + // Methods the Process slice helper exposes are not runtime methods; a + // local named rt of another type would show up here and should be + // renamed rather than filtered. + var out []string + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// dispatchVerbs reads the string literals of the case clauses in +// (*Env).dispatch, so the test tracks the real switch rather than a copy. +func dispatchVerbs(t *testing.T) []string { + t.Helper() + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "cli.go", nil, 0) + if err != nil { + t.Fatal(err) + } + var verbs []string + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Name.Name != "dispatch" || fn.Body == nil { + continue + } + ast.Inspect(fn.Body, func(n ast.Node) bool { + cc, ok := n.(*ast.CaseClause) + if !ok { + return true + } + for _, e := range cc.List { + lit, ok := e.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + continue + } + v, err := strconv.Unquote(lit.Value) + if err != nil { + t.Fatal(err) + } + verbs = append(verbs, v) + } + return false + }) + } + if len(verbs) == 0 { + t.Fatal("found no case clauses in (*Env).dispatch") + } + return verbs +} diff --git a/src/crux/cmd/crux/main.go b/src/crux/cmd/crux/main.go new file mode 100644 index 0000000000..2d58ab1c46 --- /dev/null +++ b/src/crux/cmd/crux/main.go @@ -0,0 +1,39 @@ +// Command crux is the Crux agent runtime's own binary (hivecommons/hive#14138): +// run, observe and control agents through crux.Runtime (`crux run/ps/logs/ +// kick/stop`), serve that runtime over the Crux API (`crux serve`) and, with +// --server or CRUX_SERVER, drive such a server instead of an in-process +// runtime. All behaviour lives in crux/cli; this package is the process +// boundary: signals, exit code, the version string and the module's own +// file-based definition loader (crux/defs, selected with --defs / CRUX_DEFS). +// +// It builds from the nested module alone: +// +// cd src/crux && GOWORK=off go build ./cmd/crux +// +// The root module's cmd/crux is the same host wired to Hive's hive.yaml +// loader instead; the verbs and flags are identical. +package main + +import ( + "context" + "io" + "os" + + "github.com/hivecommons/hive/crux/cli" + "github.com/hivecommons/hive/crux/defs" +) + +// version is what --version reports. Release builds set it with +// -ldflags "-X main.version="; a plain `go build` reports cli.DefaultVersion. +var version = cli.DefaultVersion + +func main() { + os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr, cli.SignalContext)) +} + +func run(args []string, stdin io.Reader, stdout, stderr io.Writer, ctxFn func() (context.Context, context.CancelFunc)) int { + ctx, stop := ctxFn() + defer stop() + env := &cli.Env{Stdin: stdin, Stdout: stdout, Stderr: stderr, Loader: defs.Loader{}, Version: version} + return env.Run(ctx, args) +} diff --git a/src/crux/cmd/crux/main_test.go b/src/crux/cmd/crux/main_test.go new file mode 100644 index 0000000000..709a3ee79b --- /dev/null +++ b/src/crux/cmd/crux/main_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/cli" + "github.com/hivecommons/hive/crux/defs" +) + +// The module binary's loader must satisfy the CLI's seam without the CLI +// importing it. +var _ cli.DefinitionLoader = defs.Loader{} + +func testCtx() (context.Context, context.CancelFunc) { + return context.WithCancel(context.Background()) +} + +func TestRunPrintsUsageWithoutACommand(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run([]string{"--workdir", t.TempDir()}, strings.NewReader(""), &stdout, &stderr, testCtx) + if code != cli.ExitUsage { + t.Fatalf("exit code = %d, want %d", code, cli.ExitUsage) + } + if !strings.Contains(stderr.String(), "usage: crux") { + t.Fatalf("stderr = %q, want usage", stderr.String()) + } +} + +func TestRunRejectsUnknownCommand(t *testing.T) { + var stderr bytes.Buffer + code := run([]string{"--workdir", t.TempDir(), "frobnicate"}, strings.NewReader(""), &bytes.Buffer{}, &stderr, testCtx) + if code != cli.ExitUsage { + t.Fatalf("exit code = %d, want %d", code, cli.ExitUsage) + } + if !strings.Contains(stderr.String(), "unknown command") { + t.Fatalf("stderr = %q", stderr.String()) + } +} + +func TestRunReportsVersion(t *testing.T) { + var stdout bytes.Buffer + code := run([]string{"--version"}, strings.NewReader(""), &stdout, &bytes.Buffer{}, testCtx) + if code != cli.ExitOK || stdout.String() != "crux "+cli.DefaultVersion+"\n" { + t.Fatalf("code %d, stdout %q", code, stdout.String()) + } +} + +func TestRunPSFromDefsDirectory(t *testing.T) { + dir := t.TempDir() + defsDir := filepath.Join(dir, "defs") + writeFile(t, filepath.Join(defsDir, "fixer.yaml"), "backend: codex\nmodel: gpt-5\n") + var stdout, stderr bytes.Buffer + code := run([]string{"--workdir", filepath.Join(dir, "work"), "--local", "--defs", defsDir, "ps"}, strings.NewReader(""), &stdout, &stderr, testCtx) + if code != cli.ExitOK { + t.Fatalf("exit code = %d, stderr %q", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "fixer") { + t.Fatalf("stdout = %q, want the fixer row", stdout.String()) + } + + stdout.Reset() + stderr.Reset() + t.Setenv(cli.DefsEnvVar, filepath.Join(defsDir, "fixer.yaml")) + code = run([]string{"--workdir", filepath.Join(dir, "work"), "--local", "ps"}, strings.NewReader(""), &stdout, &stderr, testCtx) + if code != cli.ExitOK || !strings.Contains(stdout.String(), "fixer") { + t.Fatalf("%s: code %d, stdout %q, stderr %q", cli.DefsEnvVar, code, stdout.String(), stderr.String()) + } +} + +func TestRunReportsLoaderErrors(t *testing.T) { + var stderr bytes.Buffer + code := run([]string{"--workdir", t.TempDir(), "--local", "--defs", filepath.Join(t.TempDir(), "missing"), "ps"}, strings.NewReader(""), &bytes.Buffer{}, &stderr, testCtx) + if code != cli.ExitError || !strings.Contains(stderr.String(), "reading definitions") { + t.Fatalf("code %d, stderr %q", code, stderr.String()) + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/src/crux/conditions.go b/src/crux/conditions.go new file mode 100644 index 0000000000..3ee1942289 --- /dev/null +++ b/src/crux/conditions.go @@ -0,0 +1,47 @@ +package crux + +import "errors" + +// ErrRuntimeUnavailable marks a Runtime call that could not reach the +// runtime at all (a sidecar whose `crux serve` is gone), as opposed to one +// the runtime answered with an error. The in-process runtime never returns +// it. A kick that fails with it was REFUSED, not queued: the host's governor +// re-evaluates on its next tick and the kick is re-derived from current +// state rather than replayed stale (hivecommons/hive#14138 stage 4). +var ErrRuntimeUnavailable = errors.New("crux runtime unavailable") + +// ErrGitHubAppNotInstalled means a token mint was asked for while the GitHub +// App has no installation ID (installation_id 0). It is a static operator +// configuration condition, not a transient failure; pkg/github aliases it as +// ErrAppNotInstalled and the runtime classifies it separately. +var ErrGitHubAppNotInstalled = errors.New("github app has no installation for this hive's org") + +// ConditionReporter receives the runtime's self-reported error conditions: +// a (class, key) pair observed with a message and a scrubbed diagnostics +// excerpt, and cleared when the condition recovers. The runtime opts an +// error site in by naming one of the Class* constants below; what counts as +// reportable (streaks, windows, persistence) and where a report goes is the +// host's business. Hive wires pkg/fleetreport's runtime tracker through +// Manager.SetConditionReporter; a nil reporter drops observations. +type ConditionReporter interface { + Observe(class, key, message, diagnostics string) + Clear(class, key string) +} + +// Runtime error classes the agent runtime and its watchdog emit. Each is an +// explicit allow-list entry on the host side (pkg/fleetreport.Classes); the +// values are the wire names and must not change. +const ( + ClassRestartStorm = "restart_storm" + ClassKickFailedRepeatedly = "kick_failed_repeatedly" + ClassDataDiskPressure = "data_disk_pressure" + ClassGitHubAppTokenMintFailed = "github_app_token_mint_failed" + ClassGitHubAppNotInstalled = "github_app_not_installed" + ClassAgentStall = "agent_stall" + ClassAgentCrashLoop = "agent_crash_loop" + ClassCIPollNudge = "ci_poll_nudge" + // ClassRuntimeUnavailable is raised by the HOST, not the runtime: a + // sidecar host whose runtime API stopped answering (hivecommons/hive#14138 + // stage 4). The runtime cannot report its own absence. + ClassRuntimeUnavailable = "crux_runtime_unavailable" +) diff --git a/src/crux/conditions_test.go b/src/crux/conditions_test.go new file mode 100644 index 0000000000..d736352246 --- /dev/null +++ b/src/crux/conditions_test.go @@ -0,0 +1,40 @@ +package crux + +import "testing" + +type recordingReporter struct{ observed, cleared []string } + +func (r *recordingReporter) Observe(class, key, message, diagnostics string) { + r.observed = append(r.observed, class+"/"+key+": "+message+" ["+diagnostics+"]") +} +func (r *recordingReporter) Clear(class, key string) { r.cleared = append(r.cleared, class+"/"+key) } + +func TestConditionReporterContract(t *testing.T) { + var rep ConditionReporter = &recordingReporter{} + rep.Observe(ClassAgentStall, "scanner", "stalled", "diag") + rep.Clear(ClassAgentStall, "scanner") + r := rep.(*recordingReporter) + if len(r.observed) != 1 || r.observed[0] != "agent_stall/scanner: stalled [diag]" { + t.Fatalf("observed = %q", r.observed) + } + if len(r.cleared) != 1 || r.cleared[0] != "agent_stall/scanner" { + t.Fatalf("cleared = %q", r.cleared) + } +} + +func TestClassWireNamesAreStable(t *testing.T) { + want := map[string]string{ + ClassRestartStorm: "restart_storm", + ClassKickFailedRepeatedly: "kick_failed_repeatedly", + ClassDataDiskPressure: "data_disk_pressure", + ClassGitHubAppTokenMintFailed: "github_app_token_mint_failed", + ClassAgentStall: "agent_stall", + ClassAgentCrashLoop: "agent_crash_loop", + ClassCIPollNudge: "ci_poll_nudge", + } + for got, exp := range want { + if got != exp { + t.Errorf("class %q != %q", got, exp) + } + } +} diff --git a/src/pkg/crux/config/advisor_override.go b/src/crux/config/advisor_override.go similarity index 100% rename from src/pkg/crux/config/advisor_override.go rename to src/crux/config/advisor_override.go diff --git a/src/crux/config/advisor_roles.go b/src/crux/config/advisor_roles.go new file mode 100644 index 0000000000..5072dbb6cb --- /dev/null +++ b/src/crux/config/advisor_roles.go @@ -0,0 +1,61 @@ +package config + +import "strings" + +// ModelRole is one `model_roles` entry: a named model (and optionally the +// backend and reasoning effort it runs with) that agents and the advisor lane +// reference by `@name` instead of a literal model id. It lives in crux/config +// so the agent runtime can project roles into launch files (the OMP roles +// overlay) without importing the hub's config package; pkg/config aliases it. +type ModelRole struct { + // Backend the role's model runs on. Optional; when empty the consumer's + // own backend applies. + Backend string `yaml:"backend,omitempty" json:"backend,omitempty"` + // Model is the model id. Required. + Model string `yaml:"model" json:"model"` + // ReasoningEffort for backends that expose an effort control. Optional. + // Not validated when Backend is empty (the consumer's backend applies at + // launch); for omp it must be one of OMPThinkingLevels; otherwise it is + // validated against ReasoningEffortsByBackend. + ReasoningEffort string `yaml:"reasoning_effort,omitempty" json:"reasoning_effort,omitempty"` +} + +// AdvisorClaudeBackend is the backend of the Claude Code `Stop` hook adapter +// (phase 1). AdvisorCopilotBackend and AdvisorCodexBackend are the Copilot +// CLI `agentStop` and Codex CLI `Stop` adapters (phase 3, #9724). +const ( + AdvisorClaudeBackend = "claude" + AdvisorCopilotBackend = "copilot" + AdvisorCodexBackend = "codex" +) + +// AdvisorOMPBackend is the backend of the phase-2 adapter (the OMP +// `session_stop` extension and `--config` roles overlay). +const AdvisorOMPBackend = "omp" + +// NousEngineEnv is the name the hive exports an experimenter agent's +// resolved engine under. +const NousEngineEnv = "HIVE_NOUS_ENGINE" + +// AdvisorSupportedBackendList names the supported backends for operator-facing +// messages. +const AdvisorSupportedBackendList = AdvisorClaudeBackend + ", " + AdvisorOMPBackend + ", " + AdvisorCopilotBackend + ", " + AdvisorCodexBackend + +// AdvisorSupportedBackend reports whether the advisor lane has an adapter for +// the backend: the Claude Code, Copilot CLI and Codex CLI turn-end hooks and +// the OMP `session_stop` extension. +func AdvisorSupportedBackend(backend string) bool { + switch strings.ToLower(strings.TrimSpace(backend)) { + case AdvisorClaudeBackend, AdvisorOMPBackend, AdvisorCopilotBackend, AdvisorCodexBackend: + return true + } + return false +} + +// NousDispatcherSDK is the in-process Nous dispatcher; NousDispatcherEnv +// overrides nous.dispatcher at runtime and is also the name the hive exports +// the resolved dispatcher under to experimenter agents. +const ( + NousDispatcherSDK = "sdk" + NousDispatcherEnv = "HIVE_NOUS_DISPATCHER" +) diff --git a/src/pkg/crux/config/agent_config.go b/src/crux/config/agent_config.go similarity index 100% rename from src/pkg/crux/config/agent_config.go rename to src/crux/config/agent_config.go diff --git a/src/pkg/crux/config/agent_repos.go b/src/crux/config/agent_repos.go similarity index 100% rename from src/pkg/crux/config/agent_repos.go rename to src/crux/config/agent_repos.go diff --git a/src/pkg/crux/config/agent_runtime.go b/src/crux/config/agent_runtime.go similarity index 99% rename from src/pkg/crux/config/agent_runtime.go rename to src/crux/config/agent_runtime.go index 8f5f0eb3d1..51ec694281 100644 --- a/src/pkg/crux/config/agent_runtime.go +++ b/src/crux/config/agent_runtime.go @@ -1291,7 +1291,7 @@ func IsInferenceBackend(backend string) bool { // a backend here without also updating the shell side (or, if it genuinely // belongs on only one side, documenting why in cliBackendExceptions) fails // that test. -var CLIBackends = []string{"claude", "copilot", "goose", "codex", "pi", "bob", "aider", "gemini", "agy", "opencode", "kilo", "muse", "omp", "openhands"} +var CLIBackends = []string{"claude", "copilot", "goose", "codex", "pi", "bob", "aider", "gemini", "agy", "opencode", "kilo", "muse", "omp", "openhands", "omnigent"} // IsCLIBackend returns true if the backend launches an agentic CLI binary. func IsCLIBackend(backend string) bool { diff --git a/src/pkg/crux/config/backends.go b/src/crux/config/backends.go similarity index 100% rename from src/pkg/crux/config/backends.go rename to src/crux/config/backends.go diff --git a/src/pkg/crux/config/bob_test.go b/src/crux/config/bob_test.go similarity index 100% rename from src/pkg/crux/config/bob_test.go rename to src/crux/config/bob_test.go diff --git a/src/pkg/crux/config/doc.go b/src/crux/config/doc.go similarity index 100% rename from src/pkg/crux/config/doc.go rename to src/crux/config/doc.go diff --git a/src/pkg/crux/config/experimenter.go b/src/crux/config/experimenter.go similarity index 100% rename from src/pkg/crux/config/experimenter.go rename to src/crux/config/experimenter.go diff --git a/src/pkg/crux/config/gateways.go b/src/crux/config/gateways.go similarity index 100% rename from src/pkg/crux/config/gateways.go rename to src/crux/config/gateways.go diff --git a/src/pkg/crux/config/gateways_test.go b/src/crux/config/gateways_test.go similarity index 100% rename from src/pkg/crux/config/gateways_test.go rename to src/crux/config/gateways_test.go diff --git a/src/pkg/crux/config/kick_limits.go b/src/crux/config/kick_limits.go similarity index 100% rename from src/pkg/crux/config/kick_limits.go rename to src/crux/config/kick_limits.go diff --git a/src/pkg/crux/config/kick_log_source.go b/src/crux/config/kick_log_source.go similarity index 100% rename from src/pkg/crux/config/kick_log_source.go rename to src/crux/config/kick_log_source.go diff --git a/src/pkg/crux/config/leaf_test.go b/src/crux/config/leaf_test.go similarity index 100% rename from src/pkg/crux/config/leaf_test.go rename to src/crux/config/leaf_test.go diff --git a/src/pkg/crux/config/litellm_route.go b/src/crux/config/litellm_route.go similarity index 100% rename from src/pkg/crux/config/litellm_route.go rename to src/crux/config/litellm_route.go diff --git a/src/pkg/crux/config/litellm_route_test.go b/src/crux/config/litellm_route_test.go similarity index 100% rename from src/pkg/crux/config/litellm_route_test.go rename to src/crux/config/litellm_route_test.go diff --git a/src/crux/config/mode.go b/src/crux/config/mode.go new file mode 100644 index 0000000000..ca10d831e5 --- /dev/null +++ b/src/crux/config/mode.go @@ -0,0 +1,147 @@ +package config + +import ( + "fmt" + "os" + "strings" +) + +// How a host reaches the runtime (hivecommons/hive#14138, sidecar plan stage +// 1). Like ServeConfig this is a process-level setting, not a hive.yaml key: +// it is chosen by whoever composes the process (cmd/hive, a harness), so it +// is not reachable from Config and the settings-parity walk never sees it. +// The hive.yaml key lands with the supervision story (stage 4), which is +// when an operator first has something to configure about the sidecar. + +// Mode selects the Runtime implementation crux/runtime.New returns. +type Mode string + +const ( + // ModeInProcess links crux/agent into the host and constructs the + // manager directly. The default. + ModeInProcess Mode = "inprocess" + // ModeSidecar drives a `crux serve` process at RuntimeConfig.Server over + // the Crux API through crux/api/client (stage 3 of hivecommons/hive#14138). + ModeSidecar Mode = "sidecar" + + // ModeEnvVar names the environment variable RuntimeConfigFromEnv reads + // the mode from. Empty or unset means ModeInProcess. + ModeEnvVar = "CRUX_MODE" +) + +// DefaultMode is the mode an empty RuntimeConfig resolves to. +const DefaultMode = ModeInProcess + +// Supervision says who keeps the sidecar's `crux serve` process alive +// (hivecommons/hive#14138 stage 4). +type Supervision string + +const ( + // SuperviseChild: the host spawns `crux serve` itself, waits for its + // health route, and restarts it with backoff when it exits. The + // default for ModeSidecar. + SuperviseChild Supervision = "child" + // SuperviseExternal: something else (systemd, a pod sidecar container) + // runs `crux serve`; the host only reconnects. + SuperviseExternal Supervision = "external" + + // SuperviseEnvVar selects the Supervision; empty means SuperviseChild. + SuperviseEnvVar = "CRUX_SUPERVISE" + // BinaryEnvVar names the `crux` executable SuperviseChild spawns; + // empty means DefaultBinary looked up on PATH. + BinaryEnvVar = "CRUX_BINARY" + // DefaultBinary is the executable name SuperviseChild looks up. + DefaultBinary = "crux" +) + +// ParseSupervision normalises s to a Supervision; empty is SuperviseChild. +func ParseSupervision(s string) (Supervision, error) { + switch v := Supervision(strings.ToLower(strings.TrimSpace(s))); v { + case "": + return SuperviseChild, nil + case SuperviseChild, SuperviseExternal: + return v, nil + default: + return "", fmt.Errorf("unknown crux supervision %q (want %s or %s)", s, SuperviseChild, SuperviseExternal) + } +} + +// Modes lists the recognised modes, in documentation order. +func Modes() []Mode { return []Mode{ModeInProcess, ModeSidecar} } + +// ParseMode normalises s (case-insensitive, surrounding space ignored) to a +// Mode. Empty is DefaultMode; anything else unrecognised is an error naming +// the accepted values. +func ParseMode(s string) (Mode, error) { + switch m := Mode(strings.ToLower(strings.TrimSpace(s))); m { + case "": + return DefaultMode, nil + case ModeInProcess, ModeSidecar: + return m, nil + default: + return "", fmt.Errorf("unknown crux runtime mode %q (want %s)", s, modeList()) + } +} + +func modeList() string { + names := make([]string, 0, len(Modes())) + for _, m := range Modes() { + names = append(names, string(m)) + } + return strings.Join(names, " or ") +} + +// RuntimeConfig is what crux/runtime.New needs to pick and reach a runtime. +// The zero value is the in-process default. +type RuntimeConfig struct { + // Mode selects the implementation. Empty means DefaultMode. + Mode Mode + // Server is the `crux serve` address for ModeSidecar, in the forms + // api.ParseAddress accepts. Ignored in-process. Empty means the default + // socket (DefaultServeSocket). + Server string + // Token is the bearer token for Server. Ignored in-process. + Token string + // Supervise says whether the host spawns `crux serve` (SuperviseChild, + // the default) or expects it run externally. Ignored in-process. + Supervise Supervision + // Binary is the `crux` executable SuperviseChild spawns; empty means + // DefaultBinary on PATH. Ignored in-process and under SuperviseExternal. + Binary string +} + +// Validate applies the defaults and rejects an unknown mode. It is the one +// place the mode string is interpreted, so a typo fails at construction +// rather than silently running in-process. +func (c *RuntimeConfig) Validate() error { + m, err := ParseMode(string(c.Mode)) + if err != nil { + return err + } + c.Mode = m + if c.Mode == ModeSidecar && c.Server == "" { + c.Server = DefaultServeSocket() + } + sv, err := ParseSupervision(string(c.Supervise)) + if err != nil { + return err + } + c.Supervise = sv + if c.Binary == "" { + c.Binary = DefaultBinary + } + return nil +} + +// RuntimeConfigFromEnv reads ModeEnvVar, ServerEnvVar, ServeTokenEnvVar, +// SuperviseEnvVar and BinaryEnvVar. It does not validate; callers pass the +// result to crux/runtime.New, which does. +func RuntimeConfigFromEnv() RuntimeConfig { + return RuntimeConfig{ + Mode: Mode(os.Getenv(ModeEnvVar)), + Server: os.Getenv(ServerEnvVar), + Token: os.Getenv(ServeTokenEnvVar), + Supervise: Supervision(os.Getenv(SuperviseEnvVar)), + Binary: os.Getenv(BinaryEnvVar), + } +} diff --git a/src/crux/config/mode_test.go b/src/crux/config/mode_test.go new file mode 100644 index 0000000000..679ea8c86f --- /dev/null +++ b/src/crux/config/mode_test.go @@ -0,0 +1,134 @@ +package config + +import ( + "strings" + "testing" +) + +func TestParseMode(t *testing.T) { + cases := []struct { + in string + want Mode + err bool + }{ + {"", ModeInProcess, false}, + {"inprocess", ModeInProcess, false}, + {" InProcess ", ModeInProcess, false}, + {"sidecar", ModeSidecar, false}, + {"SIDECAR", ModeSidecar, false}, + {"remote", "", true}, + } + for _, tc := range cases { + got, err := ParseMode(tc.in) + if tc.err { + if err == nil { + t.Errorf("ParseMode(%q) = %q, want error", tc.in, got) + } else if !strings.Contains(err.Error(), "inprocess or sidecar") { + t.Errorf("ParseMode(%q) error %q should name the accepted modes", tc.in, err) + } + continue + } + if err != nil || got != tc.want { + t.Errorf("ParseMode(%q) = %q, %v; want %q", tc.in, got, err, tc.want) + } + } +} + +func TestModesListsEveryConstant(t *testing.T) { + want := map[Mode]bool{ModeInProcess: true, ModeSidecar: true} + for _, m := range Modes() { + delete(want, m) + } + if len(want) != 0 { + t.Fatalf("Modes() omits %v", want) + } + if DefaultMode != ModeInProcess { + t.Fatalf("DefaultMode = %q, want inprocess until the sidecar flips (#14138 stage 6)", DefaultMode) + } +} + +func TestRuntimeConfigValidate(t *testing.T) { + var zero RuntimeConfig + if err := zero.Validate(); err != nil || zero.Mode != ModeInProcess { + t.Fatalf("zero RuntimeConfig: %+v, %v; want inprocess", zero, err) + } + if zero.Server != "" { + t.Fatalf("in-process Validate must not fill Server, got %q", zero.Server) + } + + t.Setenv(ServeSocketEnvVar, "/run/test/crux.sock") + side := RuntimeConfig{Mode: " Sidecar "} + if err := side.Validate(); err != nil { + t.Fatal(err) + } + if side.Mode != ModeSidecar || side.Server != "/run/test/crux.sock" { + t.Fatalf("sidecar Validate = %+v; want normalised mode and default socket", side) + } + + explicit := RuntimeConfig{Mode: ModeSidecar, Server: "127.0.0.1:9", Token: "t"} + if err := explicit.Validate(); err != nil || explicit.Server != "127.0.0.1:9" { + t.Fatalf("explicit server must be kept: %+v, %v", explicit, err) + } + + bad := RuntimeConfig{Mode: "nope"} + if err := bad.Validate(); err == nil { + t.Fatal("unknown mode must fail Validate") + } + if explicit.Supervise != SuperviseChild || explicit.Binary != DefaultBinary { + t.Fatalf("supervision defaults = %q %q, want child/crux", explicit.Supervise, explicit.Binary) + } + ext := RuntimeConfig{Mode: ModeSidecar, Server: "127.0.0.1:9", Supervise: " External ", Binary: "/opt/crux"} + if err := ext.Validate(); err != nil || ext.Supervise != SuperviseExternal || ext.Binary != "/opt/crux" { + t.Fatalf("explicit supervision = %+v, %v", ext, err) + } + badSv := RuntimeConfig{Supervise: "systemd"} + if err := badSv.Validate(); err == nil || !strings.Contains(err.Error(), "systemd") { + t.Fatalf("unknown supervision = %v", err) + } + for _, s := range []string{"", "child", "CHILD ", "external"} { + if _, err := ParseSupervision(s); err != nil { + t.Errorf("ParseSupervision(%q) = %v", s, err) + } + } +} + +func TestRuntimeConfigFromEnv(t *testing.T) { + t.Setenv(ModeEnvVar, "sidecar") + t.Setenv(ServerEnvVar, "unix:///tmp/x.sock") + t.Setenv(ServeTokenEnvVar, "secret") + t.Setenv(SuperviseEnvVar, "external") + t.Setenv(BinaryEnvVar, "/opt/crux") + got := RuntimeConfigFromEnv() + want := RuntimeConfig{Mode: "sidecar", Server: "unix:///tmp/x.sock", Token: "secret", Supervise: "external", Binary: "/opt/crux"} + if got != want { + t.Fatalf("RuntimeConfigFromEnv() = %+v, want %+v", got, want) + } + t.Setenv(ModeEnvVar, "") + if got := RuntimeConfigFromEnv(); got.Mode != "" { + t.Fatalf("unset %s must read as empty (defaulted by Validate), got %q", ModeEnvVar, got.Mode) + } +} + +func TestResolveReasoningEffort(t *testing.T) { + cases := []struct{ backend, model, configured, want string }{ + {"agy", "gemini-pro", "", AgyDefaultEffort}, + {"agy", "gemini-pro-high", "", "high"}, + {"agy", "gemini-pro", "medium", "medium"}, + {"agy", "gemini-pro", "xhigh", AgyDefaultEffort}, + {"agy", "", "high", ""}, + {"codex", "o3", "xhigh", "xhigh"}, + {"codex", "o3", "", ""}, + {"omp", "m", "high", "high"}, + {ClaudeBackend, "opus", ClaudeEffortHigh, ClaudeEffortHigh}, + {ClaudeBackend, "opus", "bogus", ""}, + {"copilot", "gpt", "high", ""}, + } + for _, tc := range cases { + if got := ResolveReasoningEffort(tc.backend, tc.model, tc.configured); got != tc.want { + t.Errorf("ResolveReasoningEffort(%q, %q, %q) = %q, want %q", tc.backend, tc.model, tc.configured, got, tc.want) + } + } + if AgyLaunchEffort("x", "low") != "low" || AgyLaunchEffort("x-medium", "") != "medium" { + t.Fatal("AgyLaunchEffort: configured effort must win, then the model suffix") + } +} diff --git a/src/pkg/crux/config/origin_labels.go b/src/crux/config/origin_labels.go similarity index 100% rename from src/pkg/crux/config/origin_labels.go rename to src/crux/config/origin_labels.go diff --git a/src/pkg/crux/config/origin_labels_test.go b/src/crux/config/origin_labels_test.go similarity index 100% rename from src/pkg/crux/config/origin_labels_test.go rename to src/crux/config/origin_labels_test.go diff --git a/src/pkg/crux/config/providers.go b/src/crux/config/providers.go similarity index 100% rename from src/pkg/crux/config/providers.go rename to src/crux/config/providers.go diff --git a/src/pkg/crux/config/proxy_inject.go b/src/crux/config/proxy_inject.go similarity index 100% rename from src/pkg/crux/config/proxy_inject.go rename to src/crux/config/proxy_inject.go diff --git a/src/crux/config/reasoning_effort.go b/src/crux/config/reasoning_effort.go new file mode 100644 index 0000000000..e6372b88c8 --- /dev/null +++ b/src/crux/config/reasoning_effort.go @@ -0,0 +1,77 @@ +package config + +import "strings" + +// AgyDefaultEffort is the reasoning effort passed alongside agy's --model +// when the agent has no usable reasoning_effort configured (see +// AgyLaunchEffort). agy requires --effort whenever --model is given and +// otherwise ignores the model entirely; "low" is the effort agy defaults to +// on its own, so this makes the configured model take effect without +// changing behaviour. +const AgyDefaultEffort = "low" + +// AgyLaunchEffort returns the --effort agy is launched with: the configured +// reasoning effort when it is one agy accepts (low/medium/high), else the +// effort encoded in agy's model id suffix, else AgyDefaultEffort. The +// rejection mirrors the contributor relay's rule — agy refuses efforts from +// codex's wider vocabulary (e.g. xhigh), and launching with one would make +// agy ignore the model outright, the exact failure --effort exists to prevent. +func AgyLaunchEffort(model, effort string) string { + for _, v := range ReasoningEffortsByBackend["agy"] { + if v == effort { + return effort + } + } + for _, v := range ReasoningEffortsByBackend["agy"] { + if strings.HasSuffix(model, "-"+v) { + return v + } + } + return AgyDefaultEffort +} + +// ResolveReasoningEffort reports the reasoning effort the runtime actually +// launches a given backend/model pair with, given the agent's configured +// reasoning_effort. The attribution trail is resolved in TWO places — +// Manager.InvocationMetadata for a running agent, and cmd/hive's fallback +// that reads straight from config when the runtime does not know the agent — +// and both must give the same answer. It is pure, so it lives in the +// configuration leaf rather than behind the runtime seam +// (hivecommons/hive#14138, stage 1). +// +// Before this existed the fallback carried its own hardcoded "low", so +// changing AgyDefaultEffort would have left cmd/hive silently stamping PRs +// with an effort agy was no longer being launched with. An attribution trail +// that misreports is worse than one that says nothing. +// +// The rules mirror the launch path exactly: +// - agy REQUIRES --effort whenever --model is given, so with a model it runs +// at AgyLaunchEffort(model, configured) and with no model at no effort at all. +// - codex is launched with `-c model_reasoning_effort` only when an effort +// is configured; unset means codex's own default, which the hive does not +// resolve, so the honest answer is the configured value verbatim. +// - claude is launched with `--effort` only when the configured effort is +// one Claude Code accepts (the launch path drops anything else), so the +// answer is the configured value when valid and "" otherwise — never a +// value the CLI was not actually given (hivecommons/hive#8377). +// - omp is launched with `--thinking` only when an effort is configured; +// unset means omp's per-model default, so report the configured value. +// - every other backend takes its effort from its own config, which the +// hive does not resolve here, so the honest answer is "". +func ResolveReasoningEffort(backend, model, configured string) string { + switch backend { + case "agy": + if model != "" { + return AgyLaunchEffort(model, configured) + } + return "" + case "codex", "omp": + return configured + case ClaudeBackend: + if ValidEffort(ClaudeBackend, configured) { + return configured + } + return "" + } + return "" +} diff --git a/src/pkg/crux/config/reasoning_effort_test.go b/src/crux/config/reasoning_effort_test.go similarity index 100% rename from src/pkg/crux/config/reasoning_effort_test.go rename to src/crux/config/reasoning_effort_test.go diff --git a/src/pkg/crux/config/repeat_failure.go b/src/crux/config/repeat_failure.go similarity index 100% rename from src/pkg/crux/config/repeat_failure.go rename to src/crux/config/repeat_failure.go diff --git a/src/pkg/crux/config/repo_pause.go b/src/crux/config/repo_pause.go similarity index 100% rename from src/pkg/crux/config/repo_pause.go rename to src/crux/config/repo_pause.go diff --git a/src/pkg/crux/config/review_heads.go b/src/crux/config/review_heads.go similarity index 100% rename from src/pkg/crux/config/review_heads.go rename to src/crux/config/review_heads.go diff --git a/src/pkg/crux/config/rotation_config_test.go b/src/crux/config/rotation_config_test.go similarity index 100% rename from src/pkg/crux/config/rotation_config_test.go rename to src/crux/config/rotation_config_test.go diff --git a/src/pkg/crux/config/sandbox_config_test.go b/src/crux/config/sandbox_config_test.go similarity index 100% rename from src/pkg/crux/config/sandbox_config_test.go rename to src/crux/config/sandbox_config_test.go diff --git a/src/pkg/crux/config/secret_path_test.go b/src/crux/config/secret_path_test.go similarity index 100% rename from src/pkg/crux/config/secret_path_test.go rename to src/crux/config/secret_path_test.go diff --git a/src/crux/config/serve.go b/src/crux/config/serve.go new file mode 100644 index 0000000000..9971fa19e6 --- /dev/null +++ b/src/crux/config/serve.go @@ -0,0 +1,93 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/hivecommons/hive/crux/api" +) + +// Where `crux serve` listens and how a client finds it (Crux Stage 3, +// hivecommons/hive#13536; src/docs/design/crux.md "Stage 3"). These are +// process-level settings, not hive.yaml keys: the server is started by an +// operator or a harness, and the client is pointed at it by flag or +// environment. Nothing here is reachable from Config, so the settings-parity +// walk never sees it. +const ( + // ServeSocketEnvVar overrides the default unix socket path for + // `crux serve` and for a client that is given no --server. + ServeSocketEnvVar = "CRUX_SOCKET" + // ServerEnvVar is the address a `crux` client talks to instead of building + // an in-process runtime: `unix:///path`, an absolute socket path, or + // `host:port` / `http://host:port` for a TCP listener. + ServerEnvVar = "CRUX_SERVER" + // ServeTokenEnvVar is the bearer token the server requires and the client + // presents. A TCP listener refuses to start without one. + ServeTokenEnvVar = "CRUX_TOKEN" + // ServeSocketName is the socket file name under ServeSocketDir. + ServeSocketName = "crux.sock" + // ServeSocketDirName is the per-user directory under $XDG_RUNTIME_DIR or + // the system temp dir that holds the socket. + ServeSocketDirName = "crux" + // ServeSocketMode is the permission the server sets on its socket file: + // owner only, since a unix socket with no token is authenticated by the + // file system alone. + ServeSocketMode os.FileMode = 0o600 + // ServeSocketDirMode is the permission for a socket directory the server + // creates. + ServeSocketDirMode os.FileMode = 0o700 +) + +// ServeConfig is what `crux serve` listens on. Zero values mean "unix socket +// at DefaultServeSocket, no TCP". +type ServeConfig struct { + // Socket is the unix socket path. Empty selects DefaultServeSocket(). + Socket string + // Listen is an optional TCP host:port. It is refused without a Token, + // and refused on a non-loopback host: the listener is plain http, so + // the token would cross the network in cleartext. Remote access goes + // through a TLS-terminating proxy that dials the loopback listener. + Listen string + // Token is the bearer token every request must carry. Optional on the + // socket (the file mode is the credential), required for Listen. + Token string +} + +// Validate applies the defaults and the fail-closed rules: no TCP without a +// token, and no TCP off loopback (the same rule the client applies in +// api.ParseAddress: a bearer token travels over plain http only to the +// local machine). +func (c *ServeConfig) Validate() error { + if c.Socket == "" { + c.Socket = DefaultServeSocket() + } + if c.Listen != "" && c.Token == "" { + return fmt.Errorf("a TCP listener on %s needs a bearer token (--token or %s); refusing to serve the runtime unauthenticated", c.Listen, ServeTokenEnvVar) + } + if c.Listen != "" && !api.LoopbackHost(c.Listen) { + return fmt.Errorf("listen %s: a TCP listener must bind a loopback address (127.0.0.1, [::1] or localhost); the listener is plain http and the bearer token would cross the network in cleartext — put a TLS-terminating proxy in front for remote access", c.Listen) + } + return nil +} + +// DefaultServeSocket is the socket path `crux serve` and the client agree on +// when neither is told otherwise: $CRUX_SOCKET if set, else +// $XDG_RUNTIME_DIR/crux/crux.sock, else /crux-/crux.sock. The uid +// in the temp-dir fallback keeps two users on one host from colliding on a +// shared /tmp. +func DefaultServeSocket() string { + if p := os.Getenv(ServeSocketEnvVar); p != "" { + return p + } + return filepath.Join(DefaultServeSocketDir(), ServeSocketName) +} + +// DefaultServeSocketDir is the directory DefaultServeSocket places the socket +// in; see there for the rule. +func DefaultServeSocketDir() string { + if dir := os.Getenv("XDG_RUNTIME_DIR"); dir != "" { + return filepath.Join(dir, ServeSocketDirName) + } + return filepath.Join(os.TempDir(), fmt.Sprintf("%s-%d", ServeSocketDirName, os.Getuid())) +} diff --git a/src/crux/config/serve_test.go b/src/crux/config/serve_test.go new file mode 100644 index 0000000000..cc3143167b --- /dev/null +++ b/src/crux/config/serve_test.go @@ -0,0 +1,67 @@ +package config + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestServeConfigValidateDefaultsAndFailsClosed(t *testing.T) { + t.Setenv(ServeSocketEnvVar, "") + t.Setenv("XDG_RUNTIME_DIR", t.TempDir()) + + var c ServeConfig + if err := c.Validate(); err != nil { + t.Fatalf("empty config: %v", err) + } + if c.Socket != DefaultServeSocket() { + t.Fatalf("Socket = %q, want default %q", c.Socket, DefaultServeSocket()) + } + + c = ServeConfig{Listen: "127.0.0.1:9000"} + err := c.Validate() + if err == nil || !strings.Contains(err.Error(), ServeTokenEnvVar) { + t.Fatalf("TCP without token must be refused, got %v", err) + } + + c = ServeConfig{Socket: "/x/y.sock", Listen: "127.0.0.1:9000", Token: "s3cret"} + if err := c.Validate(); err != nil || c.Socket != "/x/y.sock" { + t.Fatalf("explicit socket + token: err %v socket %q", err, c.Socket) + } + + // The listener is plain http; a token alone does not make a non-loopback + // bind safe, so it is refused even with one. + for _, listen := range []string{"0.0.0.0:9000", ":9000", "[::]:9000", "10.0.0.5:9000", "crux.example:9000"} { + c = ServeConfig{Listen: listen, Token: "s3cret"} + err := c.Validate() + if err == nil || !strings.Contains(err.Error(), "loopback") || !strings.Contains(err.Error(), "listen "+listen) { + t.Fatalf("Listen %q with token must be refused as non-loopback, got %v", listen, err) + } + } + for _, listen := range []string{"[::1]:9000", "localhost:9000", "127.0.0.2:9000"} { + c = ServeConfig{Listen: listen, Token: "s3cret"} + if err := c.Validate(); err != nil { + t.Fatalf("Listen %q with token: %v", listen, err) + } + } +} + +func TestDefaultServeSocketPrecedence(t *testing.T) { + t.Setenv(ServeSocketEnvVar, "/override/crux.sock") + if got := DefaultServeSocket(); got != "/override/crux.sock" { + t.Fatalf("%s should win, got %q", ServeSocketEnvVar, got) + } + + t.Setenv(ServeSocketEnvVar, "") + runtimeDir := t.TempDir() + t.Setenv("XDG_RUNTIME_DIR", runtimeDir) + if got, want := DefaultServeSocket(), filepath.Join(runtimeDir, ServeSocketDirName, ServeSocketName); got != want { + t.Fatalf("XDG default = %q, want %q", got, want) + } + + t.Setenv("XDG_RUNTIME_DIR", "") + got := DefaultServeSocket() + if !strings.HasPrefix(filepath.Base(filepath.Dir(got)), ServeSocketDirName+"-") || filepath.Base(got) != ServeSocketName { + t.Fatalf("temp-dir default = %q, want /%s-/%s", got, ServeSocketDirName, ServeSocketName) + } +} diff --git a/src/pkg/crux/config/standby.go b/src/crux/config/standby.go similarity index 100% rename from src/pkg/crux/config/standby.go rename to src/crux/config/standby.go diff --git a/src/pkg/crux/config/task_mcp.go b/src/crux/config/task_mcp.go similarity index 100% rename from src/pkg/crux/config/task_mcp.go rename to src/crux/config/task_mcp.go diff --git a/src/pkg/crux/config/tool_presets.go b/src/crux/config/tool_presets.go similarity index 100% rename from src/pkg/crux/config/tool_presets.go rename to src/crux/config/tool_presets.go diff --git a/src/pkg/crux/config/tool_presets_test.go b/src/crux/config/tool_presets_test.go similarity index 100% rename from src/pkg/crux/config/tool_presets_test.go rename to src/crux/config/tool_presets_test.go diff --git a/src/pkg/crux/config/triggers_config.go b/src/crux/config/triggers_config.go similarity index 100% rename from src/pkg/crux/config/triggers_config.go rename to src/crux/config/triggers_config.go diff --git a/src/pkg/crux/config/turn_config.go b/src/crux/config/turn_config.go similarity index 100% rename from src/pkg/crux/config/turn_config.go rename to src/crux/config/turn_config.go diff --git a/src/pkg/crux/config/watchdog.go b/src/crux/config/watchdog.go similarity index 100% rename from src/pkg/crux/config/watchdog.go rename to src/crux/config/watchdog.go diff --git a/src/pkg/agent/copilot_models.go b/src/crux/copilot_models.go similarity index 95% rename from src/pkg/agent/copilot_models.go rename to src/crux/copilot_models.go index 4c6f5d7fa0..6ae71fef2c 100644 --- a/src/pkg/agent/copilot_models.go +++ b/src/crux/copilot_models.go @@ -1,4 +1,4 @@ -package agent +package crux import ( "strings" @@ -19,7 +19,9 @@ import ( // ids for that model (consistent with every other X.Y id in this list) and // restore the alias so a dashed input still reaches the CLI as a dotted, // accepted id instead of being dropped entirely. Keep in sync with -// copilotPinnedCLIModels in pkg/dashboard/cli_models.go. +// copilotPinnedCLIModels in pkg/dashboard/cli_models.go (the dashboard +// canonicalizes through crux.CanonicalizeCopilotModel, so this list lives in +// the module root rather than crux/agent). var copilotCLIAcceptedModels = []string{ // Anthropic — bare generation numbers are separator-free. "claude-opus-5", diff --git a/src/pkg/agent/copilot_models_test.go b/src/crux/copilot_models_test.go similarity index 79% rename from src/pkg/agent/copilot_models_test.go rename to src/crux/copilot_models_test.go index 30364bdb32..bad0ccc623 100644 --- a/src/pkg/agent/copilot_models_test.go +++ b/src/crux/copilot_models_test.go @@ -1,4 +1,4 @@ -package agent +package crux import "testing" @@ -86,31 +86,6 @@ func TestCanonicalizeCopilotModelIdempotent(t *testing.T) { } } -// TestNormalizeModelNameCopilotDrift verifies the launch-time path: the old -// blind trailing-digits dot-rewrite corrupted claude-fable-5 into the -// CLI-rejected claude-fable.5 (#4262); normalizeModelName must now emit -// CLI-accepted spellings for copilot, self-correcting stored bad ids. -func TestNormalizeModelNameCopilotDrift(t *testing.T) { - tests := []struct{ in, want string }{ - {"claude-fable-5", "claude-fable-5"}, // dashed family must NOT gain a dot - {"claude-fable.5", "claude-fable-5"}, // stored bad id self-corrects at launch - {"claude-sonnet-5", "claude-sonnet-5"}, - {"claude-opus-5", "claude-opus-5"}, - {"claude-sonnet-4-6", "claude-sonnet-4.6"}, // YAML-friendly dashed still maps to dotted - {"claude-opus-4.6", "claude-opus-4.6"}, // legitimate dot unchanged - {"gpt-5.5", "gpt-5.5"}, - {"gemini-2.5-pro", "gemini-2.5-pro"}, - {"auto", "auto"}, // auto-select sentinel flows through - {"gpt-next", "gpt-next"}, // unknown id passthrough - {"custom-7", "custom-7"}, // unknown id: no blind dot-rewrite anymore - } - for _, tt := range tests { - if got := normalizeModelNameForBackend(tt.in, "copilot", false); got != tt.want { - t.Errorf("normalizeModelNameForBackend(%q, copilot, false) = %q, want %q", tt.in, got, tt.want) - } - } -} - // TestCopilotLaunchModelRejectedFallback covers #9927: stored 5-5 selections // launch on the CLI-accepted DOTTED id instead of silently downgrading to the // bare -5 family. CopilotLaunchModel used to re-downgrade an already-correctly diff --git a/src/crux/cruxtest/recorder.go b/src/crux/cruxtest/recorder.go new file mode 100644 index 0000000000..a52252ced7 --- /dev/null +++ b/src/crux/cruxtest/recorder.go @@ -0,0 +1,1111 @@ +package cruxtest + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/effects" + "github.com/hivecommons/hive/crux/pushbroker" + "github.com/hivecommons/hive/crux/trajectory" + "github.com/hivecommons/hive/crux/watchdog" +) + +// Call is one recorded method call: the crux.Runtime (or crux.Host) method +// name and its arguments in order, contexts omitted. +type Call struct { + Method string + Args []any +} + +// String renders the call as Method(arg, arg, ...) for assertions. +func (c Call) String() string { + return fmt.Sprintf("%s%v", c.Method, c.Args) +} + +// Recorder is a crux.Runtime (and crux.Host, and a watchdog source) that +// records every call and answers from canned state, for tests of anything +// that drives a runtime without running one: the API server, the API client, +// the CLI. Every method appends a Call; the exported fields are the answers. +// Err, when set, is returned by every method that returns an error, except +// the ones with their own error field (ResumeErr, AuthorizeErr, KickErr). +// Methods that look an agent up answer "agent X not found" for a name absent +// from Procs. The zero value is usable; NewRecorder pre-sizes the maps. +type Recorder struct { + mu sync.Mutex + calls []Call + + // Procs is the agent table AllStatuses/GetStatus answer from. + Procs map[string]*crux.Process + // Pane, Buffer, FullLog and Header answer the Observer log reads. + Pane map[string][]string + Buffer map[string][]string + FullLog map[string]string + Header map[string]string + // Err is the generic error answer; see the type comment. + Err error + // KickErr is SendKick / SendKickAsync / ValidateKick's error. + KickErr error + // KickStarted is SendKickAsync's started result. + KickStarted bool + // Dispatch answers KickDispatchState. + Dispatch map[string]crux.KickDispatch + // KickLogs and KickLogContent answer ListKickLogs and ReadKickLog (keyed + // by agent, then by id). + KickLogs map[string][]crux.KickLogInfo + KickLogContent map[string]map[string]string + // Reconciled is ReconcileAgents' result. + Reconciled []string + // Breaker is the breaker state; EngageBreaker sets Engaged and returns + // Paused, ReleaseBreaker clears it and returns Resumed. + BreakerEngaged bool + BreakerPaused []string + Resumed []string + // ACMM is the fleet level. + ACMM int + // Shared is SharedDevUID. + Shared bool + // Relaunched is RelaunchBobAgentsAwaitingKey's result; Crashed is + // CheckAndRestartCrashedAgents'; Wedged is ConsentWedgedAgents'. + Relaunched []string + Crashed []string + Wedged []string + // WithModel is CountAgentsWithModel. + WithModel int + // Copilot and CopilotSource answer the Copilot credential reads; + // SetCopilotToken and ActivateCopilotToken overwrite Copilot. + Copilot string + CopilotSource string + // ClaudePaths is ClaudeCredentialCandidatePaths. + ClaudePaths []string + // OmpHomes is OmpAgentHomes. + OmpHomes []string + // Handles, Sessions and Missing answer the Resumer reads. + Handles map[string]crux.ResumeHandle + Sessions map[string]string + Missing map[string]bool + // ResumeErr is SendResumeKick's error. + ResumeErr error + // Supervision answers the per-agent Supervisor reads. + MayStart bool + Refreshable bool + RefreshKnown bool + Valid bool + Caps [4]bool + Backend string + BackendOK bool + Invocation [3]string + InvocationOK bool + Restarts struct { + Total, Last24h int + LastAt time.Time + LastReason string + OK bool + } + StartFailure crux.StartFailureSnapshot + StartFailureOK bool + Backoff struct { + Remaining time.Duration + Provider, Reason string + OK bool + } + OOM bool + AuthAvail bool + AuthKnown bool + BackendAuth map[string][2]bool + Views []trajectory.AgentView + AuthorizeErr error + + // Observers installed through Kicker.SetKickObserver and crux.Host. + KickObserver func(agentName, event, detail string) + KickOutcomeObserver func(agentName string, outcome crux.KickOutcome) + PauseObserver func(crux.PauseTransitionEvent) + RelaunchObserver func(agentName string) + PersistPause func(name string, paused bool) + RecordPrompt func(agent, trigger, prompt string) + SandboxAudit func(agent, action, detail string) + Audit crux.AuditSink + Conditions crux.ConditionReporter + + // Watchdog source answers. + WatchdogNames []string + Observations map[string]watchdog.Observation + Production map[string]time.Time + SetConds map[string][]crux.Condition + BackgroundRuns []string +} + +// NewRecorder returns a Recorder with its maps allocated. +func NewRecorder() *Recorder { + return &Recorder{ + Procs: map[string]*crux.Process{}, + Pane: map[string][]string{}, + Buffer: map[string][]string{}, + FullLog: map[string]string{}, + Header: map[string]string{}, + Dispatch: map[string]crux.KickDispatch{}, + KickLogs: map[string][]crux.KickLogInfo{}, + KickLogContent: map[string]map[string]string{}, + Handles: map[string]crux.ResumeHandle{}, + Sessions: map[string]string{}, + Missing: map[string]bool{}, + BackendAuth: map[string][2]bool{}, + Observations: map[string]watchdog.Observation{}, + Production: map[string]time.Time{}, + SetConds: map[string][]crux.Condition{}, + } +} + +// Add registers an agent in Procs with the given id and state and returns +// its record. +func (r *Recorder) Add(name, id string, state crux.ProcessState) *crux.Process { + r.mu.Lock() + defer r.mu.Unlock() + p := &crux.Process{Name: name, ID: id, State: state, Config: config.AgentConfig{ID: id, Backend: "claude", Model: "opus", Enabled: true}} + r.Procs[name] = p + return p +} + +// Calls returns a copy of the recorded calls. +func (r *Recorder) Calls() []Call { + r.mu.Lock() + defer r.mu.Unlock() + return append([]Call(nil), r.calls...) +} + +// Methods returns the recorded method names in order. +func (r *Recorder) Methods() []string { + calls := r.Calls() + out := make([]string, len(calls)) + for i, c := range calls { + out[i] = c.Method + } + return out +} + +// Last returns the most recent call to method, if any. +func (r *Recorder) Last(method string) (Call, bool) { + calls := r.Calls() + for i := len(calls) - 1; i >= 0; i-- { + if calls[i].Method == method { + return calls[i], true + } + } + return Call{}, false +} + +// Reset forgets the recorded calls. +func (r *Recorder) Reset() { + r.mu.Lock() + r.calls = nil + r.mu.Unlock() +} + +func (r *Recorder) record(method string, args ...any) { + r.mu.Lock() + r.calls = append(r.calls, Call{Method: method, Args: args}) + r.mu.Unlock() +} + +func (r *Recorder) lookup(name string) (*crux.Process, error) { + r.mu.Lock() + defer r.mu.Unlock() + p, ok := r.Procs[name] + if !ok { + return nil, fmt.Errorf("agent %s not found", name) + } + return p, nil +} + +// errFor is the error a lookup-then-act method returns: the generic Err if +// set, else not-found for an unknown agent. +func (r *Recorder) errFor(name string) error { + if r.Err != nil { + return r.Err + } + _, err := r.lookup(name) + return err +} + +func copyProcess(p *crux.Process) *crux.Process { + if p == nil { + return nil + } + c := *p + return &c +} + +// ---- Lifecycle ---- + +func (r *Recorder) Start(_ context.Context, name string) error { + r.record("Start", name) + if err := r.errFor(name); err != nil { + return err + } + r.mu.Lock() + r.Procs[name].State = crux.StateRunning + r.mu.Unlock() + return nil +} + +func (r *Recorder) Stop(name string) error { + r.record("Stop", name) + if err := r.errFor(name); err != nil { + return err + } + r.mu.Lock() + r.Procs[name].State = crux.StateStopped + r.mu.Unlock() + return nil +} + +func (r *Recorder) Restart(_ context.Context, name string) error { + r.record("Restart", name) + return r.errFor(name) +} + +func (r *Recorder) RestartWithBootstrap(_ context.Context, name, prompt string) error { + r.record("RestartWithBootstrap", name, prompt) + return r.errFor(name) +} + +func (r *Recorder) Pause(name, trigger, reason string) error { + r.record("Pause", name, trigger, reason) + return r.setPaused(name, true, trigger, reason, "") +} + +func (r *Recorder) PauseBy(name, trigger, reason, by string) error { + r.record("PauseBy", name, trigger, reason, by) + return r.setPaused(name, true, trigger, reason, by) +} + +func (r *Recorder) PauseByCause(name, trigger, reason, by string, cause crux.PauseCausation) error { + r.record("PauseByCause", name, trigger, reason, by, cause) + return r.setPaused(name, true, trigger, reason, by) +} + +func (r *Recorder) Resume(_ context.Context, name, trigger, reason string) error { + r.record("Resume", name, trigger, reason) + return r.setPaused(name, false, trigger, reason, "") +} + +func (r *Recorder) setPaused(name string, paused bool, trigger, reason, by string) error { + if err := r.errFor(name); err != nil { + return err + } + r.mu.Lock() + p := r.Procs[name] + p.Paused, p.PausedTrigger, p.PausedReason, p.PausedBy = paused, trigger, reason, by + r.mu.Unlock() + return nil +} + +func (r *Recorder) IsPaused(name string) bool { + r.record("IsPaused", name) + p, err := r.lookup(name) + return err == nil && p.Paused +} + +func (r *Recorder) ResetRestartCount(name string) error { + r.record("ResetRestartCount", name) + if err := r.errFor(name); err != nil { + return err + } + r.mu.Lock() + r.Procs[name].RestartCount = 0 + r.mu.Unlock() + return nil +} + +// ---- Observer ---- + +func (r *Recorder) GetStatus(name string) (*crux.Process, error) { + r.record("GetStatus", name) + p, err := r.lookup(name) + if err != nil { + return nil, err + } + c := copyProcess(p) + r.mu.Lock() + pane := append([]string(nil), r.Pane[name]...) + r.mu.Unlock() + if len(pane) > 0 { + c.SetPaneCapture(pane, tail) + } + return c, nil +} + +func tail(lines []string, n int) []string { + if n <= 0 || n >= len(lines) { + return lines + } + return lines[len(lines)-n:] +} + +func (r *Recorder) GetStatusFast(name string) (*crux.Process, error) { + r.record("GetStatusFast", name) + p, err := r.lookup(name) + if err != nil { + return nil, err + } + return copyProcess(p), nil +} + +func (r *Recorder) AllStatuses() map[string]*crux.Process { + r.record("AllStatuses") + r.mu.Lock() + defer r.mu.Unlock() + out := make(map[string]*crux.Process, len(r.Procs)) + for k, v := range r.Procs { + out[k] = copyProcess(v) + } + return out +} + +func (r *Recorder) ResolveAgent(nameOrID string) string { + r.record("ResolveAgent", nameOrID) + r.mu.Lock() + defer r.mu.Unlock() + if _, ok := r.Procs[nameOrID]; ok { + return nameOrID + } + for name, p := range r.Procs { + if p.ID == nameOrID { + return name + } + } + return nameOrID +} + +func (r *Recorder) GetOutput(name string, lines int) ([]string, error) { + r.record("GetOutput", name, lines) + if err := r.errFor(name); err != nil { + return nil, err + } + r.mu.Lock() + defer r.mu.Unlock() + return tail(append([]string(nil), r.Pane[name]...), lines), nil +} + +func (r *Recorder) GetBufferOutput(name string, lines int) ([]string, error) { + r.record("GetBufferOutput", name, lines) + if err := r.errFor(name); err != nil { + return nil, err + } + r.mu.Lock() + defer r.mu.Unlock() + return tail(append([]string(nil), r.Buffer[name]...), lines), nil +} + +func (r *Recorder) CaptureFullLog(name string) (string, error) { + r.record("CaptureFullLog", name) + if err := r.errFor(name); err != nil { + return "", err + } + r.mu.Lock() + defer r.mu.Unlock() + return r.FullLog[name], nil +} + +func (r *Recorder) FullLogHeader(name string) string { + r.record("FullLogHeader", name) + r.mu.Lock() + defer r.mu.Unlock() + return r.Header[name] +} + +// ---- Kicker ---- + +func (r *Recorder) SendKick(name string, message string) error { + r.record("SendKick", name, message) + if r.KickErr != nil { + return r.KickErr + } + return r.errFor(name) +} + +func (r *Recorder) SendKickWithSource(name string, message string, source string) error { + r.record("SendKickWithSource", name, message, source) + if r.KickErr != nil { + return r.KickErr + } + return r.errFor(name) +} + +func (r *Recorder) SendKickAsync(name string, message string) (bool, error) { + r.record("SendKickAsync", name, message) + if r.KickErr != nil { + return false, r.KickErr + } + if err := r.errFor(name); err != nil { + return false, err + } + return r.KickStarted, nil +} + +func (r *Recorder) ValidateKick(name string) error { + r.record("ValidateKick", name) + if r.KickErr != nil { + return r.KickErr + } + return r.errFor(name) +} + +func (r *Recorder) KickDispatchState(name string) (crux.KickDispatch, bool) { + r.record("KickDispatchState", name) + r.mu.Lock() + defer r.mu.Unlock() + d, ok := r.Dispatch[name] + return d, ok +} + +func (r *Recorder) ListKickLogs(name string) ([]crux.KickLogInfo, error) { + r.record("ListKickLogs", name) + if err := r.errFor(name); err != nil { + return nil, err + } + r.mu.Lock() + defer r.mu.Unlock() + return append([]crux.KickLogInfo(nil), r.KickLogs[name]...), nil +} + +func (r *Recorder) ReadKickLog(name, id string) (string, error) { + r.record("ReadKickLog", name, id) + if err := r.errFor(name); err != nil { + return "", err + } + r.mu.Lock() + defer r.mu.Unlock() + content, ok := r.KickLogContent[name][id] + if !ok { + return "", fmt.Errorf("kick log %s not found", id) + } + return content, nil +} + +func (r *Recorder) SetKickObserver(fn func(agentName, event, detail string)) { + r.record("SetKickObserver", fn != nil) + r.mu.Lock() + r.KickObserver = fn + r.mu.Unlock() +} + +// EmitKick calls the installed kick observer, if any. +func (r *Recorder) EmitKick(agentName, event, detail string) { + r.mu.Lock() + fn := r.KickObserver + r.mu.Unlock() + if fn != nil { + fn(agentName, event, detail) + } +} + +// ---- Definitions ---- + +func (r *Recorder) AddAgent(name string, cfg config.AgentConfig) { + r.record("AddAgent", name, cfg) + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.Procs[name]; exists { + return + } + r.Procs[name] = &crux.Process{Name: name, ID: cfg.ID, Config: cfg, State: crux.StateStopped} +} + +func (r *Recorder) RemoveAgent(name string) { + r.record("RemoveAgent", name) + r.mu.Lock() + delete(r.Procs, name) + r.mu.Unlock() +} + +func (r *Recorder) UpdateConfig(name string, cfg config.AgentConfig) error { + r.record("UpdateConfig", name, cfg) + if err := r.errFor(name); err != nil { + return err + } + r.mu.Lock() + r.Procs[name].Config = cfg + r.mu.Unlock() + return nil +} + +func (r *Recorder) ReconcileAgents(configs map[string]config.AgentConfig) []string { + names := make([]string, 0, len(configs)) + for n := range configs { + names = append(names, n) + } + sort.Strings(names) + r.record("ReconcileAgents", names) + return append([]string(nil), r.Reconciled...) +} + +func (r *Recorder) SetModelOverride(name, model string) error { + r.record("SetModelOverride", name, model) + if err := r.errFor(name); err != nil { + return err + } + r.mu.Lock() + r.Procs[name].ModelOverride = model + r.mu.Unlock() + return nil +} + +func (r *Recorder) SetBackendOverride(name, backend string) error { + r.record("SetBackendOverride", name, backend) + if err := r.errFor(name); err != nil { + return err + } + r.mu.Lock() + r.Procs[name].BackendOverride = backend + r.mu.Unlock() + return nil +} + +func (r *Recorder) PinCLI(name, version string) error { + r.record("PinCLI", name, version) + return r.errFor(name) +} + +func (r *Recorder) PinModel(name, model string) error { + r.record("PinModel", name, model) + return r.errFor(name) +} + +func (r *Recorder) UnpinCLI(name string) error { + r.record("UnpinCLI", name) + return r.errFor(name) +} + +func (r *Recorder) UnpinModel(name string) error { + r.record("UnpinModel", name) + return r.errFor(name) +} + +func (r *Recorder) ClearModeOverrides(names []string) { + r.record("ClearModeOverrides", append([]string(nil), names...)) +} + +func (r *Recorder) SyncModeFiles(level int) { + r.record("SyncModeFiles", level) +} + +func (r *Recorder) RefreshInferenceRoutes(backend string) { + r.record("RefreshInferenceRoutes", backend) +} + +// ---- Fleet ---- + +func (r *Recorder) BreakerState() (bool, []string) { + r.record("BreakerState") + r.mu.Lock() + defer r.mu.Unlock() + return r.BreakerEngaged, append([]string(nil), r.BreakerPaused...) +} + +func (r *Recorder) EngageBreaker() []string { + r.record("EngageBreaker") + r.mu.Lock() + defer r.mu.Unlock() + r.BreakerEngaged = true + return append([]string(nil), r.BreakerPaused...) +} + +func (r *Recorder) ReleaseBreaker(context.Context) []string { + r.record("ReleaseBreaker") + r.mu.Lock() + defer r.mu.Unlock() + r.BreakerEngaged = false + return append([]string(nil), r.Resumed...) +} + +func (r *Recorder) SetACMMLevel(level int) { + r.record("SetACMMLevel", level) + r.mu.Lock() + r.ACMM = level + r.mu.Unlock() +} + +func (r *Recorder) GetACMMLevel() int { + r.record("GetACMMLevel") + r.mu.Lock() + defer r.mu.Unlock() + return r.ACMM +} + +func (r *Recorder) RelaunchBobAgentsAwaitingKey(context.Context) []string { + r.record("RelaunchBobAgentsAwaitingKey") + return append([]string(nil), r.Relaunched...) +} + +func (r *Recorder) SharedDevUID() bool { + r.record("SharedDevUID") + return r.Shared +} + +// ---- Credentials ---- + +func (r *Recorder) SubmitLoginCode(name, code string) error { + r.record("SubmitLoginCode", name, code) + return r.errFor(name) +} + +func (r *Recorder) CopilotToken() string { + r.record("CopilotToken") + r.mu.Lock() + defer r.mu.Unlock() + return r.Copilot +} + +func (r *Recorder) CopilotTokenSource() string { + r.record("CopilotTokenSource") + r.mu.Lock() + defer r.mu.Unlock() + return r.CopilotSource +} + +func (r *Recorder) SetCopilotToken(token string) { + r.record("SetCopilotToken", token) + r.mu.Lock() + r.Copilot = token + r.mu.Unlock() +} + +func (r *Recorder) ActivateCopilotToken(token string) error { + r.record("ActivateCopilotToken", token) + if r.Err != nil { + return r.Err + } + r.mu.Lock() + r.Copilot = token + r.mu.Unlock() + return nil +} + +func (r *Recorder) ReloadClaudeToken() { + r.record("ReloadClaudeToken") +} + +func (r *Recorder) ClaudeCredentialCandidatePaths() []string { + r.record("ClaudeCredentialCandidatePaths") + return append([]string(nil), r.ClaudePaths...) +} + +func (r *Recorder) OmpAgentHomes() []string { + r.record("OmpAgentHomes") + return append([]string(nil), r.OmpHomes...) +} + +// ---- Resumer ---- + +func (r *Recorder) ResumeHandle(name string) (crux.ResumeHandle, bool) { + r.record("ResumeHandle", name) + r.mu.Lock() + defer r.mu.Unlock() + h, ok := r.Handles[name] + return h, ok +} + +func (r *Recorder) SendResumeKick(name, message, sessionID string) error { + r.record("SendResumeKick", name, message, sessionID) + if r.ResumeErr != nil { + return r.ResumeErr + } + return r.errFor(name) +} + +func (r *Recorder) SessionID(name string) (string, bool) { + r.record("SessionID", name) + r.mu.Lock() + defer r.mu.Unlock() + id, ok := r.Sessions[name] + return id, ok +} + +func (r *Recorder) SessionMissing(name string) bool { + r.record("SessionMissing", name) + r.mu.Lock() + defer r.mu.Unlock() + return r.Missing[name] +} + +// ---- Supervisor ---- + +func (r *Recorder) GovernorMayStart(name string) bool { + r.record("GovernorMayStart", name) + return r.MayStart +} + +func (r *Recorder) MarkStartupLaunchQueued(names []string) { + r.record("MarkStartupLaunchQueued", append([]string(nil), names...)) +} + +func (r *Recorder) CheckAndRestartCrashedAgents(context.Context) []string { + r.record("CheckAndRestartCrashedAgents") + return append([]string(nil), r.Crashed...) +} + +func (r *Recorder) ConsentWedgedAgents() []string { + r.record("ConsentWedgedAgents") + return append([]string(nil), r.Wedged...) +} + +func (r *Recorder) RefreshAgentTokens(context.Context) { + r.record("RefreshAgentTokens") +} + +func (r *Recorder) AgentCredentialRefreshable(agentName string) (bool, bool) { + r.record("AgentCredentialRefreshable", agentName) + return r.Refreshable, r.RefreshKnown +} + +func (r *Recorder) AgentCapabilities(agentName string) (bool, bool, bool, bool) { + r.record("AgentCapabilities", agentName) + return r.Caps[0], r.Caps[1], r.Caps[2], r.Caps[3] +} + +func (r *Recorder) EffectiveBackend(agentName string) (string, bool) { + r.record("EffectiveBackend", agentName) + return r.Backend, r.BackendOK +} + +func (r *Recorder) InvocationMetadata(agentName string) (string, string, string, bool) { + r.record("InvocationMetadata", agentName) + return r.Invocation[0], r.Invocation[1], r.Invocation[2], r.InvocationOK +} + +func (r *Recorder) RestartTelemetry(name string) (int, int, time.Time, string, bool) { + r.record("RestartTelemetry", name) + t := r.Restarts + return t.Total, t.Last24h, t.LastAt, t.LastReason, t.OK +} + +func (r *Recorder) StartFailureState(name string) (crux.StartFailureSnapshot, bool) { + r.record("StartFailureState", name) + return r.StartFailure, r.StartFailureOK +} + +func (r *Recorder) ProviderErrorBackoffRemaining(name string) (time.Duration, string, string, bool) { + r.record("ProviderErrorBackoffRemaining", name) + b := r.Backoff + return b.Remaining, b.Provider, b.Reason, b.OK +} + +func (r *Recorder) ArchiveAllKickLogs(reason string) { + r.record("ArchiveAllKickLogs", reason) +} + +func (r *Recorder) CountAgentsWithModel() int { + r.record("CountAgentsWithModel") + return r.WithModel +} + +func (r *Recorder) CrashOOMSuspected(name string) bool { + r.record("CrashOOMSuspected", name) + return r.OOM +} + +func (r *Recorder) BackendAuthAvailable(backend string) (bool, bool) { + r.record("BackendAuthAvailable", backend) + r.mu.Lock() + defer r.mu.Unlock() + v := r.BackendAuth[backend] + return v[0], v[1] +} + +func (r *Recorder) AgentAuthAvailable(agentName string) (bool, bool) { + r.record("AgentAuthAvailable", agentName) + return r.AuthAvail, r.AuthKnown +} + +func (r *Recorder) SendGitHubAPIBudgetNudge(name, message string) { + r.record("SendGitHubAPIBudgetNudge", name, message) +} + +func (r *Recorder) RecordAudit(action, agentName string, fields map[string]any) { + r.record("RecordAudit", action, agentName, fields) +} + +func (r *Recorder) AgentHasValidCredential(agentName string) bool { + r.record("AgentHasValidCredential", agentName) + return r.Valid +} + +func (r *Recorder) RefreshAgentTokenFor(_ context.Context, name string) error { + r.record("RefreshAgentTokenFor", name) + return r.errFor(name) +} + +func (r *Recorder) TrajectoryAgents() []trajectory.AgentView { + r.record("TrajectoryAgents") + return append([]trajectory.AgentView(nil), r.Views...) +} + +// ---- Restorer ---- + +func (r *Recorder) SeedPauseState(name string, pausedAt time.Time, trigger, reason, by string) { + r.record("SeedPauseState", name, pausedAt, trigger, reason, by) +} + +func (r *Recorder) SeedRestartTelemetry(name string, count int, events []crux.RestartEvent, lastReason string) { + r.record("SeedRestartTelemetry", name, count, append([]crux.RestartEvent(nil), events...), lastReason) +} + +func (r *Recorder) SeedTurnLoss(name string, loss crux.TurnLoss) { + r.record("SeedTurnLoss", name, loss) +} + +func (r *Recorder) SeedLastKick(name string, t time.Time) { + r.record("SeedLastKick", name, t) +} + +func (r *Recorder) SeedKickHistory(name string, records []crux.KickRecord) { + r.record("SeedKickHistory", name, append([]crux.KickRecord(nil), records...)) +} + +func (r *Recorder) RestoreBreaker(engaged bool, paused []string) { + r.record("RestoreBreaker", engaged, append([]string(nil), paused...)) + r.mu.Lock() + r.BreakerEngaged, r.BreakerPaused = engaged, append([]string(nil), paused...) + r.mu.Unlock() +} + +// ---- Background ---- +// +// Each loop records its start and returns when ctx ends. + +func (r *Recorder) StartAgentTokenRefresh(ctx context.Context) { + r.background(ctx, "StartAgentTokenRefresh") +} + +func (r *Recorder) StartCredentialWatchdog(ctx context.Context) { + r.background(ctx, "StartCredentialWatchdog") +} + +func (r *Recorder) StartCopilotSessionRefresh(ctx context.Context) { + r.background(ctx, "StartCopilotSessionRefresh") +} + +func (r *Recorder) StartTranscriptActivityRefresh(ctx context.Context) { + r.background(ctx, "StartTranscriptActivityRefresh") +} + +// Background lists the Background loops that have been started, in order; +// safe to call while they run. +func (r *Recorder) Background() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.BackgroundRuns...) +} + +func (r *Recorder) background(ctx context.Context, name string) { + r.record(name) + r.mu.Lock() + r.BackgroundRuns = append(r.BackgroundRuns, name) + r.mu.Unlock() + <-ctx.Done() +} + +// ---- Authorizer ---- + +func (r *Recorder) AuthorizePROpen(agentName string, fileUID int) error { + r.record("AuthorizePROpen", agentName, fileUID) + return r.AuthorizeErr +} + +func (r *Recorder) AuthorizePushBranch(agentName string, fileUID int) error { + r.record("AuthorizePushBranch", agentName, fileUID) + return r.AuthorizeErr +} + +func (r *Recorder) AuthorizeIssueOpen(agentName string, fileUID int, kind string) error { + r.record("AuthorizeIssueOpen", agentName, fileUID, kind) + return r.AuthorizeErr +} + +func (r *Recorder) AuthorizeMerge(agentName string, fileUID int, repo string) error { + r.record("AuthorizeMerge", agentName, fileUID, repo) + return r.AuthorizeErr +} + +func (r *Recorder) AuthorizeReviewRequest(agentName string, fileUID int) error { + r.record("AuthorizeReviewRequest", agentName, fileUID) + return r.AuthorizeErr +} + +// ---- crux.Host (observers kept; the rest recorded) ---- + +func (r *Recorder) SetAppAuth(auth crux.AppTokenMinter) { r.record("SetAppAuth", auth != nil) } +func (r *Recorder) SetAgentMint(issuer crux.AgentMintIssuer) { r.record("SetAgentMint", issuer != nil) } +func (r *Recorder) SetLinearCredentialResolver(fn func() crux.LinearCredential) { + r.record("SetLinearCredentialResolver", fn != nil) +} +func (r *Recorder) SetBobAPIKeyResolver(fn func() string) { + r.record("SetBobAPIKeyResolver", fn != nil) +} +func (r *Recorder) SetBobKeySourceResolver(fn func() string) { + r.record("SetBobKeySourceResolver", fn != nil) +} +func (r *Recorder) SetSandboxPushMinter(minter pushbroker.TokenMinter) { + r.record("SetSandboxPushMinter", minter != nil) +} +func (r *Recorder) SetSandboxPRClient(client crux.PRCreator) { + r.record("SetSandboxPRClient", client != nil) +} +func (r *Recorder) SetBuildInfo(fn func() (version, commit string)) { + r.record("SetBuildInfo", fn != nil) +} +func (r *Recorder) SetGatewayBackendChecker(fn func(backend string) bool) { + r.record("SetGatewayBackendChecker", fn != nil) +} +func (r *Recorder) SetExplainModeDefaultResolver(fn func() string) { + r.record("SetExplainModeDefaultResolver", fn != nil) +} + +func (r *Recorder) SetAdvisorEnabledResolver(fn func(string) bool) { + r.record("SetAdvisorEnabledResolver", fn != nil) +} + +func (r *Recorder) SetAdvisorRolesResolver(fn func() map[string]config.ModelRole) { + r.record("SetAdvisorRolesResolver", fn != nil) +} + +func (r *Recorder) SetRepeatFailureEnabledResolver(fn func() bool) { + r.record("SetRepeatFailureEnabledResolver", fn != nil) +} +func (r *Recorder) SetRepoAutoMergeEnabledResolver(fn func(repo string) bool) { + r.record("SetRepoAutoMergeEnabledResolver", fn != nil) +} +func (r *Recorder) SetKickLogSourceResolver(fn func() string) { + r.record("SetKickLogSourceResolver", fn != nil) +} +func (r *Recorder) SetInferenceCallbacks(setRoute func(agentName, backend, model string), clearRoute func(agentName string)) { + r.record("SetInferenceCallbacks", setRoute != nil, clearRoute != nil) +} +func (r *Recorder) SetSandboxConfig(cfg config.AgentSandboxConfig) { r.record("SetSandboxConfig", cfg) } +func (r *Recorder) SetSandboxMutationBoundary(boundary effects.Boundary) { + r.record("SetSandboxMutationBoundary", boundary != nil) +} +func (r *Recorder) SetDeadSessionRecoveryOwner(ownedElsewhere bool) { + r.record("SetDeadSessionRecoveryOwner", ownedElsewhere) +} +func (r *Recorder) SetSpanStarter(fn crux.SpanStarter) { r.record("SetSpanStarter", fn != nil) } + +func (r *Recorder) SetKickOutcomeObserver(fn func(agentName string, outcome crux.KickOutcome)) { + r.record("SetKickOutcomeObserver", fn != nil) + r.mu.Lock() + r.KickOutcomeObserver = fn + r.mu.Unlock() +} + +func (r *Recorder) SetPauseTransitionObserver(fn func(crux.PauseTransitionEvent)) { + r.record("SetPauseTransitionObserver", fn != nil) + r.mu.Lock() + r.PauseObserver = fn + r.mu.Unlock() +} + +func (r *Recorder) SetRelaunchObserver(fn func(agentName string)) { + r.record("SetRelaunchObserver", fn != nil) + r.mu.Lock() + r.RelaunchObserver = fn + r.mu.Unlock() +} + +func (r *Recorder) SetPersistPauseCallback(fn func(name string, paused bool)) { + r.record("SetPersistPauseCallback", fn != nil) + r.mu.Lock() + r.PersistPause = fn + r.mu.Unlock() +} + +func (r *Recorder) SetRecordPromptCallback(fn func(agent, trigger, prompt string)) { + r.record("SetRecordPromptCallback", fn != nil) + r.mu.Lock() + r.RecordPrompt = fn + r.mu.Unlock() +} + +func (r *Recorder) SetSandboxAuditCallback(fn func(agent, action, detail string)) { + r.record("SetSandboxAuditCallback", fn != nil) + r.mu.Lock() + r.SandboxAudit = fn + r.mu.Unlock() +} + +func (r *Recorder) SetAuditSink(sink crux.AuditSink) { + r.record("SetAuditSink", sink != nil) + r.mu.Lock() + r.Audit = sink + r.mu.Unlock() +} + +func (r *Recorder) SetConditionReporter(rep crux.ConditionReporter) { + r.record("SetConditionReporter", rep != nil) + r.mu.Lock() + r.Conditions = rep + r.mu.Unlock() +} + +// Observers returns the observer callbacks installed through crux.Host, for +// a test to fire them. +func (r *Recorder) Observers() (kickOutcome func(string, crux.KickOutcome), pause func(crux.PauseTransitionEvent), relaunch func(string), persist func(string, bool), prompt func(string, string, string), sandbox func(string, string, string), audit crux.AuditSink, conditions crux.ConditionReporter) { + r.mu.Lock() + defer r.mu.Unlock() + return r.KickOutcomeObserver, r.PauseObserver, r.RelaunchObserver, r.PersistPause, r.RecordPrompt, r.SandboxAudit, r.Audit, r.Conditions +} + +// ---- watchdog.Source ---- + +// WatchdogSource returns the recorder as the runtime half of a watchdog +// fleet, the way *agent.Manager does. +func (r *Recorder) WatchdogSource() watchdog.Source { + r.record("WatchdogSource") + return watchdogSource{r} +} + +type watchdogSource struct{ r *Recorder } + +func (s watchdogSource) AgentNames() []string { + s.r.record("AgentNames") + return append([]string(nil), s.r.WatchdogNames...) +} + +func (s watchdogSource) Observe(name string) (watchdog.Observation, error) { + s.r.record("Observe", name) + s.r.mu.Lock() + defer s.r.mu.Unlock() + o, ok := s.r.Observations[name] + if !ok { + return watchdog.Observation{}, fmt.Errorf("agent %s not found", name) + } + return o, nil +} + +func (s watchdogSource) LastProduction(name string) (time.Time, bool) { + s.r.record("LastProduction", name) + s.r.mu.Lock() + defer s.r.mu.Unlock() + t, ok := s.r.Production[name] + return t, ok +} + +func (s watchdogSource) SetConditions(name string, conds []crux.Condition) { + s.r.record("SetConditions", name, append([]crux.Condition(nil), conds...)) + s.r.mu.Lock() + s.r.SetConds[name] = append([]crux.Condition(nil), conds...) + s.r.mu.Unlock() +} + +// ErrRecorder is a convenient generic failure for tests that set Err. +var ErrRecorder = errors.New("recorder: injected failure") + +var ( + _ crux.Runtime = (*Recorder)(nil) + _ crux.Host = (*Recorder)(nil) +) diff --git a/src/crux/cruxtest/recorder_test.go b/src/crux/cruxtest/recorder_test.go new file mode 100644 index 0000000000..8d3e9f1954 --- /dev/null +++ b/src/crux/cruxtest/recorder_test.go @@ -0,0 +1,329 @@ +package cruxtest + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/watchdog" +) + +// TestRecorderCoversRuntime calls every crux.Runtime and crux.Host method by +// reflection with zero arguments and checks each one was recorded under its +// own name, so a new interface method cannot ship without the fake growing +// with it (the Background loops take a cancelled context so they return). +func TestRecorderCoversRuntime(t *testing.T) { + r := NewRecorder() + r.Add("alpha", "a1", crux.StateRunning) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + for _, iface := range []reflect.Type{ + reflect.TypeOf((*crux.Runtime)(nil)).Elem(), + reflect.TypeOf((*crux.Host)(nil)).Elem(), + } { + for i := 0; i < iface.NumMethod(); i++ { + m := iface.Method(i) + r.Reset() + fn := reflect.ValueOf(r).MethodByName(m.Name) + args := make([]reflect.Value, m.Type.NumIn()) + for j := range args { + in := m.Type.In(j) + if in == reflect.TypeOf((*context.Context)(nil)).Elem() { + args[j] = reflect.ValueOf(ctx) + } else { + args[j] = reflect.Zero(in) + } + } + fn.Call(args) + if _, ok := r.Last(m.Name); !ok { + t.Errorf("%s not recorded; got %v", m.Name, r.Methods()) + } + } + } +} + +func TestRecorderState(t *testing.T) { + r := NewRecorder() + p := r.Add("alpha", "a1", crux.StateStopped) + p.RestartCount = 3 + r.Pane["alpha"] = []string{"a", "b", "c"} + r.Buffer["alpha"] = []string{"x", "y"} + r.FullLog["alpha"] = "log" + r.Header["alpha"] = "hdr" + ctx := context.Background() + + if err := r.Start(ctx, "alpha"); err != nil || r.Procs["alpha"].State != crux.StateRunning { + t.Fatalf("Start: %v %v", err, r.Procs["alpha"].State) + } + if err := r.Start(ctx, "nope"); err == nil { + t.Fatal("Start unknown agent should fail") + } + if err := r.Stop("alpha"); err != nil || r.Procs["alpha"].State != crux.StateStopped { + t.Fatal("Stop") + } + if err := r.PauseBy("alpha", "t", "r", "me"); err != nil || !r.IsPaused("alpha") || r.Procs["alpha"].PausedBy != "me" { + t.Fatal("PauseBy") + } + if err := r.PauseByCause("alpha", "t", "r", "me", crux.PauseCausation{}); err != nil { + t.Fatal("PauseByCause") + } + if err := r.Resume(ctx, "alpha", "t", "r"); err != nil || r.IsPaused("alpha") { + t.Fatal("Resume") + } + if r.IsPaused("nope") { + t.Fatal("unknown agent is not paused") + } + if err := r.ResetRestartCount("alpha"); err != nil || r.Procs["alpha"].RestartCount != 0 { + t.Fatal("ResetRestartCount") + } + got, err := r.GetStatus("alpha") + if err != nil || got.ID != "a1" || len(got.PaneLines(2)) != 2 { + t.Fatalf("GetStatus: %v %+v", err, got) + } + if _, err := r.GetStatus("nope"); err == nil { + t.Fatal("GetStatus unknown") + } + if _, err := r.GetStatusFast("nope"); err == nil { + t.Fatal("GetStatusFast unknown") + } + if fast, err := r.GetStatusFast("alpha"); err != nil || fast.Name != "alpha" { + t.Fatal("GetStatusFast") + } + if all := r.AllStatuses(); len(all) != 1 || all["alpha"] == p { + t.Fatal("AllStatuses must copy") + } + if r.ResolveAgent("a1") != "alpha" || r.ResolveAgent("alpha") != "alpha" || r.ResolveAgent("zz") != "zz" { + t.Fatal("ResolveAgent") + } + if out, err := r.GetOutput("alpha", 2); err != nil || len(out) != 2 || out[0] != "b" { + t.Fatalf("GetOutput: %v %v", err, out) + } + if out, err := r.GetBufferOutput("alpha", 0); err != nil || len(out) != 2 { + t.Fatal("GetBufferOutput") + } + if _, err := r.GetOutput("nope", 1); err == nil { + t.Fatal("GetOutput unknown") + } + if _, err := r.GetBufferOutput("nope", 1); err == nil { + t.Fatal("GetBufferOutput unknown") + } + if l, err := r.CaptureFullLog("alpha"); err != nil || l != "log" || r.FullLogHeader("alpha") != "hdr" { + t.Fatal("CaptureFullLog") + } + if _, err := r.CaptureFullLog("nope"); err == nil { + t.Fatal("CaptureFullLog unknown") + } + + // Kicker. + r.KickStarted = true + if started, err := r.SendKickAsync("alpha", "m"); err != nil || !started { + t.Fatal("SendKickAsync") + } + if _, err := r.SendKickAsync("nope", "m"); err == nil { + t.Fatal("SendKickAsync unknown") + } + r.KickErr = errors.New("busy") + if err := r.SendKick("alpha", "m"); err == nil { + t.Fatal("SendKick KickErr") + } + if _, err := r.SendKickAsync("alpha", "m"); err == nil { + t.Fatal("SendKickAsync KickErr") + } + if err := r.ValidateKick("alpha"); err == nil { + t.Fatal("ValidateKick KickErr") + } + r.KickErr = nil + if err := r.SendKick("alpha", "m"); err != nil || r.ValidateKick("alpha") != nil { + t.Fatal("SendKick ok") + } + r.Dispatch["alpha"] = crux.KickDispatch{} + if _, ok := r.KickDispatchState("alpha"); !ok { + t.Fatal("KickDispatchState") + } + r.KickLogs["alpha"] = []crux.KickLogInfo{{ID: "k1"}} + r.KickLogContent["alpha"] = map[string]string{"k1": "body"} + if logs, err := r.ListKickLogs("alpha"); err != nil || len(logs) != 1 { + t.Fatal("ListKickLogs") + } + if _, err := r.ListKickLogs("nope"); err == nil { + t.Fatal("ListKickLogs unknown") + } + if body, err := r.ReadKickLog("alpha", "k1"); err != nil || body != "body" { + t.Fatal("ReadKickLog") + } + if _, err := r.ReadKickLog("alpha", "k2"); err == nil { + t.Fatal("ReadKickLog missing id") + } + if _, err := r.ReadKickLog("nope", "k1"); err == nil { + t.Fatal("ReadKickLog unknown agent") + } + var kicks []string + r.SetKickObserver(func(a, e, d string) { kicks = append(kicks, a+e+d) }) + r.EmitKick("alpha", "sent", "ok") + if len(kicks) != 1 || kicks[0] != "alphasentok" { + t.Fatal("EmitKick") + } + r.SetKickObserver(nil) + r.EmitKick("alpha", "sent", "ok") + + // Definitions. + r.AddAgent("beta", r.Procs["alpha"].Config) + r.AddAgent("beta", r.Procs["alpha"].Config) + if len(r.Procs) != 2 { + t.Fatal("AddAgent") + } + cfg := r.Procs["beta"].Config + cfg.Model = "sonnet" + if err := r.UpdateConfig("beta", cfg); err != nil || r.Procs["beta"].Config.Model != "sonnet" { + t.Fatal("UpdateConfig") + } + if err := r.UpdateConfig("nope", cfg); err == nil { + t.Fatal("UpdateConfig unknown") + } + r.RemoveAgent("beta") + if len(r.Procs) != 1 { + t.Fatal("RemoveAgent") + } + r.Reconciled = []string{"alpha"} + if got := r.ReconcileAgents(map[string]config.AgentConfig{"b": {}, "a": {}}); len(got) != 1 { + t.Fatal("ReconcileAgents") + } + if c, _ := r.Last("ReconcileAgents"); !reflect.DeepEqual(c.Args[0], []string{"a", "b"}) { + t.Fatalf("ReconcileAgents args %v", c.Args) + } + if err := r.SetModelOverride("alpha", "m"); err != nil || r.Procs["alpha"].ModelOverride != "m" { + t.Fatal("SetModelOverride") + } + if err := r.SetBackendOverride("alpha", "b"); err != nil || r.Procs["alpha"].BackendOverride != "b" { + t.Fatal("SetBackendOverride") + } + if r.SetModelOverride("nope", "m") == nil || r.SetBackendOverride("nope", "b") == nil { + t.Fatal("overrides unknown") + } + if r.PinCLI("alpha", "1") != nil || r.PinModel("alpha", "m") != nil || r.UnpinCLI("alpha") != nil || r.UnpinModel("alpha") != nil { + t.Fatal("pins") + } + if r.PinCLI("nope", "1") == nil || r.PinModel("nope", "m") == nil || r.UnpinCLI("nope") == nil || r.UnpinModel("nope") == nil { + t.Fatal("pins unknown") + } + + // Fleet. + r.BreakerPaused = []string{"alpha"} + r.Resumed = []string{"alpha"} + if got := r.EngageBreaker(); len(got) != 1 || !r.BreakerEngaged { + t.Fatal("EngageBreaker") + } + if eng, paused := r.BreakerState(); !eng || len(paused) != 1 { + t.Fatal("BreakerState") + } + if got := r.ReleaseBreaker(ctx); len(got) != 1 || r.BreakerEngaged { + t.Fatal("ReleaseBreaker") + } + r.SetACMMLevel(4) + if r.GetACMMLevel() != 4 { + t.Fatal("ACMM") + } + r.RestoreBreaker(true, []string{"x"}) + if eng, paused := r.BreakerState(); !eng || paused[0] != "x" { + t.Fatal("RestoreBreaker") + } + + // Credentials. + r.SetCopilotToken("t1") + if r.CopilotToken() != "t1" { + t.Fatal("SetCopilotToken") + } + if err := r.ActivateCopilotToken("t2"); err != nil || r.CopilotToken() != "t2" { + t.Fatal("ActivateCopilotToken") + } + if r.SubmitLoginCode("nope", "c") == nil || r.SubmitLoginCode("alpha", "c") != nil { + t.Fatal("SubmitLoginCode") + } + + // Resumer. + r.Handles["alpha"] = crux.ResumeHandle{} + r.Sessions["alpha"] = "s1" + r.Missing["alpha"] = true + if _, ok := r.ResumeHandle("alpha"); !ok { + t.Fatal("ResumeHandle") + } + if id, ok := r.SessionID("alpha"); !ok || id != "s1" || !r.SessionMissing("alpha") { + t.Fatal("SessionID") + } + if r.SendResumeKick("alpha", "m", "s1") != nil || r.SendResumeKick("nope", "m", "s1") == nil { + t.Fatal("SendResumeKick") + } + r.ResumeErr = crux.ErrResumeBusy + if !errors.Is(r.SendResumeKick("alpha", "m", "s1"), crux.ErrResumeBusy) { + t.Fatal("ResumeErr") + } + + // Supervisor odds and ends. + r.BackendAuth["claude"] = [2]bool{true, true} + if a, k := r.BackendAuthAvailable("claude"); !a || !k { + t.Fatal("BackendAuthAvailable") + } + if r.RefreshAgentTokenFor(ctx, "nope") == nil || r.RefreshAgentTokenFor(ctx, "alpha") != nil { + t.Fatal("RefreshAgentTokenFor") + } + + // Generic Err wins over lookups. + r.Err = ErrRecorder + if !errors.Is(r.Start(ctx, "alpha"), ErrRecorder) || !errors.Is(r.ActivateCopilotToken("x"), ErrRecorder) { + t.Fatal("Err") + } + r.Err = nil + + // Host observers are kept and reported. + r.SetKickOutcomeObserver(func(string, crux.KickOutcome) {}) + r.SetPauseTransitionObserver(func(crux.PauseTransitionEvent) {}) + r.SetRelaunchObserver(func(string) {}) + r.SetPersistPauseCallback(func(string, bool) {}) + r.SetRecordPromptCallback(func(string, string, string) {}) + r.SetSandboxAuditCallback(func(string, string, string) {}) + ko, po, ro, pp, rp, sa, _, _ := r.Observers() + if ko == nil || po == nil || ro == nil || pp == nil || rp == nil || sa == nil { + t.Fatal("Observers") + } + + // Background loops return when ctx ends and record their start. + bctx, cancel := context.WithCancel(ctx) + cancel() + r.StartAgentTokenRefresh(bctx) + if got := r.Background(); len(got) != 1 || got[0] != "StartAgentTokenRefresh" { + t.Fatal("background") + } + + // Watchdog source. + r.WatchdogNames = []string{"alpha"} + r.Observations["alpha"] = watchdog.Observation{} + r.Production["alpha"] = time.Unix(1, 0) + src := r.WatchdogSource() + if len(src.AgentNames()) != 1 { + t.Fatal("AgentNames") + } + if _, err := src.Observe("alpha"); err != nil { + t.Fatal("Observe") + } + if _, err := src.Observe("nope"); err == nil { + t.Fatal("Observe unknown") + } + if ts, ok := src.LastProduction("alpha"); !ok || !ts.Equal(time.Unix(1, 0)) { + t.Fatal("LastProduction") + } + src.SetConditions("alpha", []crux.Condition{{Type: "Ready"}}) + if len(r.SetConds["alpha"]) != 1 { + t.Fatal("SetConditions") + } + + if c, ok := r.Last("SetConditions"); !ok || c.String() == "" { + t.Fatal("Last/String") + } + if _, ok := r.Last("Never"); ok { + t.Fatal("Last missing") + } +} diff --git a/src/crux/cruxtest/runtime.go b/src/crux/cruxtest/runtime.go new file mode 100644 index 0000000000..c37c67debd --- /dev/null +++ b/src/crux/cruxtest/runtime.go @@ -0,0 +1,83 @@ +// Package cruxtest builds real Crux runtimes and process output buffers for +// consumer tests without coupling those tests to crux/agent. Constructors do +// not start agents; calling lifecycle methods still has real runtime effects. +package cruxtest + +import ( + "log/slog" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/pushbroker" + "github.com/hivecommons/hive/crux/sandbox" +) + +// Runtime adds fixture controls to the production runtime surface. Consumer +// tests can configure kick logs, inject dispatch results and observe routing +// callbacks without obtaining the concrete manager. Keep these controls here, +// rather than widening crux.Runtime with test-only setup methods. +type Runtime interface { + crux.Runtime + // Host is the in-process wiring surface cmd/hive installs after + // construction; consumer tests that exercise a boot path need it too. + crux.Host + crux.ActivityObserver + RecordKickDispatchForTest(crux.KickDispatch) + SetKickLogDir(dir string) + SetKickLogSourceResolver(func() string) + KickLogSource() string + SetInferenceCallbacks(setRoute func(name, backend, model string), clearRoute func(name string)) + SetSandboxConfig(config.AgentSandboxConfig) + // SetSandboxLauncher substitutes the OCI launcher so kick-path tests can + // run a sandboxed agent without a container runtime. + SetSandboxLauncher(sandbox.Launcher) + + // Boot tests read back the sandbox wiring they installed and inject + // sandbox audit rows. + SandboxGitHubWiring() (crux.PRCreator, pushbroker.TokenMinter) + SandboxAuditForTest(agent, action, detail string) +} + +// Option configures a test runtime's construction. +type Option func(*options) + +type options struct { + uidMapPath *string +} + +// WithUIDMapPath selects the UID map used by the fixture. A path under the +// test's temporary directory keeps host UID assignments out of the fixture. +func WithUIDMapPath(path string) Option { + return func(opts *options) { opts.uidMapPath = &path } +} + +// NewRuntime constructs the same stopped agents as the production manager. +// Project facts and logger are supplied by the caller, preserving the test's +// ACMM gates and logging. With no options, production constructor defaults apply. +func NewRuntime(defs map[string]config.AgentConfig, logger *slog.Logger, project crux.ProjectContext, opts ...Option) Runtime { + var cfg options + for _, opt := range opts { + if opt != nil { + opt(&cfg) + } + } + var managerOpts []agent.ManagerOption + if cfg.uidMapPath != nil { + managerOpts = append(managerOpts, agent.WithUIDMapPath(*cfg.uidMapPath)) + } + return agent.NewManagerWithOptions(defs, logger, project, managerOpts...) +} + +// NewRingBuffer constructs output for a crux.Process fixture. It is +// crux.NewRingBuffer; it stays here so existing fixtures keep one import. +func NewRingBuffer(capacity int) *crux.RingBuffer { + return crux.NewRingBuffer(capacity) +} + +// NewUIDMap constructs a persisted UID-map fixture, including the runtime's +// default base and proxy UIDs. Tests can save it under a temporary directory +// and pass that path to WithUIDMapPath. +func NewUIDMap() *agent.UIDMap { + return agent.NewUIDMap() +} diff --git a/src/crux/cruxtest/runtime_test.go b/src/crux/cruxtest/runtime_test.go new file mode 100644 index 0000000000..9fb768a1d8 --- /dev/null +++ b/src/crux/cruxtest/runtime_test.go @@ -0,0 +1,62 @@ +package cruxtest_test + +import ( + "io" + "log/slog" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/cruxtest" +) + +func TestNewRuntimeUsesSuppliedUIDMap(t *testing.T) { + for _, tc := range []struct { + name string + uid int + }{ + {name: "missing map"}, + {name: "isolated agent", uid: 2345}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "uid-map.json") + if tc.uid != 0 { + uids := cruxtest.NewUIDMap() + uids.Agents["fixture"] = tc.uid + if err := uids.Save(path); err != nil { + t.Fatal(err) + } + } + rt := cruxtest.NewRuntime(map[string]config.AgentConfig{ + "fixture": {ID: "fixture-id", Enabled: true, Backend: "claude", Model: "sonnet"}, + }, slog.New(slog.NewTextHandler(io.Discard, nil)), crux.ProjectContext{}, cruxtest.WithUIDMapPath(path)) + proc, err := rt.GetStatusFast("fixture") + if err != nil { + t.Fatal(err) + } + if proc.UID != tc.uid || proc.State != crux.StateStopped || proc.Config.Model != "sonnet" { + t.Fatalf("UID/state/model = %d/%s/%s, want %d/%s/sonnet", proc.UID, proc.State, proc.Config.Model, tc.uid, crux.StateStopped) + } + if got := rt.ResolveAgent("fixture-id"); got != "fixture" { + t.Fatalf("ResolveAgent(fixture-id) = %q, want fixture", got) + } + }) + } +} + +func TestFixtureControlsShareRuntimeState(t *testing.T) { + fixture := cruxtest.NewRuntime(nil, slog.New(slog.NewTextHandler(io.Discard, nil)), crux.ProjectContext{}, + cruxtest.WithUIDMapPath(filepath.Join(t.TempDir(), "uid-map.json"))) + var rt crux.Runtime = fixture + controls := rt.(cruxtest.Runtime) + queuedAt := time.Date(2026, 10, 24, 9, 0, 0, 0, time.UTC) + controls.RecordKickDispatchForTest(crux.KickDispatch{ + Agent: "fixture", Phase: crux.KickPhaseDelivered, QueuedAt: queuedAt, + }) + got, ok := rt.KickDispatchState("fixture") + if !ok || got.Phase != crux.KickPhaseDelivered || !got.QueuedAt.Equal(queuedAt) { + t.Fatalf("dispatch through production surface = %+v/%v, want injected delivery", got, ok) + } +} diff --git a/src/crux/defs/definition.go b/src/crux/defs/definition.go new file mode 100644 index 0000000000..4c4cdd9db0 --- /dev/null +++ b/src/crux/defs/definition.go @@ -0,0 +1,147 @@ +package defs + +import ( + "fmt" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/hivecommons/hive/crux/config" +) + +// Definition is the portable YAML format for a whole agent: the dashboard's +// import/export schema and the document pkg/defsrc parses in a Hive. Only the +// fields listed here, and applied by Merge, can be set by a definition; +// anything absent is, by construction, not settable from a file. The struct +// is kept field-for-field with pkg/defsrc.AgentDefinition so a document +// written for one loads in the other. +type Definition struct { + APIVersion string `yaml:"apiVersion" json:"apiVersion"` + Kind string `yaml:"kind" json:"kind"` + Metadata Metadata `yaml:"metadata" json:"metadata"` + Spec Spec `yaml:"spec" json:"spec"` +} + +// Metadata names and presents the agent. +type Metadata struct { + Name string `yaml:"name" json:"name"` + DisplayName string `yaml:"displayName,omitempty" json:"displayName,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Emoji string `yaml:"emoji,omitempty" json:"emoji,omitempty"` + Color string `yaml:"color,omitempty" json:"color,omitempty"` + SpecVersion int `yaml:"specVersion,omitempty" json:"specVersion,omitempty"` +} + +// Spec is the agent's behaviour. Cadences is accepted so a Hive export parses, +// but it is governor scheduling (pkg/config.Cadence) that no bare crux runs, +// so Merge ignores it — as pkg/defsrc's merge does. +type Spec struct { + Backend string `yaml:"backend,omitempty" json:"backend,omitempty"` + Model string `yaml:"model,omitempty" json:"model,omitempty"` + Role string `yaml:"role,omitempty" json:"role,omitempty"` + Mode string `yaml:"mode,omitempty" json:"mode,omitempty"` + SortOrder int `yaml:"sortOrder,omitempty" json:"sortOrder,omitempty"` + BeadRole string `yaml:"beadRole,omitempty" json:"beadRole,omitempty"` + StaleTimeout int `yaml:"staleTimeout,omitempty" json:"staleTimeout,omitempty"` + RestartStrategy string `yaml:"restartStrategy,omitempty" json:"restartStrategy,omitempty"` + ClearOnKick bool `yaml:"clearOnKick,omitempty" json:"clearOnKick,omitempty"` + IncludeRepos bool `yaml:"includeRepos,omitempty" json:"includeRepos,omitempty"` + LaneKeywords []string `yaml:"laneKeywords,omitempty" json:"laneKeywords,omitempty"` + DetectKeywords []string `yaml:"detectKeywords,omitempty" json:"detectKeywords,omitempty"` + Aliases []string `yaml:"aliases,omitempty" json:"aliases,omitempty"` + Cadences map[string]string `yaml:"cadences,omitempty" json:"cadences,omitempty"` + PromptTemplate string `yaml:"promptTemplate,omitempty" json:"promptTemplate,omitempty"` + Channels []config.ChannelConfig `yaml:"channels,omitempty" json:"channels,omitempty"` + Tools *config.ToolsConfig `yaml:"tools,omitempty" json:"tools,omitempty"` + Connections []config.ConnectionConfig `yaml:"connections,omitempty" json:"connections,omitempty"` +} + +// ParseDefinition parses and structurally validates a portable definition. +func ParseDefinition(data []byte) (*Definition, error) { + var def Definition + if err := yaml.Unmarshal(data, &def); err != nil { + return nil, fmt.Errorf("invalid definition YAML: %w", err) + } + if def.Kind != DefinitionKind { + return nil, fmt.Errorf("expected kind %q, got %q", DefinitionKind, def.Kind) + } + if def.Metadata.Name == "" { + return nil, fmt.Errorf("metadata.name is required") + } + return &def, nil +} + +// Name is the agent name the definition yields: metadata.name lower-cased +// with spaces turned into hyphens, as the dashboard import names it. +func (d *Definition) Name() string { + return strings.ToLower(strings.ReplaceAll(d.Metadata.Name, " ", "-")) +} + +// Merge applies the definition's fields over base and returns the result, +// through the same allow-list as pkg/defsrc.MergeAllowedFields: empty +// strings and nil slices leave base alone; clearOnKick and includeRepos carry +// meaning at their zero value, so the definition's value is authoritative. +func (d *Definition) Merge(base config.AgentConfig) config.AgentConfig { + merged := base + if d.Metadata.DisplayName != "" { + merged.DisplayName = d.Metadata.DisplayName + } + if d.Metadata.Description != "" { + merged.Description = d.Metadata.Description + } + if d.Metadata.Emoji != "" { + merged.Emoji = d.Metadata.Emoji + } + if d.Metadata.Color != "" { + merged.Color = d.Metadata.Color + } + if d.Spec.Backend != "" { + merged.Backend = d.Spec.Backend + } + if d.Spec.Model != "" { + merged.Model = d.Spec.Model + } + if d.Spec.Role != "" { + merged.Role = d.Spec.Role + } + if d.Spec.Mode != "" { + merged.Mode = d.Spec.Mode + } + if d.Spec.SortOrder != 0 { + merged.SortOrder = d.Spec.SortOrder + } + if d.Spec.BeadRole != "" { + merged.BeadRole = d.Spec.BeadRole + } + if d.Spec.StaleTimeout != 0 { + merged.StaleTimeout = d.Spec.StaleTimeout + } + if d.Spec.RestartStrategy != "" { + merged.RestartStrategy = d.Spec.RestartStrategy + } + merged.ClearOnKick = d.Spec.ClearOnKick + includeRepos := d.Spec.IncludeRepos + merged.IncludeRepos = &includeRepos + if len(d.Spec.LaneKeywords) > 0 { + merged.LaneKeywords = d.Spec.LaneKeywords + } + if len(d.Spec.DetectKeywords) > 0 { + merged.DetectKeywords = d.Spec.DetectKeywords + } + if len(d.Spec.Aliases) > 0 { + merged.Aliases = d.Spec.Aliases + } + if len(d.Spec.Channels) > 0 { + merged.Channels = d.Spec.Channels + } + if d.Spec.Tools != nil { + merged.Tools = d.Spec.Tools + } + if len(d.Spec.Connections) > 0 { + merged.Connections = d.Spec.Connections + } + // A portable definition has no clear_on_kick key to detect, so its value + // is explicit by construction: the merge above already wrote it. + merged.MarkExplicit(false, true) + return merged +} diff --git a/src/crux/defs/defs.go b/src/crux/defs/defs.go new file mode 100644 index 0000000000..3ccf9d3067 --- /dev/null +++ b/src/crux/defs/defs.go @@ -0,0 +1,215 @@ +// Package defs is the Crux module's own agent-definition loader: the +// cli.DefinitionLoader the module's `crux` binary (crux/cmd/crux) ships with, +// so a checkout of the module alone can run agents without Hive's config +// loader or pkg/defsrc (hivecommons/hive#14138). +// +// It reads plain YAML from a directory or a file (`--defs`, `$CRUX_DEFS`): +// +// - a portable AgentDefinition (`kind: AgentDefinition`, metadata.name, +// spec.*), the dashboard's import/export schema, merged over the same +// defaults an import gets through the same field allow-list; +// - a bare per-agent overlay: the AgentConfig YAML itself, named by the +// file's base name. +// +// Either way the result is a config.AgentConfig with the per-agent defaults +// (id, role, replicas, enabled, clear_on_kick) applied. Where definitions come +// from in a Hive — hive.yaml, repo-tree definition sources, the dashboard +// overlay — stays with the host binary in the root module (cmd/crux); this +// package knows only files. +package defs + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" +) + +// DefinitionKind is the required `kind` of a portable AgentDefinition +// document; it mirrors the dashboard's export kind so an export round-trips. +const DefinitionKind = "AgentDefinition" + +// Defaults a definition is merged over: the values the dashboard's import +// fills in and cmd/crux's hiveLoader applies. +const ( + // DefaultBackend is the backend a portable definition that names none + // runs on. + DefaultBackend = "copilot" + // DefaultRestartStrategy is the restart strategy of a portable definition + // that names none. + DefaultRestartStrategy = "immediate" + // DefaultReplicas is the replica count of an agent that names none. + DefaultReplicas = 1 + // MaxNameLen bounds an agent name, as the dashboard import does. + MaxNameLen = 64 + // Ext is the only file extension the directory walk reads, matching the + // per-agent overlay loader and `crux run`'s definition-path rule. + Ext = ".yaml" +) + +// namePattern is the dashboard import's agent-name rule: lower-case words +// joined by hyphens or underscores, so a definition cannot name a path. +var namePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) + +// Loader implements cli.DefinitionLoader over files. The zero value is ready +// to use. +type Loader struct{} + +// LoadConfig returns the agents defined at path, which may be a directory of +// definition files or a single file, and the zero ProjectContext: this loader +// knows no org, repos or policies. An empty path yields an empty runtime. +func (Loader) LoadConfig(path string) (map[string]config.AgentConfig, crux.ProjectContext, error) { + if path == "" { + return map[string]config.AgentConfig{}, crux.ProjectContext{}, nil + } + defs, err := Load(path) + if err != nil { + return nil, crux.ProjectContext{}, err + } + return defs, crux.ProjectContext{}, nil +} + +// LoadDefinitionFile reads one definition file; see LoadFile. +func (Loader) LoadDefinitionFile(path string) (string, config.AgentConfig, error) { + return LoadFile(path) +} + +// Load reads every definition at path: the file itself, or each `.yaml` +// directly inside it when path is a directory. Two files defining the same +// agent name are an error rather than a silent last-wins. +func Load(path string) (map[string]config.AgentConfig, error) { + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("reading definitions %s: %w", path, err) + } + defs := map[string]config.AgentConfig{} + if !info.IsDir() { + name, ac, err := LoadFile(path) + if err != nil { + return nil, err + } + defs[name] = ac + return defs, nil + } + entries, err := os.ReadDir(path) + if err != nil { + return nil, fmt.Errorf("reading definitions %s: %w", path, err) + } + files := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), Ext) { + continue + } + files = append(files, filepath.Join(path, entry.Name())) + } + sort.Strings(files) + from := map[string]string{} + for _, file := range files { + name, ac, err := LoadFile(file) + if err != nil { + return nil, err + } + if prev, dup := from[name]; dup { + return nil, fmt.Errorf("definition %s: agent %q is already defined by %s", file, name, prev) + } + from[name] = file + defs[name] = ac + } + return defs, nil +} + +// LoadFile reads one agent definition and returns the agent's name and its +// config with ApplyDefaults applied. A `kind: AgentDefinition` document is +// named by metadata.name; anything else is read as the AgentConfig YAML +// itself and named by the file's base name. +func LoadFile(path string) (string, config.AgentConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", config.AgentConfig{}, fmt.Errorf("reading definition %s: %w", path, err) + } + var name string + var ac config.AgentConfig + if isPortable(data) { + def, err := ParseDefinition(data) + if err != nil { + return "", config.AgentConfig{}, fmt.Errorf("definition %s: %w", path, err) + } + name = def.Name() + ac = def.Merge(config.AgentConfig{ + Backend: DefaultBackend, + RestartStrategy: DefaultRestartStrategy, + Enabled: true, + }) + } else { + name = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + if err := yaml.Unmarshal(data, &ac); err != nil { + return "", config.AgentConfig{}, fmt.Errorf("definition %s: %w", path, err) + } + // The per-agent overlay loader marks its entries managed; keep the + // same shape so a file moved between a Hive and a bare crux behaves + // the same. + ac.Managed = true + } + if err := ValidateName(name); err != nil { + return "", config.AgentConfig{}, fmt.Errorf("definition %s: %w", path, err) + } + ac.SetLoadState(name, path) + ApplyDefaults(name, &ac) + return name, ac, nil +} + +// ApplyDefaults fills the per-agent defaults a hive.yaml load applies to +// every agent: id and role from the name, one replica, enabled and +// clear_on_kick unless the YAML said otherwise. Hive's table of well-known +// agent names (emoji, colour, keywords) is deliberately not here; that is +// presentation the host owns. +func ApplyDefaults(name string, ac *config.AgentConfig) { + if ac.ID == "" { + ac.ID = name + } + if ac.Role == "" { + ac.Role = name + } + if ac.Replicas == 0 { + ac.Replicas = DefaultReplicas + } + if !ac.Enabled && !ac.EnabledExplicitlySet() { + ac.Enabled = true + } + if !ac.ClearOnKickExplicitlySet() { + ac.ClearOnKick = true + } +} + +// ValidateName enforces the dashboard import's agent-name rule. +func ValidateName(name string) error { + if name == "" { + return fmt.Errorf("agent name is empty") + } + if len(name) > MaxNameLen { + return fmt.Errorf("agent name %q is longer than %d characters", name, MaxNameLen) + } + if !namePattern.MatchString(name) { + return fmt.Errorf("agent name %q contains invalid characters", name) + } + return nil +} + +// isPortable reports whether data declares the AgentDefinition kind without +// committing to a full parse. +func isPortable(data []byte) bool { + var probe struct { + Kind string `yaml:"kind"` + } + if err := yaml.Unmarshal(data, &probe); err != nil { + return false + } + return probe.Kind == DefinitionKind +} diff --git a/src/crux/defs/defs_test.go b/src/crux/defs/defs_test.go new file mode 100644 index 0000000000..679eb5fcc4 --- /dev/null +++ b/src/crux/defs/defs_test.go @@ -0,0 +1,260 @@ +package defs + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/config" +) + +func writeFile(t *testing.T, path, content string) string { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +const portableDefinition = `apiVersion: hive.kubestellar.io/v1 +kind: AgentDefinition +metadata: + name: Scout Agent + displayName: Scout + description: looks around + emoji: "x" + color: "#123456" +spec: + backend: claude + model: claude-sonnet-4-6 + role: scout + mode: ADVISORY + sortOrder: 7 + beadRole: worker + staleTimeout: 60 + restartStrategy: never + clearOnKick: true + includeRepos: true + laneKeywords: [a] + detectKeywords: [b] + aliases: [sc] + cadences: + default: "@hourly" + tools: + preset: read-only + connections: + - name: docs + type: mcp +` + +const overlayDefinition = `backend: codex +model: gpt-5 +role: fixer +enabled: false +clear_on_kick: false +` + +func TestLoadFilePortable(t *testing.T) { + path := writeFile(t, filepath.Join(t.TempDir(), "scout.yaml"), portableDefinition) + name, ac, err := LoadFile(path) + if err != nil { + t.Fatal(err) + } + if name != "scout-agent" { + t.Fatalf("name = %q", name) + } + if ac.ID != "scout-agent" || ac.Backend != "claude" || ac.Model != "claude-sonnet-4-6" || ac.Role != "scout" { + t.Fatalf("ac = %+v", ac) + } + if ac.DisplayName != "Scout" || ac.Description != "looks around" || ac.Emoji != "x" || ac.Color != "#123456" { + t.Fatalf("presentation not merged: %+v", ac) + } + if ac.Mode != "ADVISORY" || ac.SortOrder != 7 || ac.BeadRole != "worker" || ac.StaleTimeout != 60 || ac.RestartStrategy != "never" { + t.Fatalf("spec not merged: %+v", ac) + } + if !ac.ClearOnKick || ac.IncludeRepos == nil || !*ac.IncludeRepos { + t.Fatalf("flags: clear_on_kick=%v include_repos=%v", ac.ClearOnKick, ac.IncludeRepos) + } + if len(ac.LaneKeywords) != 1 || len(ac.DetectKeywords) != 1 || len(ac.Aliases) != 1 || ac.Tools == nil || len(ac.Connections) != 1 { + t.Fatalf("lists not merged: %+v", ac) + } + if !ac.Enabled || ac.Replicas != DefaultReplicas || ac.Name() != "scout-agent" || ac.SourceFile() != path { + t.Fatalf("defaults: enabled=%v replicas=%d name=%q source=%q", ac.Enabled, ac.Replicas, ac.Name(), ac.SourceFile()) + } +} + +func TestLoadFilePortableDefaultsAndExplicitClearOnKick(t *testing.T) { + path := writeFile(t, filepath.Join(t.TempDir(), "min.yaml"), "kind: AgentDefinition\nmetadata:\n name: minimal\nspec: {}\n") + name, ac, err := LoadFile(path) + if err != nil { + t.Fatal(err) + } + if name != "minimal" || ac.Backend != DefaultBackend || ac.RestartStrategy != DefaultRestartStrategy || ac.Role != "minimal" { + t.Fatalf("defaults not applied: name=%q ac=%+v", name, ac) + } + // A portable definition's clearOnKick is authoritative even at false: + // the per-agent default must not override it. + if ac.ClearOnKick { + t.Fatalf("clear_on_kick = true, want the definition's false") + } +} + +func TestLoadFileOverlay(t *testing.T) { + path := writeFile(t, filepath.Join(t.TempDir(), "fixer.yaml"), overlayDefinition) + name, ac, err := LoadFile(path) + if err != nil { + t.Fatal(err) + } + if name != "fixer" || ac.ID != "fixer" || ac.Backend != "codex" || ac.Model != "gpt-5" || ac.Role != "fixer" { + t.Fatalf("name=%q ac=%+v", name, ac) + } + if ac.Enabled || ac.ClearOnKick { + t.Fatalf("explicit enabled/clear_on_kick overridden: %+v", ac) + } + if !ac.Managed || ac.Replicas != 1 { + t.Fatalf("overlay shape: managed=%v replicas=%d", ac.Managed, ac.Replicas) + } + + path = writeFile(t, filepath.Join(t.TempDir(), "bare.yaml"), "backend: copilot\n") + _, ac, err = LoadFile(path) + if err != nil { + t.Fatal(err) + } + if !ac.Enabled || !ac.ClearOnKick || ac.Role != "bare" { + t.Fatalf("overlay defaults: %+v", ac) + } +} + +func TestLoadFileErrors(t *testing.T) { + dir := t.TempDir() + if _, _, err := LoadFile(filepath.Join(dir, "missing.yaml")); err == nil { + t.Fatal("missing file") + } + cases := map[string]string{ + "bad-yaml.yaml": "backend: [\n", + "no-name.yaml": "kind: AgentDefinition\nspec: {}\n", + "Bad Name.yaml": "backend: copilot\n", + "bad-kind.yaml": "kind: AgentDefinition\nmetadata:\n name: 'Not/Valid'\n", + "bad-portable.yaml": "kind: AgentDefinition\nmetadata: [\n", + } + for file, content := range cases { + path := writeFile(t, filepath.Join(dir, file), content) + if _, _, err := LoadFile(path); err == nil { + t.Errorf("%s: expected an error", file) + } + } +} + +func TestLoadDirectoryAndFile(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "scout.yaml"), portableDefinition) + writeFile(t, filepath.Join(dir, "fixer.yaml"), overlayDefinition) + writeFile(t, filepath.Join(dir, "notes.txt"), "ignored") + writeFile(t, filepath.Join(dir, "sub", "deep.yaml"), "backend: copilot\n") + + defs, err := Load(dir) + if err != nil { + t.Fatal(err) + } + if len(defs) != 2 { + t.Fatalf("defs = %v", keys(defs)) + } + if _, ok := defs["scout-agent"]; !ok { + t.Fatalf("portable definition missing: %v", keys(defs)) + } + if _, ok := defs["fixer"]; !ok { + t.Fatalf("overlay missing: %v", keys(defs)) + } + + one, err := Load(filepath.Join(dir, "fixer.yaml")) + if err != nil { + t.Fatal(err) + } + if len(one) != 1 || one["fixer"].Model != "gpt-5" { + t.Fatalf("single file: %v", one) + } + + if _, err := Load(filepath.Join(dir, "nope")); err == nil { + t.Fatal("missing path") + } + writeFile(t, filepath.Join(dir, "broken.yaml"), "backend: [\n") + if _, err := Load(dir); err == nil { + t.Fatal("broken file in dir") + } +} + +func TestLoadRejectsDuplicateNames(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "a.yaml"), "kind: AgentDefinition\nmetadata:\n name: same\n") + writeFile(t, filepath.Join(dir, "b.yaml"), "kind: AgentDefinition\nmetadata:\n name: same\n") + _, err := Load(dir) + if err == nil || !strings.Contains(err.Error(), "already defined") { + t.Fatalf("err = %v", err) + } +} + +func TestLoaderInterface(t *testing.T) { + var l Loader + defs, project, err := l.LoadConfig("") + if err != nil || len(defs) != 0 || project.Org != "" { + t.Fatalf("empty: %v %v %v", defs, project, err) + } + if _, _, err := l.LoadConfig(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Fatal("missing path") + } + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "fixer.yaml"), overlayDefinition) + defs, _, err = l.LoadConfig(dir) + if err != nil || len(defs) != 1 { + t.Fatalf("dir: %v %v", defs, err) + } + name, _, err := l.LoadDefinitionFile(filepath.Join(dir, "fixer.yaml")) + if err != nil || name != "fixer" { + t.Fatalf("file: %q %v", name, err) + } +} + +func TestApplyDefaultsLeavesSetValues(t *testing.T) { + ac := config.AgentConfig{ID: "x", Role: "r", Replicas: 3} + ac.MarkExplicit(true, true) + ApplyDefaults("name", &ac) + if ac.ID != "x" || ac.Role != "r" || ac.Replicas != 3 || ac.Enabled || ac.ClearOnKick { + t.Fatalf("ac = %+v", ac) + } +} + +func TestValidateName(t *testing.T) { + if err := ValidateName(""); err == nil { + t.Fatal("empty") + } + if err := ValidateName(strings.Repeat("a", MaxNameLen+1)); err == nil { + t.Fatal("too long") + } + if err := ValidateName("../etc"); err == nil { + t.Fatal("path") + } + if err := ValidateName("ok-name_1"); err != nil { + t.Fatal(err) + } +} + +func TestParseDefinitionErrors(t *testing.T) { + if _, err := ParseDefinition([]byte("kind: Other\nmetadata:\n name: x\n")); err == nil { + t.Fatal("wrong kind") + } + if _, err := ParseDefinition([]byte(":\n - [")); err == nil { + t.Fatal("bad yaml") + } +} + +func keys(m map[string]config.AgentConfig) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/src/crux/effects/effects.go b/src/crux/effects/effects.go new file mode 100644 index 0000000000..d9d5d8005a --- /dev/null +++ b/src/crux/effects/effects.go @@ -0,0 +1,331 @@ +// Package effects declares the small external-mutation boundary seam shared by +// callers that must not import the convergence implementation. +package effects + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "sort" + "strings" + "sync" + "time" +) + +const ( + KindPullRequestCreate = "pull_request_create" + KindIssueCreate = "issue_create" + KindIssueComment = "issue_comment" + KindPullRequestMerge = "pull_request_merge" + KindLabelMutation = "label_mutation" + KindReviewSubmit = "review_submit" + KindReviewRequest = "review_request" + KindBranchPush = "branch_push" + KindBranchUpdate = "branch_update" + KindSubIssueLink = "sub_issue_link" + KindDependencyLink = "dependency_link" + KindStatusCreate = "status_create" +) + +var ErrDenied = errors.New("mutation boundary denied") + +// ErrNotApplied marks an effect error as a DEFINITE refusal: the external +// system answered and the effect certainly did not take place, so a +// journaling boundary may record it NotApplied (retry authorized) rather than +// Unknown (reconciliation required). Wrap with NotApplied. +var ErrNotApplied = errors.New("external effect definitely not applied") + +// ErrNeedsReconciliation reports that the boundary refused to run an effect +// because an earlier attempt at the same logical operation is unresolved and +// must be reconciled (see Reconcile) before any retry. +var ErrNeedsReconciliation = errors.New("external effect needs reconciliation before retry") + +// NotApplied wraps err as a definite refusal (ErrNotApplied) while keeping +// err's own message and chain, so callers still see the original error. +func NotApplied(err error) error { + if err == nil { + return nil + } + return notAppliedError{err: err} +} + +type notAppliedError struct{ err error } + +func (e notAppliedError) Error() string { return e.err.Error() } + +func (e notAppliedError) Unwrap() []error { return []error{e.err, ErrNotApplied} } + +// ExternalState is one authoritative observation of whether a claimed effect +// exists externally, used to resolve an operation left unresolved. +type ExternalState struct { + Applied bool + Provenance string +} + +// Reconciler is implemented by boundaries that journal effects and can +// resolve an unresolved operation from authoritative external state. +type Reconciler interface { + Reconcile(ctx context.Context, claim Claim, state ExternalState) error +} + +// Reconcile resolves claim's operation on boundary from state. Boundaries that +// do not journal (nil, NoopBoundary) have nothing to resolve. +func Reconcile(ctx context.Context, boundary Boundary, claim Claim, state ExternalState) error { + r, ok := boundary.(Reconciler) + if !ok { + return nil + } + return r.Reconcile(ctx, claim, state) +} + +// Expirer is implemented by boundaries that journal effects and can give up +// on an operation left unresolved (Planned/Unknown) for longer than a TTL +// when no authoritative observation is obtainable. +type Expirer interface { + ExpireUnresolved(ctx context.Context, claim Claim, ttl time.Duration) (bool, error) +} + +// ExpireUnresolved resolves claim's operation to not-applied when the +// boundary journals it as unresolved and its last transition is older than +// ttl, authorizing a retry. It reports whether anything was expired. +// Boundaries that do not journal (nil, NoopBoundary) have nothing to expire. +// +// This is the escape hatch for an operation nothing can reconcile — the +// external lookup itself keeps failing — so an unresolved entry can never +// fence every identical request forever. Callers run their own duplicate +// check (an open-issue scan, an open-PR-by-head lookup) before any retry, so +// expiry trades a bounded chance of one duplicate for a guaranteed end to a +// deadlock. +func ExpireUnresolved(ctx context.Context, boundary Boundary, claim Claim, ttl time.Duration) (bool, error) { + e, ok := boundary.(Expirer) + if !ok { + return false, nil + } + return e.ExpireUnresolved(ctx, claim, ttl) +} + +// DefaultUnresolvedTTL bounds how long an operation may stay unresolved when +// its external state cannot be observed before ExecuteReconciling expires it +// so a retry may run. It outlasts a single GitHub incident while keeping a +// stuck entry from fencing identical requests for days. +const DefaultUnresolvedTTL = time.Hour + +// ExecuteReconciling runs effect through boundary like Execute, but does not +// leave an unresolved operation fencing retries forever. When the boundary +// refuses with ErrNeedsReconciliation (an earlier attempt at the SAME logical +// operation was journaled Unknown or never acknowledged), observe reads the +// authoritative external state, the operation is resolved from it and the +// same logical operation is replayed once: an applied observation replays as +// idempotent success without running effect, a not-applied one runs it again. +// When observe is nil or fails, the operation is expired to not-applied once +// its last transition is older than ttl (DefaultUnresolvedTTL when zero) and +// effect runs again; a younger entry keeps the refusal. +func ExecuteReconciling(ctx context.Context, boundary Boundary, claim Claim, observe func(context.Context) (ExternalState, error), ttl time.Duration, effect func(context.Context) (Result, error)) (Result, error) { + out, err := Execute(ctx, boundary, claim, effect) + if !errors.Is(err, ErrNeedsReconciliation) { + return out, err + } + journalErr := err + lookupErr := errors.New("no external observation for " + claim.Kind) + if observe != nil { + var state ExternalState + if state, lookupErr = observe(ctx); lookupErr == nil { + if recErr := Reconcile(ctx, boundary, claim, state); recErr != nil { + return Result{}, fmt.Errorf("%w (reconciliation failed: %v)", journalErr, recErr) + } + return Execute(ctx, boundary, claim, effect) + } + } + if ttl <= 0 { + ttl = DefaultUnresolvedTTL + } + expired, expErr := ExpireUnresolved(ctx, boundary, claim, ttl) + if expErr != nil { + return Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v; expiry failed: %v)", journalErr, lookupErr, expErr) + } + if !expired { + return Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v)", journalErr, lookupErr) + } + return Execute(ctx, boundary, claim, effect) +} + +// Lookup is implemented by boundaries that journal effects and can report, +// without running anything, whether claim's logical operation is already +// applied. Callers use it to recognise an equivalent operation under a +// neighbouring key (e.g. the previous idempotency bucket) that Execute's +// exact-ID replay cannot see. +type Lookup interface { + LookupApplied(ctx context.Context, claim Claim) (Result, bool) +} + +// LookupApplied reports whether boundary has claim's operation journaled as +// applied, with its recorded provenance. Boundaries that do not journal (nil, +// NoopBoundary) never report an applied operation. +func LookupApplied(ctx context.Context, boundary Boundary, claim Claim) (Result, bool) { + l, ok := boundary.(Lookup) + if !ok { + return Result{}, false + } + return l.LookupApplied(ctx, claim) +} + +type Claim struct { + Repo string + Kind string + Target string + Actor string + Inputs map[string]string + + // ObserveApplied revalidates a cached label mutation against current + // external state. It is not part of the operation's logical identity. + // Other effect kinds retain permanent Applied deduplication. + ObserveApplied func(context.Context) (ExternalState, error) `json:"-"` +} + +type Result struct { + Provenance string +} + +type Boundary interface { + Execute(ctx context.Context, claim Claim, effect func(context.Context) (Result, error)) (Result, error) +} + +type NoopBoundary struct{} + +func (NoopBoundary) Execute(ctx context.Context, _ Claim, effect func(context.Context) (Result, error)) (Result, error) { + return effect(ctx) +} + +func Execute(ctx context.Context, boundary Boundary, claim Claim, effect func(context.Context) (Result, error)) (Result, error) { + if boundary == nil { + boundary = NoopBoundary{} + } + return boundary.Execute(ctx, claim, effect) +} + +func StableDigest(parts ...string) string { + h := sha256.New() + for _, p := range parts { + h.Write([]byte(p)) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +func StableInputs(inputs map[string]string) string { + if len(inputs) == 0 { + return "" + } + keys := make([]string, 0, len(inputs)) + for k := range inputs { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + for _, k := range keys { + b.WriteString(k) + b.WriteByte('=') + b.WriteString(inputs[k]) + b.WriteByte('\n') + } + return b.String() +} + +type Stats struct { + Mode string `json:"mode"` + Journaled int64 `json:"journaled"` + Fenced int64 `json:"fenced"` + Denied int64 `json:"denied"` +} + +type Recorder struct { + mu sync.Mutex + stats Stats +} + +func (r *Recorder) SetMode(mode string) { + if r == nil { + return + } + r.mu.Lock() + defer r.mu.Unlock() + r.stats.Mode = mode +} + +func (r *Recorder) IncJournaled() { + if r == nil { + return + } + r.mu.Lock() + r.stats.Journaled++ + r.mu.Unlock() +} + +func (r *Recorder) IncFenced() { + if r == nil { + return + } + r.mu.Lock() + r.stats.Fenced++ + r.mu.Unlock() +} + +func (r *Recorder) IncDenied() { + if r == nil { + return + } + r.mu.Lock() + r.stats.Denied++ + r.mu.Unlock() +} + +func (r *Recorder) Snapshot() Stats { + if r == nil { + return Stats{} + } + r.mu.Lock() + defer r.mu.Unlock() + return r.stats +} + +type LoggingBoundary struct { + Next Boundary + Logger *slog.Logger + Recorder *Recorder +} + +func (b LoggingBoundary) Execute(ctx context.Context, claim Claim, effect func(context.Context) (Result, error)) (Result, error) { + res, err := Execute(ctx, b.Next, claim, effect) + if err != nil && errors.Is(err, ErrDenied) && b.Logger != nil { + b.Logger.Warn("mutation boundary denied external effect", + "repo", claim.Repo, "kind", claim.Kind, "target", claim.Target, "error", err) + } + if err == nil && b.Recorder != nil { + b.Recorder.IncJournaled() + } + return res, err +} + +func (b LoggingBoundary) Reconcile(ctx context.Context, claim Claim, state ExternalState) error { + return Reconcile(ctx, b.Next, claim, state) +} + +func (b LoggingBoundary) ExpireUnresolved(ctx context.Context, claim Claim, ttl time.Duration) (bool, error) { + return ExpireUnresolved(ctx, b.Next, claim, ttl) +} + +func (c Claim) Validate() error { + if strings.TrimSpace(c.Repo) == "" || !strings.Contains(c.Repo, "/") { + return fmt.Errorf("repo must be owner/name") + } + if strings.TrimSpace(c.Kind) == "" { + return fmt.Errorf("kind is required") + } + if strings.TrimSpace(c.Target) == "" { + return fmt.Errorf("target is required") + } + return nil +} diff --git a/src/pkg/effects/effects_test.go b/src/crux/effects/effects_test.go similarity index 100% rename from src/pkg/effects/effects_test.go rename to src/crux/effects/effects_test.go diff --git a/src/pkg/effects/execute_reconciling_test.go b/src/crux/effects/execute_reconciling_test.go similarity index 100% rename from src/pkg/effects/execute_reconciling_test.go rename to src/crux/effects/execute_reconciling_test.go diff --git a/src/pkg/effects/reconcile_test.go b/src/crux/effects/reconcile_test.go similarity index 100% rename from src/pkg/effects/reconcile_test.go rename to src/crux/effects/reconcile_test.go diff --git a/src/pkg/gitidentity/gitidentity.go b/src/crux/gitidentity/gitidentity.go similarity index 100% rename from src/pkg/gitidentity/gitidentity.go rename to src/crux/gitidentity/gitidentity.go diff --git a/src/pkg/gitidentity/gitidentity_test.go b/src/crux/gitidentity/gitidentity_test.go similarity index 100% rename from src/pkg/gitidentity/gitidentity_test.go rename to src/crux/gitidentity/gitidentity_test.go diff --git a/src/crux/go.mod b/src/crux/go.mod new file mode 100644 index 0000000000..4de541c8ef --- /dev/null +++ b/src/crux/go.mod @@ -0,0 +1,32 @@ +module github.com/hivecommons/hive/crux + +go 1.26.10 + +require ( + github.com/google/cel-go v0.31.0 + github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/trace v1.47.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/kr/text v0.2.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect +) + +require ( + cel.dev/expr v0.25.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect + golang.org/x/text v0.43.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/src/crux/go.sum b/src/crux/go.sum new file mode 100644 index 0000000000..f5b7c49322 --- /dev/null +++ b/src/crux/go.sum @@ -0,0 +1,53 @@ +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/cel-go v0.31.0 h1:H0bhpFTqOvmHrBGrWKp7ZlhBm5Hh8PYUEXnwxT1LL7A= +github.com/google/cel-go v0.31.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= +github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/crux/hivecap/credentials.go b/src/crux/hivecap/credentials.go new file mode 100644 index 0000000000..194e96f2de --- /dev/null +++ b/src/crux/hivecap/credentials.go @@ -0,0 +1,235 @@ +package hivecap + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "sync" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/config" +) + +// Credential writers for a sidecar runtime (hivecommons/hive#14138 stage 4). +// In-process, hive hands the manager its GitHub App auth (SetAppAuth), the +// opt-in mint issuer (SetAgentMint) and the Bob key resolvers directly; here +// the same three seams are satisfied over the callback. Token material is +// returned to the manager or written to the existing per-agent cache files +// and goes nowhere else: never logged, never kept in these adapters beyond +// the Bob key's short cache. + +const ( + // agentTokenCachePerms is the mode a cache file is created with when + // the entrypoint did not pre-create it (owner-only; the agent cannot + // read it, which WriteAgentToken says out loud). + agentTokenCachePerms = 0o600 + // agentTokenCacheDirPerms is the cache directory mode. + agentTokenCacheDirPerms = 0o755 + // DefaultBobCacheTTL is how long BobCredentials keeps hive's answer: + // the resolvers are read on the launch path under the manager lock, + // so one round trip per minute, not per launch. + DefaultBobCacheTTL = time.Minute +) + +// ErrProxyInjectionUnsupported is returned by AppTokenWriter when +// HIVE_PROXY_INJECT_GH_AUTH is on: in-process that flag diverts the real +// token to hive's proxy registry and writes a placeholder; a sidecar has no +// such registry yet (stage 5), and writing the real token would silently +// undo the opt-in, so the write is refused and the agent runs on no token. +var ErrProxyInjectionUnsupported = errors.New("hivecap: proxy-side GitHub credential injection is not available in sidecar mode") + +// AppTokenWriter implements crux.AppTokenMinter: WriteAgentToken mints the +// tier-scoped, installation-wide token hive's AppAuth mints in-process +// (POST /v1/tokens with the agent and tier, no repos, hive's TTL) and +// writes it in place into crux.AgentTokenCachePath. In place, not +// temp-and-rename: the entrypoint pre-creates the file dev:hive- +// 0640 and a rename would replace that inode (see pkg/github.AppAuth). +type AppTokenWriter struct { + caps callback.TokenMinter + // CacheDir is the token cache directory; empty means + // crux.DefaultAgentTokenCacheDir. + CacheDir string + // Logger receives the outcome lines (never the token); nil means + // slog.Default(). + Logger *slog.Logger + // injectOn reports HIVE_PROXY_INJECT_GH_AUTH; tests replace it. + injectOn func() bool +} + +// NewAppTokenWriter builds a writer over caps. +func NewAppTokenWriter(caps callback.TokenMinter) *AppTokenWriter { + return &AppTokenWriter{caps: caps, injectOn: config.ProxyInjectGHAuth} +} + +func (w *AppTokenWriter) logger() *slog.Logger { + if w.Logger == nil { + return slog.Default() + } + return w.Logger +} + +func (w *AppTokenWriter) dir() string { + if w.CacheDir == "" { + return crux.DefaultAgentTokenCacheDir + } + return w.CacheDir +} + +// WriteAgentToken mints and caches agentName's token. A hive without a +// GitHub App answers callback.ErrNotConfigured and an unreachable hive +// callback.ErrUnavailable; both leave the cache file untouched (a stale +// token is better than none until it expires) and are returned so the +// manager's condition reporting sees them. +func (w *AppTokenWriter) WriteAgentToken(ctx context.Context, agentName, tier string, agentUID int) error { + if w == nil || w.caps == nil { + return errors.New("hivecap: no capabilities") + } + if w.injectOn != nil && w.injectOn() { + return fmt.Errorf("%w: agent %s", ErrProxyInjectionUnsupported, agentName) + } + resp, err := w.caps.MintToken(ctx, callback.MintTokenRequest{Agent: agentName, Tier: tier}) + if err != nil { + return fmt.Errorf("minting scoped token for %s: %w", agentName, err) + } + if !resp.ExpiresAt.IsZero() && !resp.ExpiresAt.After(time.Now()) { + return fmt.Errorf("minting scoped token for %s: already expired at %s", agentName, resp.ExpiresAt.Format(time.RFC3339)) + } + dir := w.dir() + if err := os.MkdirAll(dir, agentTokenCacheDirPerms); err != nil { + return fmt.Errorf("creating agent token dir: %w", err) + } + cachePath := crux.AgentTokenCachePath(dir, agentName) + _, statErr := os.Stat(cachePath) + preCreated := statErr == nil + f, err := os.OpenFile(cachePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, agentTokenCachePerms) + if err != nil { + return fmt.Errorf("opening agent token cache %s: %w", cachePath, err) + } + if _, err := f.WriteString(resp.Token); err != nil { + _ = f.Close() + return fmt.Errorf("writing agent token cache %s: %w", cachePath, err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("closing agent token cache %s: %w", cachePath, err) + } + if !preCreated && agentUID > 0 { + w.logger().Warn("agent token cache was not pre-created by entrypoint — the agent CANNOT read it; ensure the sidecar's entrypoint created this file as dev:hive- 0640", + "agent", agentName, "uid", agentUID, "path", cachePath) + } + w.logger().Info("per-agent token cached via hive callback", "agent", agentName, "tier", tier, "uid", agentUID, "pre_created", preCreated, "expires_at", resp.ExpiresAt) + return nil +} + +// AgentMint implements crux.AgentMintIssuer over POST /v1/tokens/mint. +// Enabled reports whether the capability exists on this side at all; a hive +// whose mint is off answers an empty token (the in-process "" contract) or +// not_configured, both of which MintAgentToken turns into "" with no error +// so the manager's additive mint path stays a no-op. An unreachable hive +// is an error the manager logs and swallows (the App token is unaffected). +type AgentMint struct { + caps callback.AgentMinter + timeout time.Duration +} + +// NewAgentMint builds the issuer over caps; nil when caps cannot mint. +func NewAgentMint(caps any, timeout time.Duration) AgentMint { + if timeout <= 0 { + timeout = callback.DefaultTimeout + } + m, _ := caps.(callback.AgentMinter) + return AgentMint{caps: m, timeout: timeout} +} + +// Enabled is true when the callback can be asked. +func (m AgentMint) Enabled() bool { return m.caps != nil } + +// MintAgentToken asks hive for agentName's mint credential at tier. +func (m AgentMint) MintAgentToken(agentName, tier string) (string, error) { + if m.caps == nil { + return "", nil + } + ctx, cancel := background(m.timeout) + defer cancel() + resp, err := m.caps.MintAgentToken(ctx, callback.AgentMintRequest{Agent: agentName, Tier: tier}) + if errors.Is(err, callback.ErrNotConfigured) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("hivecap: minting agent token: %w", err) + } + if !resp.ExpiresAt.IsZero() && !resp.ExpiresAt.After(time.Now()) { + return "", fmt.Errorf("hivecap: minted agent token already expired at %s", resp.ExpiresAt.Format(time.RFC3339)) + } + return resp.Token, nil +} + +// BobCredentials serves SetBobAPIKeyResolver (Key) and +// SetBobKeySourceResolver (Source) from GET /v1/bob/credential, caching +// hive's answer for TTL. Unreachable or not configured is an empty +// credential: no key is injected and no source file is checked, exactly +// the unconfigured in-process behaviour. +type BobCredentials struct { + caps callback.BobCredentials + timeout time.Duration + // TTL bounds the cache; non-positive means DefaultBobCacheTTL. + TTL time.Duration + // now is the clock; tests replace it. + now func() time.Time + + mu sync.Mutex + cred callback.BobCredential + fetched time.Time +} + +// NewBobCredentials builds the resolver over caps; a caps without the Bob +// half resolves empty. +func NewBobCredentials(caps any, timeout time.Duration) *BobCredentials { + if timeout <= 0 { + timeout = callback.DefaultTimeout + } + b, _ := caps.(callback.BobCredentials) + return &BobCredentials{caps: b, timeout: timeout, now: time.Now} +} + +func (b *BobCredentials) ttl() time.Duration { + if b.TTL <= 0 { + return DefaultBobCacheTTL + } + return b.TTL +} + +// resolve returns the cached credential, refreshing it when stale. A failed +// refresh keeps serving the previous answer until TTL, then resolves empty. +func (b *BobCredentials) resolve() callback.BobCredential { + if b == nil || b.caps == nil { + return callback.BobCredential{} + } + b.mu.Lock() + defer b.mu.Unlock() + now := b.now() + if !b.fetched.IsZero() && now.Sub(b.fetched) < b.ttl() { + return b.cred + } + ctx, cancel := background(b.timeout) + defer cancel() + cred, err := b.caps.BobCredential(ctx) + if err != nil { + if !b.fetched.IsZero() && now.Sub(b.fetched) < 2*b.ttl() { + return b.cred + } + b.cred, b.fetched = callback.BobCredential{}, time.Time{} + return b.cred + } + b.cred, b.fetched = cred, now + return cred +} + +// Key is the Bob API key ("" when none). +func (b *BobCredentials) Key() string { return b.resolve().APIKey } + +// Source is where hive found the key ("file:", "env:", ""). +func (b *BobCredentials) Source() string { return b.resolve().Source } diff --git a/src/crux/hivecap/credentials_test.go b/src/crux/hivecap/credentials_test.go new file mode 100644 index 0000000000..898e843040 --- /dev/null +++ b/src/crux/hivecap/credentials_test.go @@ -0,0 +1,295 @@ +package hivecap + +import ( + "bytes" + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api/callback" +) + +// credDirect is direct plus the agent_mint and bob families. +type credDirect struct { + *direct + mu sync.Mutex + enabled bool + mintErr error + mintReqs []callback.AgentMintRequest + bob callback.BobCredential + bobErr error + bobCalls int +} + +func (c *credDirect) MintAgentToken(_ context.Context, req callback.AgentMintRequest) (callback.AgentMintResponse, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.mintReqs = append(c.mintReqs, req) + if c.mintErr != nil { + return callback.AgentMintResponse{}, c.mintErr + } + if !c.enabled { + return callback.AgentMintResponse{}, nil + } + return callback.AgentMintResponse{Token: "mint-" + req.Agent + "-" + req.Tier, ExpiresAt: time.Now().Add(time.Hour)}, nil +} + +func (c *credDirect) BobCredential(context.Context) (callback.BobCredential, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.bobCalls++ + return c.bob, c.bobErr +} + +func unreachableClient(t *testing.T) *callback.Client { + t.Helper() + dir, err := os.MkdirTemp("", "hcap") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + c, err := callback.NewClient("unix://"+filepath.Join(dir, "nobody.sock"), "") + if err != nil { + t.Fatal(err) + } + return c +} + +// The sidecar App token writer asks for exactly what the in-process writer +// asks AppAuth for — the agent's tier, installation-wide, no repos — and +// writes the token in place to the same cache path, with the same expiry +// whether hive is reached directly or over the socket. +func TestAppTokenWriterParity(t *testing.T) { + exp := time.Now().Add(42 * time.Minute).Truncate(time.Second) + run := func(t *testing.T, d *direct, caps callback.TokenMinter) (string, []byte) { + t.Helper() + cache := t.TempDir() + w := NewAppTokenWriter(caps) + w.CacheDir = cache + w.injectOn = func() bool { return false } + var logBuf bytes.Buffer + w.Logger = slog.New(slog.NewTextHandler(&logBuf, nil)) + // Pre-created by the "entrypoint", as the real path expects. + path := crux.AgentTokenCachePath(cache, "worker") + if err := os.WriteFile(path, []byte("stale"), 0o640); err != nil { + t.Fatal(err) + } + if err := w.WriteAgentToken(context.Background(), "worker", "trusted", 1000); err != nil { + t.Fatalf("WriteAgentToken: %v", err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if len(d.mints) != 1 || d.mints[0].Agent != "worker" || d.mints[0].Tier != "trusted" || d.mints[0].Repo != "" || len(d.mints[0].Repos) != 0 || d.mints[0].TTLSeconds != 0 { + t.Errorf("mint request = %+v, want installation-wide trusted for worker", d.mints) + } + if strings.Contains(logBuf.String(), string(got)) { + t.Errorf("token leaked into log:\n%s", logBuf.String()) + } + if !strings.Contains(logBuf.String(), exp.UTC().Format("2006-01-02T15:04:05")) && !strings.Contains(logBuf.String(), exp.Format("2006-01-02T15:04:05")) { + t.Errorf("expiry not reported: %s", logBuf.String()) + } + if strings.Contains(logBuf.String(), "not pre-created") { + t.Errorf("pre-created file misreported: %s", logBuf.String()) + } + return filepath.Base(path), got + } + dd := &direct{expiry: exp} + dp, dtok := run(t, dd, dd) + sd := &direct{expiry: exp} + sp, stok := run(t, sd, overSocket(t, sd)) + if dp != sp || !bytes.Equal(dtok, stok) || string(dtok) != "ghs_trusted_" { + t.Errorf("direct (%s %q) vs socket (%s %q) differ", dp, dtok, sp, stok) + } +} + +func TestAppTokenWriterFailsClosed(t *testing.T) { + cache := t.TempDir() + path := crux.AgentTokenCachePath(cache, "w") + if err := os.WriteFile(path, []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + ctx := context.Background() + check := func(t *testing.T, w *AppTokenWriter, want error) { + t.Helper() + w.CacheDir = cache + w.injectOn = func() bool { return false } + err := w.WriteAgentToken(ctx, "w", "t", 0) + if err == nil || (want != nil && !errors.Is(err, want)) { + t.Errorf("err = %v, want %v", err, want) + } + if got, _ := os.ReadFile(path); string(got) != "old" { + t.Errorf("cache touched on failure: %q", got) + } + } + t.Run("unreachable", func(t *testing.T) { + check(t, NewAppTokenWriter(unreachableClient(t)), callback.ErrUnavailable) + }) + t.Run("not configured", func(t *testing.T) { + check(t, NewAppTokenWriter(overSocket(t, &direct{absent: map[string]bool{callback.CapabilityTokens: true}})), callback.ErrNotConfigured) + }) + t.Run("expired", func(t *testing.T) { + check(t, NewAppTokenWriter(&direct{expiry: time.Now().Add(-time.Second)}), nil) + }) + t.Run("proxy injection", func(t *testing.T) { + w := NewAppTokenWriter(&direct{}) + w.CacheDir = cache + w.injectOn = func() bool { return true } + if err := w.WriteAgentToken(ctx, "w", "t", 0); !errors.Is(err, ErrProxyInjectionUnsupported) { + t.Errorf("err = %v", err) + } + if got, _ := os.ReadFile(path); string(got) != "old" { + t.Errorf("cache touched: %q", got) + } + }) + t.Run("nil", func(t *testing.T) { + var w *AppTokenWriter + if err := w.WriteAgentToken(ctx, "w", "t", 0); err == nil { + t.Error("nil writer must error") + } + }) + t.Run("default inject hook", func(t *testing.T) { + if NewAppTokenWriter(&direct{}).injectOn == nil { + t.Error("inject hook unset") + } + }) +} + +// A cache file the entrypoint did not pre-create is still written (0600) +// but the agent is told it cannot read it. +func TestAppTokenWriterCreatesAndWarns(t *testing.T) { + cache := filepath.Join(t.TempDir(), "nested") + w := NewAppTokenWriter(&direct{}) + w.CacheDir = cache + w.injectOn = func() bool { return false } + var logBuf bytes.Buffer + w.Logger = slog.New(slog.NewTextHandler(&logBuf, nil)) + if err := w.WriteAgentToken(context.Background(), "w", "t", 1000); err != nil { + t.Fatal(err) + } + st, err := os.Stat(crux.AgentTokenCachePath(cache, "w")) + if err != nil { + t.Fatal(err) + } + if st.Mode().Perm() != agentTokenCachePerms { + t.Errorf("mode = %v", st.Mode()) + } + if !strings.Contains(logBuf.String(), "not pre-created") { + t.Errorf("missing warning: %s", logBuf.String()) + } + if w.dir() != cache || NewAppTokenWriter(&direct{}).dir() != crux.DefaultAgentTokenCacheDir { + t.Error("dir default wrong") + } +} + +// The mint issuer is the in-process contract: "" with no error when mint is +// off or hive has no mint, the token otherwise, and an error (which the +// manager logs and swallows) only when hive cannot be reached. +func TestAgentMintIssuer(t *testing.T) { + d := &credDirect{direct: &direct{}, enabled: true} + for name, caps := range map[string]any{"direct": d, "socket": overSocket(t, d)} { + t.Run(name, func(t *testing.T) { + m := NewAgentMint(caps, 0) + if !m.Enabled() { + t.Fatal("must be enabled") + } + tok, err := m.MintAgentToken("a", "trusted") + if err != nil || tok != "mint-a-trusted" { + t.Errorf("mint = %q, %v", tok, err) + } + }) + } + if len(d.mintReqs) != 2 || d.mintReqs[0] != d.mintReqs[1] || d.mintReqs[0].Agent != "a" { + t.Errorf("requests differ across modes: %+v", d.mintReqs) + } + + off := &credDirect{direct: &direct{}} + if tok, err := NewAgentMint(off, 0).MintAgentToken("a", "t"); err != nil || tok != "" { + t.Errorf("mint off = %q, %v", tok, err) + } + none := NewAgentMint(&direct{}, 0) + if none.Enabled() { + t.Error("plain capabilities cannot mint") + } + if tok, err := none.MintAgentToken("a", "t"); err != nil || tok != "" { + t.Errorf("no family = %q, %v", tok, err) + } + absent := &credDirect{direct: &direct{absent: map[string]bool{callback.CapabilityAgentMint: true}}, enabled: true} + if tok, err := NewAgentMint(overSocket(t, absent), 0).MintAgentToken("a", "t"); err != nil || tok != "" { + t.Errorf("not configured over socket = %q, %v", tok, err) + } + if tok, err := NewAgentMint(unreachableClient(t), 200*time.Millisecond).MintAgentToken("a", "t"); tok != "" || !errors.Is(err, callback.ErrUnavailable) { + t.Errorf("unreachable = %q, %v", tok, err) + } + failing := &credDirect{direct: &direct{}, enabled: true, mintErr: errors.New("boom")} + if tok, err := NewAgentMint(failing, 0).MintAgentToken("a", "t"); tok != "" || err == nil { + t.Errorf("failure = %q, %v", tok, err) + } +} + +func TestBobCredentialsCacheAndFailClosed(t *testing.T) { + d := &credDirect{direct: &direct{}, bob: callback.BobCredential{APIKey: "bob-key", Source: "env:BOB_API_KEY"}} + now := time.Unix(1_700_000_000, 0) + b := NewBobCredentials(overSocket(t, d), 0) + b.now = func() time.Time { return now } + if b.Key() != "bob-key" || b.Source() != "env:BOB_API_KEY" { + t.Fatalf("key/source = %q/%q", b.Key(), b.Source()) + } + if d.bobCalls != 1 { + t.Errorf("calls = %d, want one (cached)", d.bobCalls) + } + // Past TTL hive is asked again; a failure keeps the last answer for one + // more TTL, then resolves empty. + d.mu.Lock() + d.bobErr = errors.New("boom") + d.mu.Unlock() + now = now.Add(DefaultBobCacheTTL + time.Second) + if b.Key() != "bob-key" || d.bobCalls != 2 { + t.Errorf("stale-but-served: key=%q calls=%d", b.Key(), d.bobCalls) + } + now = now.Add(DefaultBobCacheTTL) + if b.Key() != "" || b.Source() != "" { + t.Errorf("expired failure must resolve empty: %q/%q", b.Key(), b.Source()) + } + d.mu.Lock() + d.bobErr = nil + d.mu.Unlock() + if b.Key() != "bob-key" { + t.Error("recovery not picked up") + } + + if n := NewBobCredentials(&direct{}, 0); n.Key() != "" || n.Source() != "" { + t.Error("capabilities without bob must resolve empty") + } + var nilB *BobCredentials + if nilB.Key() != "" { + t.Error("nil resolver must resolve empty") + } + u := NewBobCredentials(unreachableClient(t), 200*time.Millisecond) + if u.Key() != "" || u.Source() != "" { + t.Error("unreachable must resolve empty") + } + short := NewBobCredentials(d, 0) + short.TTL = time.Hour + if short.ttl() != time.Hour { + t.Error("TTL override ignored") + } +} + +func TestSetCarriesCredentialWriters(t *testing.T) { + s := New(&credDirect{direct: &direct{}, enabled: true}, 0) + if s.AppTokens == nil || !s.Mint.Enabled() || s.Bob == nil { + t.Errorf("Set missing credential writers: %+v", s) + } + if s2 := New(&direct{}, 0); s2.Mint.Enabled() { + t.Error("plain capabilities must not claim mint") + } +} diff --git a/src/crux/hivecap/hivecap.go b/src/crux/hivecap/hivecap.go new file mode 100644 index 0000000000..1520730332 --- /dev/null +++ b/src/crux/hivecap/hivecap.go @@ -0,0 +1,571 @@ +// Package hivecap turns a callback.Capabilities (hive's capability set, +// reached over the callback endpoint or called directly) into the seam +// interfaces the Crux ring already consumes: pushbroker.TokenScoper, +// proxy.GitHubRESTAccounting, proxy.UsageSink, proxy.CanaryScanner, +// toolapprove.TextScanner, crux.ConditionReporter and agent.PRCreator +// (hivecommons/hive#14138 stage 2; src/docs/design/crux.md "Credentials and +// hive-provided capabilities"), and caches hive's crux.Host policy snapshot +// for the runtime's policy resolvers (Policy, stage 4). +// +// Every adapter fails closed. The seams that can return an error return one +// wrapping callback.ErrUnavailable or callback.ErrUnauthorized; the seams +// that cannot (the scanners, the sinks, the snapshot) return the value that +// takes the ring's existing refusal path: a canary detection, a Blocked +// verdict, ok=false. A capability hive reports as not configured +// (callback.ErrNotConfigured) is treated as the nil seam it stands in for. +// +// The package is written against callback.Capabilities, not *callback.Client, +// so the in-process mode can hand it hive's direct implementation and the +// sidecar mode a client; the parity test asserts both behave the same. +package hivecap + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/toolapprove" +) + +const ( + // UnavailableSourceSuffix marks a canary "detection" that is really the + // endpoint being unreachable, so the proxy's audit line says which. + UnavailableSourceSuffix = " (hive callback unavailable)" + // TextFindingKind and TextFindingRule name the synthetic finding a + // Blocked verdict carries when the scan could not run. + TextFindingKind = "callback" + TextFindingRule = "hive_unavailable" +) + +// Set is every adapter over one Capabilities. Build it with New; the fields +// are the values the runtime's setters take. +type Set struct { + caps callback.Capabilities + timeout time.Duration + + Accounting GitHubRESTAccounting + Usage UsageSink + Canary CanaryScanner + Text TextScanner + Conditions ConditionReporter + PullRequest PRCreator + Linear LinearCredentials + Policy *Policy + Inference *InferenceRoutes + Mutation MutationBoundary + // AppTokens, Mint and Bob are the stage-4 credential writers: the + // sidecar's SetAppAuth, SetAgentMint and SetBob*Resolver arguments. + AppTokens *AppTokenWriter + Mint AgentMint + Bob *BobCredentials +} + +// New builds the adapters over caps. timeout bounds the calls made from +// seams whose callers pass no context (the sinks and scanners); non-positive +// selects callback.DefaultTimeout. +func New(caps callback.Capabilities, timeout time.Duration) *Set { + if timeout <= 0 { + timeout = callback.DefaultTimeout + } + s := &Set{caps: caps, timeout: timeout} + s.Accounting = GitHubRESTAccounting{caps: caps, timeout: timeout} + s.Usage = UsageSink{caps: caps, timeout: timeout} + s.Canary = CanaryScanner{caps: caps, timeout: timeout} + s.Text = TextScanner{caps: caps, timeout: timeout} + s.Conditions = ConditionReporter{caps: caps, timeout: timeout} + s.PullRequest = PRCreator{caps: caps} + s.Linear = LinearCredentials{caps: caps, timeout: timeout} + s.AppTokens = NewAppTokenWriter(caps) + s.Mint = NewAgentMint(caps, timeout) + s.Bob = NewBobCredentials(caps, timeout) + s.Policy = NewPolicy(caps, timeout) + s.Inference = NewInferenceRoutes(caps, timeout) + s.Mutation = NewMutationBoundary(caps) + return s +} + +// TokenScoper returns a pushbroker.TokenScoper whose mints are attributed +// to agent. ttl is the lifetime to request; zero lets hive choose. +func (s *Set) TokenScoper(agent string, ttl time.Duration) TokenScoper { + return TokenScoper{caps: s.caps, Agent: agent, TTL: ttl} +} + +func background(timeout time.Duration) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), timeout) +} + +// TokenScoper implements pushbroker.TokenScoper: ScopedTokenForRepos mints +// through the callback. The token is returned to the caller and held +// nowhere else; pushbroker keeps it for one push. +type TokenScoper struct { + caps callback.Capabilities + // Agent is stamped on every mint request. + Agent string + // TTL is the lifetime requested; zero lets hive choose. + TTL time.Duration +} + +// ScopedTokenForRepos mints a token narrowed to tier and, when non-empty, +// to repos (bare repository names). A hive without a GitHub App answers +// callback.ErrNotConfigured, which is the error the nil-App path gives +// today; unreachable is callback.ErrUnavailable. Both fail the push. +func (t TokenScoper) ScopedTokenForRepos(ctx context.Context, tier string, repos []string) (string, error) { + if t.caps == nil { + return "", errors.New("hivecap: no capabilities") + } + req := callback.MintTokenRequest{Agent: t.Agent, Tier: tier, Repos: repos} + if t.TTL > 0 { + req.TTLSeconds = int(t.TTL / time.Second) + } + resp, err := t.caps.MintToken(ctx, req) + if err != nil { + return "", fmt.Errorf("hivecap: minting scoped token: %w", err) + } + if !resp.ExpiresAt.IsZero() && !resp.ExpiresAt.After(time.Now()) { + return "", fmt.Errorf("hivecap: minted token already expired at %s", resp.ExpiresAt.Format(time.RFC3339)) + } + return resp.Token, nil +} + +// GitHubRESTAccounting implements proxy.GitHubRESTAccounting. Records are +// best effort (a failure is dropped and counted by the client's Status); +// CoreRateLimitSnapshot answers ok=false when hive cannot, so the agent +// GitHub budget applies the per-agent cap alone. +type GitHubRESTAccounting struct { + caps callback.Capabilities + timeout time.Duration +} + +func (a GitHubRESTAccounting) RecordRESTRequest(caller, method, path string, status int, h http.Header) { + a.record(caller, method, path, status, false, h) +} + +func (a GitHubRESTAccounting) RecordInjectedRESTRequest(caller, method, path string, status int, h http.Header) { + a.record(caller, method, path, status, true, h) +} + +func (a GitHubRESTAccounting) record(caller, method, path string, status int, injected bool, h http.Header) { + if a.caps == nil { + return + } + ctx, cancel := background(a.timeout) + defer cancel() + _ = a.caps.RecordREST(ctx, callback.RESTRecord{ + Caller: caller, Method: method, Path: path, Status: status, Injected: injected, + RateLimit: rateLimitHeaders(h), + }) +} + +// rateLimitHeaders keeps only the X-RateLimit-* response headers: the +// accounting needs nothing else and the rest of a GitHub response header +// set can carry request ids and cookies that have no business on the wire. +func rateLimitHeaders(h http.Header) map[string]string { + var out map[string]string + for k, v := range h { + if !strings.HasPrefix(http.CanonicalHeaderKey(k), callback.RateLimitHeaderPrefix) || len(v) == 0 { + continue + } + if out == nil { + out = map[string]string{} + } + out[http.CanonicalHeaderKey(k)] = v[0] + } + return out +} + +func (a GitHubRESTAccounting) CoreRateLimitSnapshot() (remaining int, reset time.Time, ok bool) { + if a.caps == nil { + return 0, time.Time{}, false + } + ctx, cancel := background(a.timeout) + defer cancel() + snap, err := a.caps.RateLimit(ctx) + if err != nil || !snap.OK { + return 0, time.Time{}, false + } + return snap.Remaining, snap.Reset, true +} + +// UsageSink implements proxy.UsageSink; records are best effort. +type UsageSink struct { + caps callback.Capabilities + timeout time.Duration +} + +func (u UsageSink) Record(agent, model string, in, out int64) { + u.send(callback.UsageRecord{Agent: agent, Model: model, Kind: callback.UsageKindGateway, InputTokens: in, OutputTokens: out}) +} + +func (u UsageSink) RecordCopilot(agent, model string, in, out int64) { + u.send(callback.UsageRecord{Agent: agent, Model: model, Kind: callback.UsageKindCopilot, InputTokens: in, OutputTokens: out}) +} + +func (u UsageSink) RecordCopilotWithCache(agent, model string, in, out, cacheRead, cacheCreation int64) { + u.send(callback.UsageRecord{Agent: agent, Model: model, Kind: callback.UsageKindCopilotCache, InputTokens: in, OutputTokens: out, CacheRead: cacheRead, CacheCreation: cacheCreation}) +} + +func (u UsageSink) send(rec callback.UsageRecord) { + if u.caps == nil { + return + } + ctx, cancel := background(u.timeout) + defer cancel() + _ = u.caps.RecordUsage(ctx, rec) +} + +// CanaryScanner implements proxy.CanaryScanner. Scan has no error return, +// so an unreachable hive is reported as a detection with an empty Token and +// Source suffixed UnavailableSourceSuffix: with the proxy's fail-closed +// setting the write is refused, without it the write is audited, which is +// what an unscannable body does today. A hive with no canary registry +// (callback.ErrNotConfigured) scans nothing, like a nil registry. +type CanaryScanner struct { + caps callback.Capabilities + timeout time.Duration +} + +func (c CanaryScanner) Scan(agentName, text, source string) (crux.CanaryLeak, bool) { + if c.caps == nil { + return crux.CanaryLeak{}, false + } + ctx, cancel := background(c.timeout) + defer cancel() + resp, err := c.caps.ScanCanary(ctx, callback.CanaryScanRequest{Agent: agentName, Text: text, Source: source}) + if err != nil { + if errors.Is(err, callback.ErrNotConfigured) { + return crux.CanaryLeak{}, false + } + return crux.CanaryLeak{Agent: agentName, Source: source + UnavailableSourceSuffix}, true + } + if !resp.Detected { + return crux.CanaryLeak{}, false + } + leak := crux.CanaryLeak{Agent: resp.Agent, Source: resp.Source} + if leak.Agent == "" { + leak.Agent = agentName + } + if leak.Source == "" { + leak.Source = source + } + return leak, true +} + +// TextScanner implements toolapprove.TextScanner. An unreachable hive is a +// Blocked verdict with one synthetic finding, the denial +// toolapprove.ErrNoTextScanner produces with no scanner installed. There is +// no not-configured case: a hive always has the rule set. +type TextScanner struct { + caps callback.Capabilities + timeout time.Duration +} + +func (t TextScanner) ScanInput(text string) toolapprove.TextVerdict { + return t.scan(callback.ScanInput, text) +} + +func (t TextScanner) ScanOutput(text string) toolapprove.TextVerdict { + return t.scan(callback.ScanOutput, text) +} + +func (t TextScanner) scan(direction, text string) toolapprove.TextVerdict { + if t.caps == nil { + return unavailableVerdict(errors.New("hivecap: no capabilities")) + } + ctx, cancel := background(t.timeout) + defer cancel() + resp, err := t.caps.ScanText(ctx, callback.TextScanRequest{Direction: direction, Text: text}) + if err != nil { + return unavailableVerdict(err) + } + v := toolapprove.TextVerdict{Blocked: resp.Blocked, CriticalInjection: resp.CriticalInjection} + for _, f := range resp.Findings { + v.Findings = append(v.Findings, toolapprove.TextFinding{Kind: f.Kind, Snippet: f.Snippet, Rule: f.Rule, High: f.High}) + } + return v +} + +func unavailableVerdict(err error) toolapprove.TextVerdict { + return toolapprove.TextVerdict{ + Blocked: true, + Findings: []toolapprove.TextFinding{{Kind: TextFindingKind, Rule: TextFindingRule, Snippet: err.Error(), High: true}}, + } +} + +// ConditionReporter implements crux.ConditionReporter; reports are best +// effort. +type ConditionReporter struct { + caps callback.Capabilities + timeout time.Duration +} + +func (r ConditionReporter) Observe(class, key, message, diagnostics string) { + r.send(callback.ConditionReport{Action: callback.ConditionObserve, Class: class, Key: key, Message: message, Diagnostics: diagnostics}) +} + +func (r ConditionReporter) Clear(class, key string) { + r.send(callback.ConditionReport{Action: callback.ConditionClear, Class: class, Key: key}) +} + +func (r ConditionReporter) send(rep callback.ConditionReport) { + if r.caps == nil { + return + } + ctx, cancel := background(r.timeout) + defer cancel() + _ = r.caps.ReportCondition(ctx, rep) +} + +// PRCreator implements agent.PRCreator over the callback. +type PRCreator struct { + caps callback.Capabilities +} + +func (p PRCreator) CreatePR(ctx context.Context, repo, head, base, title, body string) (crux.PRResult, error) { + if p.caps == nil { + return crux.PRResult{}, errors.New("hivecap: no capabilities") + } + resp, err := p.caps.CreatePR(ctx, callback.CreatePRRequest{Repo: repo, Head: head, Base: base, Title: title, Body: body}) + if err != nil { + return crux.PRResult{}, fmt.Errorf("hivecap: creating pull request: %w", err) + } + return crux.PRResult{ + Number: resp.Number, URL: resp.URL, Author: resp.Author, + AlreadyExisted: resp.AlreadyExisted, DuplicateTree: resp.DuplicateTree, DuplicateReason: resp.DuplicateReason, + }, nil +} + +// LinearCredentials resolves the Linear credential to inject. Resolve has +// the shape of the proxy's resolver minus the agent.LinearCredential type +// (which lives in crux/agent); the host maps the two fields. Unreachable or +// not configured is an empty credential: the proxy injects nothing. +type LinearCredentials struct { + caps callback.Capabilities + timeout time.Duration +} + +// Resolve returns the access token and API key, both "" when none. +func (l LinearCredentials) Resolve() (accessToken, apiKey string) { + if l.caps == nil { + return "", "" + } + ctx, cancel := background(l.timeout) + defer cancel() + cred, err := l.caps.LinearCredential(ctx) + if err != nil { + return "", "" + } + return cred.AccessToken, cred.APIKey +} + +// StatusSource is callback.Client.Status: the endpoint's reachability as +// the client last saw it. +type StatusSource func() callback.Status + +// Watch turns a StatusSource into the callback.ConditionClassUnavailable +// condition: it polls every interval until ctx ends, reporting Observe on +// the transition to unreachable and Clear on recovery. The reporter is the +// serving process's own (its event stream), not the callback: the condition +// describes the callback being down, so it cannot travel over it. The +// first poll happens at once so a dead endpoint is reported without +// waiting an interval; non-positive interval selects DefaultWatchInterval. +func Watch(ctx context.Context, status StatusSource, reporter crux.ConditionReporter, interval time.Duration) { + if status == nil || reporter == nil { + return + } + if interval <= 0 { + interval = DefaultWatchInterval + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + down := false + for { + st := status() + switch { + case !st.Reachable && !down: + down = true + reporter.Observe(callback.ConditionClassUnavailable, WatchConditionKey, "hive callback endpoint unreachable", unavailableDiagnostics(st)) + case st.Reachable && down: + down = false + reporter.Clear(callback.ConditionClassUnavailable, WatchConditionKey) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +// DefaultWatchInterval is how often Watch samples the status. +const DefaultWatchInterval = 10 * time.Second + +// WatchConditionKey is the condition key Watch reports under; one endpoint +// per runtime, so one key. +const WatchConditionKey = "endpoint" + +func unavailableDiagnostics(st callback.Status) string { + parts := []string{"error=" + st.LastError, fmt.Sprintf("failed_calls=%d", st.FailedCalls)} + if !st.Since.IsZero() { + parts = append(parts, "since="+st.Since.UTC().Format(time.RFC3339)) + } + if !st.LastSuccess.IsZero() { + parts = append(parts, "last_success="+st.LastSuccess.UTC().Format(time.RFC3339)) + } + return strings.Join(parts, " ") +} + +// Policy caches hive's crux.Host policy snapshot (GET /v1/policy) for the +// runtime's policy resolvers. The manager reads several of them while +// holding its own lock, so they never call out: they answer from the last +// snapshot WatchPolicy fetched, and keep answering from it while hive is +// unreachable. The one exception is RepoAutoMergeEnabled, read only on the +// merge-authorisation path, which asks hive per repository and fails closed. +type Policy struct { + src callback.PolicySource + timeout time.Duration + + mu sync.RWMutex + cur callback.Policy + ok bool +} + +// NewPolicy builds the cache over caps; a caps that is not a +// callback.PolicySource never has a snapshot. Non-positive timeout selects +// callback.DefaultTimeout. +func NewPolicy(caps callback.Capabilities, timeout time.Duration) *Policy { + if timeout <= 0 { + timeout = callback.DefaultTimeout + } + src, _ := caps.(callback.PolicySource) + return &Policy{src: src, timeout: timeout} +} + +// Refresh fetches the snapshot and, on success, makes it current. +func (p *Policy) Refresh(ctx context.Context) (callback.Policy, error) { + if p == nil || p.src == nil { + return callback.Policy{}, callback.ErrNotConfigured + } + if _, has := ctx.Deadline(); !has { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, p.timeout) + defer cancel() + } + pol, err := p.src.Policy(ctx) + if err != nil { + return callback.Policy{}, err + } + p.mu.Lock() + p.cur, p.ok = pol, true + p.mu.Unlock() + return pol, nil +} + +// Snapshot is the current snapshot; ok is false before the first +// successful Refresh. +func (p *Policy) Snapshot() (callback.Policy, bool) { + if p == nil { + return callback.Policy{}, false + } + p.mu.RLock() + defer p.mu.RUnlock() + return p.cur, p.ok +} + +// BuildInfo is the SetBuildInfo source: hive's version and commit, or +// "unknown" (the manager's own default) without a snapshot. +func (p *Policy) BuildInfo() (version, commit string) { + if pol, _ := p.Snapshot(); pol.Build != nil { + return pol.Build.Version, pol.Build.Commit + } + return "unknown", "unknown" +} + +// ExplainModeDefault is the SetExplainModeDefaultResolver resolver; "" (no +// hive-wide default) without a snapshot. +func (p *Policy) ExplainModeDefault() string { + if pol, _ := p.Snapshot(); pol.ExplainModeDefault != nil { + return *pol.ExplainModeDefault + } + return "" +} + +// KickLogSource is the SetKickLogSourceResolver resolver; "" without a +// snapshot (the manager normalises it to its default source). +func (p *Policy) KickLogSource() string { + if pol, _ := p.Snapshot(); pol.KickLogSource != nil { + return *pol.KickLogSource + } + return "" +} + +// GatewayBackend is the SetGatewayBackendChecker predicate: backend names a +// gateway hive has configured. The empty backend is the default, never a +// named gateway, matching hive's own predicate. +func (p *Policy) GatewayBackend(backend string) bool { + if backend == "" { + return false + } + pol, _ := p.Snapshot() + if pol.GatewayBackends == nil { + return false + } + for _, name := range *pol.GatewayBackends { + if strings.EqualFold(name, backend) { + return true + } + } + return false +} + +// RepoAutoMergeEnabled is the SetRepoAutoMergeEnabledResolver predicate. It +// asks hive; any failure is false, so an unreachable hive refuses the merge +// rather than allowing it. +func (p *Policy) RepoAutoMergeEnabled(repo string) bool { + if p == nil || p.src == nil { + return false + } + ctx, cancel := background(p.timeout) + defer cancel() + ok, err := p.src.RepoAutoMergeEnabled(ctx, repo) + return err == nil && ok +} + +// DefaultPolicyInterval is how often WatchPolicy re-reads the snapshot, so a +// hive config reload reaches the runtime within it. +const DefaultPolicyInterval = 15 * time.Second + +// WatchPolicy refreshes p every interval until ctx ends and calls apply with +// each snapshot that differs from the last one applied (the first +// successful one included). A failed refresh keeps the previous snapshot. +func WatchPolicy(ctx context.Context, p *Policy, interval time.Duration, apply func(callback.Policy)) { + if p == nil || p.src == nil || apply == nil { + return + } + if interval <= 0 { + interval = DefaultPolicyInterval + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + var last []byte + for { + if pol, err := p.Refresh(ctx); err == nil { + if b, err := json.Marshal(pol); err == nil && (last == nil || !bytes.Equal(b, last)) { + last = b + apply(pol) + } + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} diff --git a/src/crux/hivecap/hivecap_test.go b/src/crux/hivecap/hivecap_test.go new file mode 100644 index 0000000000..4323c23a79 --- /dev/null +++ b/src/crux/hivecap/hivecap_test.go @@ -0,0 +1,669 @@ +package hivecap + +import ( + "context" + "errors" + "net/http" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/proxy" + "github.com/hivecommons/hive/crux/pushbroker" + "github.com/hivecommons/hive/crux/toolapprove" +) + +// The adapters must satisfy the ring's seams without the ring changing. +var ( + _ pushbroker.TokenScoper = TokenScoper{} + _ proxy.GitHubRESTAccounting = GitHubRESTAccounting{} + _ proxy.UsageSink = UsageSink{} + _ proxy.CanaryScanner = CanaryScanner{} + _ toolapprove.TextScanner = TextScanner{} + _ crux.ConditionReporter = ConditionReporter{} + // agent.PRCreator's method set, spelled out so this test does not link + // crux/agent. + _ interface { + CreatePR(ctx context.Context, repo, head, base, title, body string) (crux.PRResult, error) + } = PRCreator{} +) + +// direct is a Capabilities implemented in-process, the in-process mode's +// end of the contract. +type direct struct { + mu sync.Mutex + absent map[string]bool + fail error + mints []callback.MintTokenRequest + rest []callback.RESTRecord + usage []callback.UsageRecord + conds []callback.ConditionReport + detect bool + block bool + expiry time.Time +} + +func (d *direct) Configured(c string) bool { return !d.absent[c] } + +func (d *direct) MintToken(_ context.Context, req callback.MintTokenRequest) (callback.MintTokenResponse, error) { + d.mu.Lock() + defer d.mu.Unlock() + d.mints = append(d.mints, req) + if d.fail != nil { + return callback.MintTokenResponse{}, d.fail + } + exp := d.expiry + if exp.IsZero() { + exp = time.Now().Add(time.Hour) + } + return callback.MintTokenResponse{Token: "ghs_" + req.Tier + "_" + strings.Join(req.Repos, ","), ExpiresAt: exp}, nil +} + +func (d *direct) RecordREST(_ context.Context, rec callback.RESTRecord) error { + d.mu.Lock() + defer d.mu.Unlock() + d.rest = append(d.rest, rec) + return d.fail +} + +func (d *direct) RateLimit(context.Context) (callback.RateLimitSnapshot, error) { + if d.fail != nil { + return callback.RateLimitSnapshot{}, d.fail + } + return callback.RateLimitSnapshot{Remaining: 123, Reset: time.Unix(1_800_000_000, 0).UTC(), OK: true}, nil +} + +func (d *direct) RecordUsage(_ context.Context, rec callback.UsageRecord) error { + d.mu.Lock() + defer d.mu.Unlock() + d.usage = append(d.usage, rec) + return d.fail +} + +func (d *direct) ScanCanary(_ context.Context, req callback.CanaryScanRequest) (callback.CanaryScanResponse, error) { + if d.fail != nil { + return callback.CanaryScanResponse{}, d.fail + } + return callback.CanaryScanResponse{Detected: d.detect, Agent: req.Agent, Source: req.Source}, nil +} + +func (d *direct) ScanText(_ context.Context, req callback.TextScanRequest) (callback.TextScanResponse, error) { + if d.fail != nil { + return callback.TextScanResponse{}, d.fail + } + if d.block { + return callback.TextScanResponse{Blocked: true, CriticalInjection: req.Direction == callback.ScanInput, Findings: []callback.TextFinding{{Kind: "secret", Snippet: "gh***", Rule: "sec-1", High: true}}}, nil + } + return callback.TextScanResponse{}, nil +} + +func (d *direct) ReportCondition(_ context.Context, rep callback.ConditionReport) error { + d.mu.Lock() + defer d.mu.Unlock() + d.conds = append(d.conds, rep) + return d.fail +} + +func (d *direct) LinearCredential(context.Context) (callback.LinearCredential, error) { + if d.fail != nil { + return callback.LinearCredential{}, d.fail + } + return callback.LinearCredential{AccessToken: "lin_tok"}, nil +} + +func (d *direct) CreatePR(_ context.Context, req callback.CreatePRRequest) (callback.CreatePRResponse, error) { + if d.fail != nil { + return callback.CreatePRResponse{}, d.fail + } + return callback.CreatePRResponse{Number: 7, URL: "u/" + req.Repo, Author: "bot", AlreadyExisted: true, DuplicateReason: "issue"}, nil +} + +// overSocket serves d on a unix socket and returns a client for it: the +// sidecar mode's end of the contract. +func overSocket(t *testing.T, d callback.Capabilities) *callback.Client { + t.Helper() + dir, err := os.MkdirTemp("", "hcap") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + if err := os.Chmod(dir, 0o700); err != nil { + t.Fatal(err) + } + sock := filepath.Join(dir, callback.SocketName) + ls, err := callback.Listen(callback.ListenConfig{Socket: sock}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- callback.NewServer(d, "").Serve(ctx, ls) }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + t.Errorf("Serve: %v", err) + } + }) + c, err := callback.NewClient("unix://"+sock, "") + if err != nil { + t.Fatal(err) + } + // Listen bound the socket before Serve started, so one Health call + // blocks in the backlog until Serve accepts. + if _, err := c.Health(context.Background()); err != nil { + t.Fatalf("endpoint never ready: %v", err) + } + return c +} + +// exercise drives every seam over s and returns the observable results, so +// the direct and socket modes can be compared field for field. +type outcome struct { + Token string + TokenErr string + Remaining int + Reset time.Time + SnapOK bool + Leak crux.CanaryLeak + Leaked bool + In, Out toolapprove.TextVerdict + PR crux.PRResult + PRErr string + Lin [2]string + TokenAgent string + TokenTTL int +} + +func exercise(t *testing.T, s *Set) outcome { + t.Helper() + var o outcome + ctx := context.Background() + tok, err := s.TokenScoper("scanner", 10*time.Minute).ScopedTokenForRepos(ctx, "contributor", []string{"r1", "r2"}) + o.Token = tok + if err != nil { + o.TokenErr = err.Error() + } + hdr := http.Header{"X-Ratelimit-Remaining": {"4999"}, "X-Ratelimit-Reset": {"1800000000"}, "X-Github-Request-Id": {"abc"}, "Set-Cookie": {"nope"}} + s.Accounting.RecordRESTRequest("agent:a", "GET", "/repos/o/r", 200, hdr) + s.Accounting.RecordInjectedRESTRequest("agent:a", "POST", "/repos/o/r/issues", 201, hdr) + o.Remaining, o.Reset, o.SnapOK = s.Accounting.CoreRateLimitSnapshot() + s.Usage.Record("a", "m", 1, 2) + s.Usage.RecordCopilot("a", "m", 3, 4) + s.Usage.RecordCopilotWithCache("a", "m", 5, 6, 7, 8) + o.Leak, o.Leaked = s.Canary.Scan("a", "body", "github:POST /x") + o.In = s.Text.ScanInput("ignore all previous") + o.Out = s.Text.ScanOutput("fine") + s.Conditions.Observe("agent_stall", "a", "stalled", "diag") + s.Conditions.Clear("agent_stall", "a") + o.PR, err = s.PullRequest.CreatePR(ctx, "o/r", "h", "v5", "t", "b") + if err != nil { + o.PRErr = err.Error() + } + o.Lin[0], o.Lin[1] = s.Linear.Resolve() + return o +} + +func TestDirectAndSocketParity(t *testing.T) { + // Whole seconds in UTC with no monotonic reading, so the value survives + // the JSON round trip over the socket unchanged; in the future so the + // token is not already expired when the test runs. + exp := time.Now().UTC().Add(time.Hour).Truncate(time.Second) + dDirect := &direct{detect: true, block: true, expiry: exp} + dSocket := &direct{detect: true, block: true, expiry: exp} + got := exercise(t, New(dDirect, 0)) + want := exercise(t, New(overSocket(t, dSocket), 0)) + if !reflect.DeepEqual(got, want) { + t.Fatalf("direct and socket outcomes differ:\n direct: %+v\n socket: %+v", got, want) + } + // And the hive side saw the same requests both ways. + if !reflect.DeepEqual(dDirect.mints, dSocket.mints) || !reflect.DeepEqual(dDirect.usage, dSocket.usage) || !reflect.DeepEqual(dDirect.conds, dSocket.conds) || !reflect.DeepEqual(dDirect.rest, dSocket.rest) { + t.Errorf("hive-side records differ:\n direct: %+v %+v %+v %+v\n socket: %+v %+v %+v %+v", dDirect.mints, dDirect.usage, dDirect.conds, dDirect.rest, dSocket.mints, dSocket.usage, dSocket.conds, dSocket.rest) + } + + // Spot-check the values themselves. + if got.Token != "ghs_contributor_r1,r2" || got.TokenErr != "" { + t.Errorf("token = %q, %q", got.Token, got.TokenErr) + } + m := dDirect.mints[0] + if m.Agent != "scanner" || m.TTLSeconds != 600 || m.Tier != "contributor" { + t.Errorf("mint request = %+v", m) + } + if !got.SnapOK || got.Remaining != 123 || got.Reset.Unix() != 1_800_000_000 { + t.Errorf("snapshot = %d %v %v", got.Remaining, got.Reset, got.SnapOK) + } + if len(dDirect.rest) != 2 || dDirect.rest[0].Injected || !dDirect.rest[1].Injected { + t.Errorf("rest records = %+v", dDirect.rest) + } + // Only X-RateLimit-* headers cross. + for _, r := range dDirect.rest { + if len(r.RateLimit) != 2 || r.RateLimit["X-Ratelimit-Remaining"] != "4999" { + t.Errorf("rate-limit headers = %v", r.RateLimit) + } + } + if len(dDirect.usage) != 3 || dDirect.usage[2].Kind != callback.UsageKindCopilotCache || dDirect.usage[2].CacheCreation != 8 || dDirect.usage[1].Kind != callback.UsageKindCopilot || dDirect.usage[0].Kind != callback.UsageKindGateway { + t.Errorf("usage = %+v", dDirect.usage) + } + if !got.Leaked || got.Leak.Agent != "a" || got.Leak.Source != "github:POST /x" || got.Leak.Token != "" { + t.Errorf("leak = %+v %v", got.Leak, got.Leaked) + } + if !got.In.Blocked || !got.In.CriticalInjection || len(got.In.Findings) != 1 || got.In.Findings[0].Rule != "sec-1" || !got.In.Findings[0].High || got.Out.CriticalInjection { + t.Errorf("verdicts = %+v / %+v", got.In, got.Out) + } + if len(dDirect.conds) != 2 || dDirect.conds[0].Action != callback.ConditionObserve || dDirect.conds[0].Diagnostics != "diag" || dDirect.conds[1].Action != callback.ConditionClear { + t.Errorf("conditions = %+v", dDirect.conds) + } + if got.PR.Number != 7 || got.PR.URL != "u/o/r" || !got.PR.AlreadyExisted || got.PR.DuplicateReason != "issue" || !got.PR.IsDuplicate() { + t.Errorf("pr = %+v", got.PR) + } + if got.Lin != [2]string{"lin_tok", ""} { + t.Errorf("linear = %v", got.Lin) + } +} + +func TestUnreachableFailsClosed(t *testing.T) { + dir, err := os.MkdirTemp("", "hcap") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.RemoveAll(dir) }() + c, err := callback.NewClient("unix://"+filepath.Join(dir, "nobody.sock"), "") + if err != nil { + t.Fatal(err) + } + s := New(c, 200*time.Millisecond) + o := exercise(t, s) + + if o.Token != "" || !strings.Contains(o.TokenErr, callback.ErrUnavailable.Error()) { + t.Errorf("token must fail closed: %q, %q", o.Token, o.TokenErr) + } + if o.SnapOK || o.Remaining != 0 || !o.Reset.IsZero() { + t.Errorf("snapshot must be ok=false: %d %v %v", o.Remaining, o.Reset, o.SnapOK) + } + if !o.Leaked || o.Leak.Agent != "a" || !strings.HasSuffix(o.Leak.Source, UnavailableSourceSuffix) || o.Leak.Token != "" { + t.Errorf("canary must report a detection: %+v %v", o.Leak, o.Leaked) + } + for _, v := range []toolapprove.TextVerdict{o.In, o.Out} { + if !v.Blocked || len(v.Findings) != 1 || v.Findings[0].Kind != TextFindingKind || v.Findings[0].Rule != TextFindingRule { + t.Errorf("text verdict must block: %+v", v) + } + } + if o.PR != (crux.PRResult{}) || !strings.Contains(o.PRErr, callback.ErrUnavailable.Error()) { + t.Errorf("pr must fail: %+v, %q", o.PR, o.PRErr) + } + if o.Lin != [2]string{} { + t.Errorf("linear must be empty: %v", o.Lin) + } + st := c.Status() + if st.Reachable || st.FailedCalls == 0 { + t.Errorf("client status = %+v", st) + } + // Sinks never panic and never block past the timeout. + start := time.Now() + s.Usage.Record("a", "m", 1, 1) + s.Conditions.Observe("c", "k", "m", "d") + if time.Since(start) > 2*time.Second { + t.Error("sinks blocked past the timeout") + } +} + +func TestNotConfiguredIsTheNilSeam(t *testing.T) { + d := &direct{absent: map[string]bool{callback.CapabilityTokens: true, callback.CapabilityCanary: true, callback.CapabilityLinear: true, callback.CapabilityPulls: true}, detect: true} + for name, caps := range map[string]callback.Capabilities{"direct": d, "socket": overSocket(t, d)} { + s := New(caps, 0) + _, err := s.TokenScoper("a", 0).ScopedTokenForRepos(context.Background(), "", nil) + if name == "socket" && !errors.Is(err, callback.ErrNotConfigured) { + t.Errorf("%s: no App should be ErrNotConfigured: %v", name, err) + } + if name == "socket" { + // No registry: scans nothing, like a nil registry; NOT a refusal. + if leak, ok := s.Canary.Scan("a", "t", "s"); ok || leak != (crux.CanaryLeak{}) { + t.Errorf("%s: absent registry scanned: %+v %v", name, leak, ok) + } + if a, k := s.Linear.Resolve(); a != "" || k != "" { + t.Errorf("%s: absent Linear returned a credential", name) + } + if _, err := s.PullRequest.CreatePR(context.Background(), "r", "h", "b", "t", ""); !errors.Is(err, callback.ErrNotConfigured) { + t.Errorf("%s: absent PR creator = %v", name, err) + } + } + } + // The direct implementation above has no 404 layer, so its own + // ErrNotConfigured is the signal. + d2 := &direct{fail: callback.ErrNotConfigured} + s := New(d2, 0) + if leak, ok := s.Canary.Scan("a", "t", "s"); ok || leak != (crux.CanaryLeak{}) { + t.Errorf("direct ErrNotConfigured scanned: %+v %v", leak, ok) + } +} + +func TestCapabilityErrorsFailClosed(t *testing.T) { + d := &direct{fail: errors.New("github: 403")} + s := New(d, 0) + if tok, err := s.TokenScoper("a", 0).ScopedTokenForRepos(context.Background(), "t", nil); tok != "" || err == nil { + t.Errorf("token = %q, %v", tok, err) + } + if _, _, ok := s.Accounting.CoreRateLimitSnapshot(); ok { + t.Error("snapshot ok on error") + } + if _, ok := s.Canary.Scan("a", "t", "s"); !ok { + t.Error("canary error must be a detection") + } + if v := s.Text.ScanInput("x"); !v.Blocked || !strings.Contains(v.Findings[0].Snippet, "github: 403") { + t.Errorf("text error must block with the cause: %+v", v) + } + if _, err := s.PullRequest.CreatePR(context.Background(), "r", "h", "b", "t", ""); err == nil { + t.Error("pr error swallowed") + } + if a, k := s.Linear.Resolve(); a != "" || k != "" { + t.Error("linear error returned a credential") + } + // Sinks swallow. + s.Usage.Record("a", "m", 1, 1) + s.Accounting.RecordRESTRequest("c", "GET", "/", 200, nil) + s.Conditions.Clear("c", "k") + if len(d.usage) != 1 || len(d.rest) != 1 || len(d.conds) != 1 { + t.Error("sinks must still deliver when the capability returns an error") + } +} + +func TestTokenScoperExpiryAndTTL(t *testing.T) { + d := &direct{expiry: time.Now().Add(-time.Minute)} + if tok, err := New(d, 0).TokenScoper("a", 0).ScopedTokenForRepos(context.Background(), "t", nil); tok != "" || err == nil || !strings.Contains(err.Error(), "already expired") { + t.Errorf("expired token accepted: %q, %v", tok, err) + } + if d.mints[0].TTLSeconds != 0 || d.mints[0].Agent != "a" { + t.Errorf("zero ttl must be omitted: %+v", d.mints[0]) + } + // Snapshot ok=false from hive is ok=false here. + okFalse := &snapNotOK{direct: &direct{}} + if _, _, ok := New(okFalse, 0).Accounting.CoreRateLimitSnapshot(); ok { + t.Error("ok=false from hive reported as ok") + } + // A detection with empty agent/source falls back to the request's. + bare := &bareDetect{direct: &direct{}} + leak, ok := New(bare, 0).Canary.Scan("agentX", "t", "srcY") + if !ok || leak.Agent != "agentX" || leak.Source != "srcY" { + t.Errorf("bare detection = %+v %v", leak, ok) + } +} + +type snapNotOK struct{ *direct } + +func (snapNotOK) RateLimit(context.Context) (callback.RateLimitSnapshot, error) { + return callback.RateLimitSnapshot{Remaining: 5}, nil +} + +type bareDetect struct{ *direct } + +func (bareDetect) ScanCanary(context.Context, callback.CanaryScanRequest) (callback.CanaryScanResponse, error) { + return callback.CanaryScanResponse{Detected: true}, nil +} + +func TestNilCapabilities(t *testing.T) { + s := New(nil, -1) + if s.timeout != callback.DefaultTimeout { + t.Errorf("timeout = %v", s.timeout) + } + if _, err := s.TokenScoper("a", 0).ScopedTokenForRepos(context.Background(), "t", nil); err == nil { + t.Error("nil caps minted") + } + if _, _, ok := s.Accounting.CoreRateLimitSnapshot(); ok { + t.Error("nil caps snapshot ok") + } + if _, ok := s.Canary.Scan("a", "t", "s"); ok { + t.Error("nil caps is a nil registry: scans nothing") + } + if v := s.Text.ScanOutput("x"); !v.Blocked { + t.Error("nil caps text scan must block") + } + if _, err := s.PullRequest.CreatePR(context.Background(), "r", "h", "b", "t", ""); err == nil { + t.Error("nil caps created a PR") + } + if a, k := s.Linear.Resolve(); a != "" || k != "" { + t.Error("nil caps linear") + } + s.Usage.Record("a", "m", 1, 1) + s.Accounting.RecordRESTRequest("c", "GET", "/", 200, nil) + s.Conditions.Observe("c", "k", "m", "d") +} + +func TestRateLimitHeadersFilter(t *testing.T) { + if got := rateLimitHeaders(nil); got != nil { + t.Errorf("nil header = %v", got) + } + h := http.Header{"x-ratelimit-limit": {"5000", "ignored"}, "X-Ratelimit-Empty": {}, "Authorization": {"Bearer x"}} + got := rateLimitHeaders(h) + if len(got) != 1 || got["X-Ratelimit-Limit"] != "5000" { + t.Errorf("filtered = %v", got) + } +} + +type watchReporter struct { + mu sync.Mutex + observed []string + cleared []string + seen chan struct{} +} + +func (r *watchReporter) Observe(class, key, message, diagnostics string) { + r.mu.Lock() + r.observed = append(r.observed, class+"/"+key+": "+message+" ["+diagnostics+"]") + r.mu.Unlock() + r.seen <- struct{}{} +} + +func (r *watchReporter) Clear(class, key string) { + r.mu.Lock() + r.cleared = append(r.cleared, class+"/"+key) + r.mu.Unlock() + r.seen <- struct{}{} +} + +func TestWatchReportsTransitionsOnly(t *testing.T) { + var mu sync.Mutex + st := callback.Status{Reachable: true} + set := func(s callback.Status) { mu.Lock(); st = s; mu.Unlock() } + source := func() callback.Status { mu.Lock(); defer mu.Unlock(); return st } + rep := &watchReporter{seen: make(chan struct{}, 8)} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { Watch(ctx, source, rep, time.Millisecond); close(done) }() + + since := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + set(callback.Status{LastError: "dial: refused", Since: since, FailedCalls: 3, LastSuccess: since.Add(-time.Hour)}) + <-rep.seen + set(callback.Status{LastError: "dial: refused", Since: since, FailedCalls: 9}) + // Still down: a second Observe must not follow; wait a few polls. + settle := time.NewTimer(20 * time.Millisecond) + select { + case <-rep.seen: + t.Fatal("repeated Observe while still unreachable") + case <-settle.C: + } + set(callback.Status{Reachable: true}) + <-rep.seen + cancel() + <-done + + rep.mu.Lock() + defer rep.mu.Unlock() + if len(rep.observed) != 1 || len(rep.cleared) != 1 { + t.Fatalf("observed %v cleared %v", rep.observed, rep.cleared) + } + want := callback.ConditionClassUnavailable + "/" + WatchConditionKey + ": hive callback endpoint unreachable [error=dial: refused failed_calls=3 since=2026-01-02T03:04:05Z last_success=2026-01-02T02:04:05Z]" + if rep.observed[0] != want { + t.Errorf("observe\n got %s\nwant %s", rep.observed[0], want) + } + if rep.cleared[0] != callback.ConditionClassUnavailable+"/"+WatchConditionKey { + t.Errorf("clear %s", rep.cleared[0]) + } +} + +func TestWatchNilArgumentsReturn(t *testing.T) { + Watch(context.Background(), nil, &watchReporter{}, 0) + Watch(context.Background(), func() callback.Status { return callback.Status{} }, nil, 0) + if unavailableDiagnostics(callback.Status{LastError: "x"}) != "error=x failed_calls=0" { + t.Fatal("diagnostics") + } +} + +// policyDirect is direct plus the policy half of the contract. +type policyDirect struct { + *direct + pmu sync.Mutex + policy callback.Policy + pfail error +} + +func (p *policyDirect) Policy(context.Context) (callback.Policy, error) { + p.pmu.Lock() + defer p.pmu.Unlock() + return p.policy, p.pfail +} + +func (p *policyDirect) RepoAutoMergeEnabled(_ context.Context, repo string) (bool, error) { + p.pmu.Lock() + defer p.pmu.Unlock() + return repo == "o/merge", p.pfail +} + +func (p *policyDirect) set(pol callback.Policy, fail error) { + p.pmu.Lock() + defer p.pmu.Unlock() + p.policy, p.pfail = pol, fail +} + +// The policy resolvers answer from the cached snapshot in both modes, keep +// the last snapshot through an outage, and fail the auto-merge question +// closed. +func TestPolicyResolvers(t *testing.T) { + explain, kick := "on", "transcript" + gateways := []string{"LiteLLM-EU"} + pol := callback.Policy{ + Build: &callback.BuildInfo{Version: "v5.2", Commit: "abc"}, + ExplainModeDefault: &explain, + KickLogSource: &kick, + GatewayBackends: &gateways, + RepoAutoMerge: true, + } + d := &policyDirect{direct: &direct{}, policy: pol} + for name, caps := range map[string]callback.Capabilities{"direct": d, "socket": overSocket(t, d)} { + p := New(caps, time.Second).Policy + if v, c := p.BuildInfo(); v != "unknown" || c != "unknown" || p.ExplainModeDefault() != "" || p.KickLogSource() != "" || p.GatewayBackend("litellm-eu") { + t.Fatalf("%s: resolvers before the first refresh answered from nothing", name) + } + if _, err := p.Refresh(context.Background()); err != nil { + t.Fatalf("%s: Refresh: %v", name, err) + } + if v, c := p.BuildInfo(); v != "v5.2" || c != "abc" { + t.Errorf("%s: BuildInfo = %s %s", name, v, c) + } + if p.ExplainModeDefault() != "on" || p.KickLogSource() != "transcript" { + t.Errorf("%s: explain=%q kick=%q", name, p.ExplainModeDefault(), p.KickLogSource()) + } + if !p.GatewayBackend("litellm-eu") || p.GatewayBackend("other") || p.GatewayBackend("") { + t.Errorf("%s: gateway predicate wrong", name) + } + if !p.RepoAutoMergeEnabled("o/merge") || p.RepoAutoMergeEnabled("o/other") { + t.Errorf("%s: auto-merge predicate wrong", name) + } + d.set(pol, errors.New("hive down")) + if _, err := p.Refresh(context.Background()); err == nil { + t.Errorf("%s: Refresh through an outage succeeded", name) + } + if p.ExplainModeDefault() != "on" { + t.Errorf("%s: outage dropped the last snapshot", name) + } + if p.RepoAutoMergeEnabled("o/merge") { + t.Errorf("%s: auto-merge allowed while hive is down", name) + } + d.set(pol, nil) + } +} + +// A Capabilities without the policy half never yields a snapshot, and its +// auto-merge answer is a refusal. +func TestPolicyWithoutSource(t *testing.T) { + p := New(&direct{}, time.Second).Policy + if _, err := p.Refresh(context.Background()); !errors.Is(err, callback.ErrNotConfigured) { + t.Fatalf("Refresh = %v, want ErrNotConfigured", err) + } + if _, ok := p.Snapshot(); ok { + t.Fatal("snapshot without a source") + } + if p.RepoAutoMergeEnabled("o/merge") { + t.Fatal("auto-merge allowed without a source") + } + WatchPolicy(context.Background(), p, time.Millisecond, func(callback.Policy) { t.Fatal("applied without a source") }) +} + +// WatchPolicy applies the first snapshot and each change, not repeats. +func TestWatchPolicyAppliesChanges(t *testing.T) { + owned := false + d := &policyDirect{direct: &direct{}, policy: callback.Policy{DeadSessionRecoveryOwnedElsewhere: &owned}} + p := New(d, time.Second).Policy + ctx, cancel := context.WithCancel(context.Background()) + applied := make(chan callback.Policy, 16) + done := make(chan struct{}) + go func() { + defer close(done) + WatchPolicy(ctx, p, time.Millisecond, func(pol callback.Policy) { applied <- pol }) + }() + first := <-applied + if first.DeadSessionRecoveryOwnedElsewhere == nil || *first.DeadSessionRecoveryOwnedElsewhere { + t.Fatalf("first = %+v", first) + } + nowOwned := true + d.set(callback.Policy{DeadSessionRecoveryOwnedElsewhere: &nowOwned}, nil) + second := <-applied + if second.DeadSessionRecoveryOwnedElsewhere == nil || !*second.DeadSessionRecoveryOwnedElsewhere { + t.Fatalf("second = %+v", second) + } + cancel() + <-done + select { + case extra := <-applied: + t.Fatalf("unchanged snapshot re-applied: %+v", extra) + default: + } +} + +// The zero and nil forms of Policy are safe and answer the defaults. +func TestPolicyEdges(t *testing.T) { + if p := NewPolicy(nil, 0); p.timeout != callback.DefaultTimeout || p.src != nil { + t.Fatalf("NewPolicy(nil, 0) = %+v", p) + } + var nilPolicy *Policy + if _, err := nilPolicy.Refresh(context.Background()); !errors.Is(err, callback.ErrNotConfigured) { + t.Fatalf("nil Refresh = %v", err) + } + if _, ok := nilPolicy.Snapshot(); ok || nilPolicy.RepoAutoMergeEnabled("o/merge") || nilPolicy.GatewayBackend("x") { + t.Fatal("nil policy answered") + } + if v, c := nilPolicy.BuildInfo(); v != "unknown" || c != "unknown" { + t.Fatalf("nil BuildInfo = %s %s", v, c) + } + WatchPolicy(context.Background(), nilPolicy, 0, func(callback.Policy) { t.Fatal("nil policy applied") }) + + d := &policyDirect{direct: &direct{}} + p := NewPolicy(d, time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := p.Refresh(ctx); err != nil { + t.Fatalf("Refresh with a deadline: %v", err) + } + WatchPolicy(context.Background(), p, 0, nil) + done, stop := context.WithCancel(context.Background()) + stop() + applied := 0 + WatchPolicy(done, p, 0, func(callback.Policy) { applied++ }) + if applied != 1 { + t.Fatalf("applied %d times before a cancelled watch returned, want 1", applied) + } +} diff --git a/src/crux/hivecap/host.go b/src/crux/hivecap/host.go new file mode 100644 index 0000000000..aeb5755bd7 --- /dev/null +++ b/src/crux/hivecap/host.go @@ -0,0 +1,248 @@ +package hivecap + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/effects" +) + +// This file carries the two crux.Host setters that are calls into hive +// rather than values (hivecommons/hive#14138 stage 4): SetInferenceCallbacks +// becomes InferenceRoutes, a desired-state pusher over +// callback.InferenceRouter, and SetSandboxMutationBoundary becomes +// MutationBoundary, an effects.Boundary over callback.EffectBoundary. + +// DefaultRouteResync is how often InferenceRoutes re-pushes every route it +// holds, so a hive that restarted (and so lost its proxy's route table) +// learns them again within it. +const DefaultRouteResync = time.Minute + +// routeRetry is the backoff after a failed push; the next change or resync +// tick retries sooner. +const routeRetry = 2 * time.Second + +// InferenceRoutes is the sidecar's SetInferenceCallbacks: Set and Clear are +// the two callbacks the manager installs, and they return at once. The +// manager calls them on its launch path, sometimes under its own lock, so +// they must not wait on hive; instead they record the desired route and a +// worker (Run) pushes it, retrying until hive has it. A route hive never +// receives is fail-closed by construction: the proxy has no upstream for +// the agent and refuses its inference traffic. A hive without the +// capability (callback.ErrNotConfigured) is treated as the nil callback: +// the routes are kept but never pushed again until the next resync. +type InferenceRoutes struct { + caps callback.InferenceRouter + timeout time.Duration + + mu sync.Mutex + desired map[string]*callback.InferenceRoute // nil value = cleared + dirty map[string]bool + wake chan struct{} + pushes int + lastErr error +} + +// NewInferenceRoutes builds the pusher over caps; nil caps (or one that is +// not an InferenceRouter) records routes and pushes nothing. Non-positive +// timeout selects callback.DefaultTimeout. +func NewInferenceRoutes(caps callback.Capabilities, timeout time.Duration) *InferenceRoutes { + if timeout <= 0 { + timeout = callback.DefaultTimeout + } + r := &InferenceRoutes{timeout: timeout, desired: map[string]*callback.InferenceRoute{}, dirty: map[string]bool{}, wake: make(chan struct{}, 1)} + if caps != nil { + r.caps, _ = caps.(callback.InferenceRouter) + } + return r +} + +// Set is the setRoute callback: agent's inference traffic goes to backend +// and model. +func (r *InferenceRoutes) Set(agent, backend, model string) { + r.mu.Lock() + r.desired[agent] = &callback.InferenceRoute{Backend: backend, Model: model} + r.dirty[agent] = true + r.mu.Unlock() + r.signal() +} + +// Clear is the clearRoute callback: agent has no inference route. +func (r *InferenceRoutes) Clear(agent string) { + r.mu.Lock() + r.desired[agent] = nil + r.dirty[agent] = true + r.mu.Unlock() + r.signal() +} + +func (r *InferenceRoutes) signal() { + select { + case r.wake <- struct{}{}: + default: + } +} + +// Pending reports how many routes hive does not have yet, and the last push +// error (nil when the last push succeeded). +func (r *InferenceRoutes) Pending() (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.dirty), r.lastErr +} + +// Run pushes dirty routes until ctx ends: at once on every Set/Clear, after +// routeRetry while a push fails, and every resync (non-positive selects +// DefaultRouteResync) for every route held, dirty or not. +func (r *InferenceRoutes) Run(ctx context.Context, resync time.Duration) { + if r == nil || r.caps == nil { + return + } + if resync <= 0 { + resync = DefaultRouteResync + } + ticker := time.NewTicker(resync) + defer ticker.Stop() + var retry <-chan time.Time + for { + failed := r.flush(ctx) + retry = nil + if failed { + retry = time.After(routeRetry) + } + select { + case <-ctx.Done(): + return + case <-r.wake: + case <-retry: + case <-ticker.C: + r.mu.Lock() + for agent := range r.desired { + r.dirty[agent] = true + } + r.mu.Unlock() + } + } +} + +// flush pushes every dirty route once; it reports whether any push failed +// (and so is still dirty). +func (r *InferenceRoutes) flush(ctx context.Context) bool { + r.mu.Lock() + batch := make(map[string]*callback.InferenceRoute, len(r.dirty)) + for agent := range r.dirty { + batch[agent] = r.desired[agent] + } + r.mu.Unlock() + failed := false + for agent, route := range batch { + err := r.push(ctx, agent, route) + r.mu.Lock() + r.pushes++ + r.lastErr = err + if err == nil && batchStill(r.desired[agent], route) { + delete(r.dirty, agent) + // A cleared route hive has acknowledged needs no further entry. + if route == nil { + delete(r.desired, agent) + } + } + r.mu.Unlock() + if err != nil && !errors.Is(err, callback.ErrNotConfigured) { + failed = true + } + } + return failed +} + +// batchStill reports whether the desired route is still the one just +// pushed (a Set during the push leaves the entry dirty). +func batchStill(current, pushed *callback.InferenceRoute) bool { + if current == nil || pushed == nil { + return current == pushed + } + return *current == *pushed +} + +func (r *InferenceRoutes) push(ctx context.Context, agent string, route *callback.InferenceRoute) error { + ctx, cancel := context.WithTimeout(ctx, r.timeout) + defer cancel() + if route == nil { + return r.caps.ClearInferenceRoute(ctx, agent) + } + return r.caps.SetInferenceRoute(ctx, agent, *route) +} + +// MutationBoundary implements effects.Boundary over the callback: Execute +// begins the claim on hive, runs the effect here, and ends the claim with +// the outcome, so hive's journal and fence see a sidecar's sandbox pushes +// exactly as an in-process runtime's. It fails closed: an unreachable hive +// refuses the effect (the push does not happen); a hive without the +// capability runs the effect unjournaled, which is the nil boundary the +// in-process wiring also tolerates. +type MutationBoundary struct { + caps callback.EffectBoundary + // EndTimeout bounds the end call, which must get through even when the + // caller's context has already been cancelled by the effect's failure; + // zero means callback.DefaultTimeout. + EndTimeout time.Duration +} + +// NewMutationBoundary builds the boundary over caps; a caps that is not an +// EffectBoundary runs effects unjournaled. +func NewMutationBoundary(caps callback.Capabilities) MutationBoundary { + var b MutationBoundary + if caps != nil { + b.caps, _ = caps.(callback.EffectBoundary) + } + return b +} + +var _ effects.Boundary = MutationBoundary{} + +// Execute is effects.Boundary.Execute. +func (b MutationBoundary) Execute(ctx context.Context, claim effects.Claim, effect func(context.Context) (effects.Result, error)) (effects.Result, error) { + if b.caps == nil { + return effect(ctx) + } + begun, err := b.caps.BeginEffect(ctx, callback.EffectClaim{Repo: claim.Repo, Kind: claim.Kind, Target: claim.Target, Actor: claim.Actor, Inputs: claim.Inputs}) + if err != nil { + if errors.Is(err, callback.ErrNotConfigured) { + return effect(ctx) + } + if errors.Is(err, callback.ErrEffectDenied) { + return effects.Result{}, fmt.Errorf("%w: %v", effects.ErrDenied, err) + } + return effects.Result{}, fmt.Errorf("hivecap: mutation boundary unavailable, effect refused: %w", err) + } + res, effErr := effect(ctx) + end := callback.EndEffectRequest{Provenance: res.Provenance} + if effErr != nil { + end.Error = effErr.Error() + } + timeout := b.EndTimeout + if timeout <= 0 { + timeout = callback.DefaultTimeout + } + // The end call carries the outcome hive must journal even when the + // caller's ctx is already done, so it gets its own deadline. + endCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout) + defer cancel() + ended, endErr := b.caps.EndEffect(endCtx, begun.ID, end) + switch { + case effErr != nil: + return res, effErr + case endErr != nil: + return res, fmt.Errorf("hivecap: effect ran but hive could not journal it: %w", endErr) + case ended.Error != "": + return res, errors.New(ended.Error) + } + if ended.Provenance != "" { + res.Provenance = ended.Provenance + } + return res, nil +} diff --git a/src/crux/hivecap/host_test.go b/src/crux/hivecap/host_test.go new file mode 100644 index 0000000000..56734149de --- /dev/null +++ b/src/crux/hivecap/host_test.go @@ -0,0 +1,319 @@ +package hivecap + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/effects" +) + +// hostDirect is a direct that also owns the proxy route table and the +// mutation boundary, hive's end of the stage-4 families. +type hostDirect struct { + *direct + hmu sync.Mutex + routes map[string]callback.InferenceRoute + rfail error + begun []callback.EffectClaim + ended map[string]callback.EndEffectRequest + deny bool + efail error + endFail error +} + +func newHostDirect() *hostDirect { + return &hostDirect{direct: &direct{}, routes: map[string]callback.InferenceRoute{}, ended: map[string]callback.EndEffectRequest{}} +} + +func (h *hostDirect) SetInferenceRoute(_ context.Context, agent string, route callback.InferenceRoute) error { + h.hmu.Lock() + defer h.hmu.Unlock() + if h.rfail != nil { + return h.rfail + } + h.routes[agent] = route + return nil +} + +func (h *hostDirect) ClearInferenceRoute(_ context.Context, agent string) error { + h.hmu.Lock() + defer h.hmu.Unlock() + if h.rfail != nil { + return h.rfail + } + delete(h.routes, agent) + return nil +} + +func (h *hostDirect) BeginEffect(_ context.Context, claim callback.EffectClaim) (callback.BeginEffectResponse, error) { + h.hmu.Lock() + defer h.hmu.Unlock() + if h.efail != nil { + return callback.BeginEffectResponse{}, h.efail + } + if h.deny { + return callback.BeginEffectResponse{}, callback.ErrEffectDenied + } + h.begun = append(h.begun, claim) + return callback.BeginEffectResponse{ID: "e1"}, nil +} + +func (h *hostDirect) EndEffect(_ context.Context, id string, req callback.EndEffectRequest) (callback.EndEffectResponse, error) { + h.hmu.Lock() + defer h.hmu.Unlock() + if h.endFail != nil { + return callback.EndEffectResponse{}, h.endFail + } + h.ended[id] = req + return callback.EndEffectResponse{Provenance: "journal:" + req.Provenance, Error: req.Error}, nil +} + +func (h *hostDirect) route(agent string) (callback.InferenceRoute, bool) { + h.hmu.Lock() + defer h.hmu.Unlock() + r, ok := h.routes[agent] + return r, ok +} + +func (h *hostDirect) setRouteFail(err error) { + h.hmu.Lock() + defer h.hmu.Unlock() + h.rfail = err +} + +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.After(5 * time.Second) + tick := time.NewTicker(5 * time.Millisecond) + defer tick.Stop() + for { + if cond() { + return + } + select { + case <-tick.C: + case <-deadline: + t.Fatalf("timed out waiting for %s", what) + } + } +} + +// Set/Clear return at once and the worker delivers the route to hive in +// both modes; a cleared route disappears from hive. +func TestInferenceRoutesPushAndClear(t *testing.T) { + for _, mode := range []string{"direct", "socket"} { + t.Run(mode, func(t *testing.T) { + h := newHostDirect() + var caps callback.Capabilities = h + if mode == "socket" { + caps = overSocket(t, h) + } + r := NewInferenceRoutes(caps, time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { defer close(done); r.Run(ctx, time.Hour) }() + r.Set("a", "litellm", "gpt") + waitFor(t, "route pushed", func() bool { + got, ok := h.route("a") + return ok && got == callback.InferenceRoute{Backend: "litellm", Model: "gpt"} + }) + if n, err := r.Pending(); n != 0 || err != nil { + t.Fatalf("Pending = %d, %v", n, err) + } + r.Clear("a") + waitFor(t, "route cleared", func() bool { _, ok := h.route("a"); return !ok }) + r.mu.Lock() + _, kept := r.desired["a"] + r.mu.Unlock() + if kept { + t.Fatal("acknowledged clear should drop the desired entry") + } + cancel() + <-done + }) + } +} + +// A push that fails stays pending and is retried until hive takes it; a +// Set during the outage supersedes the earlier route. +func TestInferenceRoutesRetryUntilDelivered(t *testing.T) { + h := newHostDirect() + h.setRouteFail(errors.New("proxy down")) + r := NewInferenceRoutes(h, time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go r.Run(ctx, time.Hour) + r.Set("a", "old", "") + waitFor(t, "failed push recorded", func() bool { n, err := r.Pending(); return n == 1 && err != nil }) + r.Set("a", "new", "m") + h.setRouteFail(nil) + waitFor(t, "route delivered after recovery", func() bool { + got, ok := h.route("a") + return ok && got.Backend == "new" + }) + if n, err := r.Pending(); n != 0 || err != nil { + t.Fatalf("Pending after recovery = %d, %v", n, err) + } +} + +// The periodic resync re-pushes every held route so a hive that lost its +// table gets it back without a manager change. +func TestInferenceRoutesResync(t *testing.T) { + h := newHostDirect() + r := NewInferenceRoutes(h, time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go r.Run(ctx, 20*time.Millisecond) + r.Set("a", "litellm", "") + waitFor(t, "initial push", func() bool { _, ok := h.route("a"); return ok }) + h.hmu.Lock() + delete(h.routes, "a") // hive restarted and forgot + h.hmu.Unlock() + waitFor(t, "resync re-push", func() bool { _, ok := h.route("a"); return ok }) +} + +// Without the capability the pusher holds the routes and never pushes: +// nil caps, a Capabilities that is not an InferenceRouter, and a hive that +// answers not_configured all behave the same and none blocks. +func TestInferenceRoutesWithoutCapability(t *testing.T) { + var nilRoutes *InferenceRoutes + nilRoutes.Run(context.Background(), 0) // returns at once + for _, caps := range []callback.Capabilities{nil, &direct{}} { + r := NewInferenceRoutes(caps, 0) + r.Set("a", "b", "") + r.Run(context.Background(), 0) // no router: returns at once + if n, _ := r.Pending(); n != 1 { + t.Fatalf("Pending without router = %d, want held", n) + } + } + h := newHostDirect() + h.absent = map[string]bool{callback.CapabilityInferenceRoutes: true} + r := NewInferenceRoutes(overSocket(t, h), time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go r.Run(ctx, time.Hour) + r.Set("a", "b", "") + waitFor(t, "not_configured push", func() bool { n, err := r.Pending(); return n == 1 && errors.Is(err, callback.ErrNotConfigured) }) + if _, ok := h.route("a"); ok { + t.Fatal("not_configured must not deliver") + } +} + +func TestBatchStill(t *testing.T) { + a, b := &callback.InferenceRoute{Backend: "a"}, &callback.InferenceRoute{Backend: "b"} + if !batchStill(nil, nil) || batchStill(a, nil) || batchStill(nil, a) || batchStill(a, b) || !batchStill(a, &callback.InferenceRoute{Backend: "a"}) { + t.Fatal("batchStill misjudged") + } +} + +// The mutation boundary journals a sandbox push through hive in both modes: +// begin before, end after with provenance or the effect's error. +func TestMutationBoundaryJournals(t *testing.T) { + for _, mode := range []string{"direct", "socket"} { + t.Run(mode, func(t *testing.T) { + h := newHostDirect() + var caps callback.Capabilities = h + if mode == "socket" { + caps = overSocket(t, h) + } + b := NewMutationBoundary(caps) + claim := effects.Claim{Repo: "o/r", Kind: effects.KindBranchPush, Target: "feat", Actor: "agent", Inputs: map[string]string{"commit": "abc"}} + res, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + return effects.Result{Provenance: "abc"}, nil + }) + if err != nil || res.Provenance != "journal:abc" { + t.Fatalf("Execute = %+v, %v", res, err) + } + h.hmu.Lock() + begun, ended := h.begun, h.ended["e1"] + h.hmu.Unlock() + if len(begun) != 1 || begun[0].Repo != "o/r" || begun[0].Kind != effects.KindBranchPush || begun[0].Inputs["commit"] != "abc" { + t.Fatalf("begun = %+v", begun) + } + if ended.Provenance != "abc" || ended.Error != "" { + t.Fatalf("ended = %+v", ended) + } + // The effect's failure is reported to hive and returned. + _, err = b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + return effects.Result{}, errors.New("push rejected") + }) + if err == nil || err.Error() != "push rejected" { + t.Fatalf("failed effect = %v", err) + } + h.hmu.Lock() + ended = h.ended["e1"] + h.hmu.Unlock() + if ended.Error != "push rejected" { + t.Fatalf("ended after failure = %+v", ended) + } + }) + } +} + +// Denied, unreachable, and not_configured each resolve as the in-process +// boundary would: ErrDenied, refuse, run unjournaled. +func TestMutationBoundaryFailureModes(t *testing.T) { + ran := 0 + effect := func(context.Context) (effects.Result, error) { ran++; return effects.Result{Provenance: "p"}, nil } + claim := effects.Claim{Repo: "o/r", Kind: effects.KindBranchPush, Target: "t"} + ctx := context.Background() + + h := newHostDirect() + h.deny = true + if _, err := NewMutationBoundary(overSocket(t, h)).Execute(ctx, claim, effect); !errors.Is(err, effects.ErrDenied) || ran != 0 { + t.Fatalf("denied = %v ran=%d", err, ran) + } + h.deny, h.efail = false, errors.New("journal broken") + if _, err := NewMutationBoundary(h).Execute(ctx, claim, effect); err == nil || errors.Is(err, effects.ErrDenied) || ran != 0 { + t.Fatalf("begin failure = %v ran=%d", err, ran) + } + dead, err := callback.NewClient("unix:///nonexistent/"+callback.SocketName, "") + if err != nil { + t.Fatal(err) + } + if _, err := NewMutationBoundary(dead).Execute(ctx, claim, effect); !errors.Is(err, callback.ErrUnavailable) || ran != 0 { + t.Fatalf("unreachable = %v ran=%d, want refused", err, ran) + } + h.efail = nil + h.absent = map[string]bool{callback.CapabilityEffects: true} + res, err := NewMutationBoundary(overSocket(t, h)).Execute(ctx, claim, effect) + if err != nil || res.Provenance != "p" || ran != 1 { + t.Fatalf("not_configured = %+v, %v ran=%d, want unjournaled run", res, err, ran) + } + for _, caps := range []callback.Capabilities{nil, &direct{}} { + if _, err := NewMutationBoundary(caps).Execute(ctx, claim, effect); err != nil { + t.Fatalf("no boundary = %v", err) + } + } + if ran != 3 { + t.Fatalf("ran = %d", ran) + } + // The effect ran but the end call failed: the caller learns the journal + // is incomplete, and the end call survives a cancelled caller context. + h.absent = nil + h.endFail = errors.New("journal write failed") + cctx, cancel := context.WithCancel(ctx) + b := NewMutationBoundary(h) + b.EndTimeout = time.Second + _, err = b.Execute(cctx, claim, func(context.Context) (effects.Result, error) { cancel(); return effects.Result{}, nil }) + if err == nil || !errors.Is(err, h.endFail) { + t.Fatalf("end failure = %v", err) + } + // hive's end answer may carry an error of its own. + h.endFail = nil + hiveErr := &hostDirectEndError{hostDirect: h} + if _, err := NewMutationBoundary(hiveErr).Execute(ctx, claim, effect); err == nil || err.Error() != "fence lost" { + t.Fatalf("hive end error = %v", err) + } +} + +type hostDirectEndError struct{ *hostDirect } + +func (hostDirectEndError) EndEffect(context.Context, string, callback.EndEffectRequest) (callback.EndEffectResponse, error) { + return callback.EndEffectResponse{Error: "fence lost"}, nil +} diff --git a/src/pkg/crux/types/host.go b/src/crux/host.go similarity index 96% rename from src/pkg/crux/types/host.go rename to src/crux/host.go index acdd9cdf3f..dd6751610f 100644 --- a/src/pkg/crux/types/host.go +++ b/src/crux/host.go @@ -1,4 +1,4 @@ -package types +package crux // This file holds the two contracts between the runtime and the host's forge // integration that used to make pkg/agent import pkg/github @@ -48,7 +48,7 @@ func (r PRResult) IsDuplicate() bool { // DefaultAgentTokenCacheDir is the directory holding per-agent GitHub token // caches: the host's token deliverer writes there and the runtime reads from // it (and exports the path to the agent as HIVE_AGENT_TOKEN_CACHE). The -// constant lives in this leaf so both sides agree on it without the runtime +// constant lives in this package so both sides agree on it without the runtime // importing the forge client. const DefaultAgentTokenCacheDir = "/var/run/hive-metrics/agent-tokens" diff --git a/src/pkg/crux/types/host_test.go b/src/crux/host_test.go similarity index 99% rename from src/pkg/crux/types/host_test.go rename to src/crux/host_test.go index 8a9112f7c9..1fbdee10e5 100644 --- a/src/pkg/crux/types/host_test.go +++ b/src/crux/host_test.go @@ -1,4 +1,4 @@ -package types +package crux import "testing" diff --git a/src/crux/host_types.go b/src/crux/host_types.go new file mode 100644 index 0000000000..a886c9e7c1 --- /dev/null +++ b/src/crux/host_types.go @@ -0,0 +1,122 @@ +package crux + +import ( + "context" + "errors" + "fmt" + "os" + "time" +) + +// Hive-provided capabilities (hivecommons/hive#14138, stage 1 of the +// sidecar plan). These are values the host hands the runtime, not things the +// runtime does: the GitHub App token minter, the opt-in agent mint, the +// Linear credential and the PR creator. They are defined here so a host can +// name them without importing crux/agent, and they are deliberately NOT +// methods of Runtime; Host is how an in-process runtime receives them, and a +// sidecar runtime will receive them over a hive-side callback endpoint +// (stage 2 decides the shape). crux/agent aliases every one of them. + +// AppTokenMinter is implemented by the host's GitHub App auth +// (pkg/github.AppAuth) to mint per-agent scoped tokens. +type AppTokenMinter interface { + WriteAgentToken(ctx context.Context, agentName, tier string, agentUID int) error +} + +// AgentMintIssuer is implemented by mint.AgentMinter. It is the OPT-IN, +// ADDITIONAL short-lived-credential path: when the mint is enabled it issues a +// scoped OIDC/JWT for an agent (subject=agent, scopes from its tier) that the +// agent can present ALONGSIDE its GitHub App token to a WIF broker. It never +// replaces WriteAgentToken. Enabled() reports false (and MintAgentToken returns +// "" with no error) when the mint is off, so wiring stays a strict no-op by +// default. +type AgentMintIssuer interface { + Enabled() bool + MintAgentToken(agentName, tier string) (string, error) +} + +// LinearCredential is what an agent authenticates to api.linear.app with. +// Exactly one of the two is expected to be set: AccessToken is the installed +// Linear agent app's OAuth token (sent as `Authorization: ****** so writes +// are authored by the app identity — the parity of GitHub's App bot); APIKey +// is the work-source personal API key (sent bare as `Authorization: …`), the +// fallback when no workspace is connected. Empty means "inject nothing". +type LinearCredential struct { + AccessToken string + APIKey string +} + +// Empty reports whether no credential is available. +func (c LinearCredential) Empty() bool { return c.AccessToken == "" && c.APIKey == "" } + +// PRCreator opens a pull request on behalf of a sandboxed agent. The host's +// GitHub client implements it; the runtime never holds the credential. +type PRCreator interface { + CreatePR(ctx context.Context, repo, head, base, title, body string) (PRResult, error) +} + +// ResumeHandle names a backend-native conversation that can be resumed after +// the authoring tmux pane is gone. +type ResumeHandle struct { + // Backend is the CLI the handle belongs to ("claude", "copilot", ...). + Backend string + // SessionID is the backend's own conversation id. + SessionID string + // Transcript is the file the backend persists that conversation in, so a + // fresh session can read the reasoning even if resume itself fails. For + // goose, whose conversations live in a SQLite store rather than a file + // named by id, it is the per-launch CLI log that names the id instead. + Transcript string + // Command resumes the conversation from a shell in the agent's pane. + Command string + // LastActive is the transcript's modification time when it was captured. + LastActive time.Time + // CapturedAt is when the hive captured the handle (PR-open time); the + // staleness clock runs from here. + CapturedAt time.Time +} + +// String renders a handle for logs. +func (h ResumeHandle) String() string { + if h.SessionID == "" { + return "" + } + return fmt.Sprintf("%s session %s (%s)", h.Backend, h.SessionID, h.Transcript) +} + +// TranscriptExists reports whether the captured transcript is still on disk. +func (h ResumeHandle) TranscriptExists() bool { + if h.Transcript == "" { + return false + } + _, err := os.Stat(h.Transcript) + return err == nil +} + +// ErrResumeBusy means the authoring session still exists but cannot take a +// follow-up right now (mid-turn, provider backoff, restart hold). It is +// transient: the caller keeps the follow-up queued and retries. +var ErrResumeBusy = errors.New("agent session busy; follow-up deferred") + +// ErrResumeSessionGone means the session that authored the PR no longer +// exists, or the agent has moved on to other work since (a newer kick, a +// restart, a relaunch). The caller must fall back to a fresh dispatch. +var ErrResumeSessionGone = errors.New("authoring session no longer live") + +// ErrResumeUnsupported means this agent's execution mode has no resumable +// conversation to type into (sandbox jobs, the headless agy runner) or the +// operator paused it. The caller falls back to a fresh dispatch. +var ErrResumeUnsupported = errors.New("agent cannot resume a session") + +// StartFailureSnapshot is an agent's current start-failure record as seen +// from outside the runtime (the dashboard status builder, the heartbeat). +// Blocked is the fleet-visible bit: the agent has failed the same way enough +// times that the runtime no longer relaunches it on the tick. +type StartFailureSnapshot struct { + Reason string + Count int + LastAt time.Time + LastExitCode *int + LastSignal string + Blocked bool +} diff --git a/src/crux/host_types_test.go b/src/crux/host_types_test.go new file mode 100644 index 0000000000..7ac6611594 --- /dev/null +++ b/src/crux/host_types_test.go @@ -0,0 +1,39 @@ +package crux + +import ( + "os" + "path/filepath" + "testing" +) + +func TestResumeHandleHelpers(t *testing.T) { + if (ResumeHandle{}).String() != "" { + t.Fatal("empty handle must render as empty") + } + transcript := filepath.Join(t.TempDir(), "session.jsonl") + h := ResumeHandle{Backend: "claude", SessionID: "abc", Transcript: transcript} + if got, want := h.String(), "claude session abc ("+transcript+")"; got != want { + t.Fatalf("String() = %q, want %q", got, want) + } + if h.TranscriptExists() { + t.Fatal("transcript not yet written must not exist") + } + if err := os.WriteFile(transcript, []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + if !h.TranscriptExists() { + t.Fatal("written transcript must exist") + } + if (ResumeHandle{SessionID: "x"}).TranscriptExists() { + t.Fatal("handle without a transcript path must report false") + } +} + +func TestLinearCredentialEmpty(t *testing.T) { + if !(LinearCredential{}).Empty() { + t.Fatal("zero credential must be empty") + } + if (LinearCredential{APIKey: "k"}).Empty() || (LinearCredential{AccessToken: "t"}).Empty() { + t.Fatal("a set field must make the credential non-empty") + } +} diff --git a/src/crux/host_wiring.go b/src/crux/host_wiring.go new file mode 100644 index 0000000000..246b490e51 --- /dev/null +++ b/src/crux/host_wiring.go @@ -0,0 +1,69 @@ +package crux + +import ( + "context" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/effects" + "github.com/hivecommons/hive/crux/pushbroker" +) + +// SpanStarter opens a trace span for a runtime operation. The host installs +// its tracer through Host.SetSpanStarter; nil means the OpenTelemetry global. +type SpanStarter func(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) + +// Host is how an in-process runtime receives the capabilities and callbacks +// Hive provides: credential minters, the audit sink, observers for kick +// outcomes and pause transitions, resolvers the launch path consults. It is +// deliberately NOT part of Runtime. A sidecar runtime cannot be handed a Go +// closure; stage 2 of hivecommons/hive#14138 records how each of these +// becomes a hive-side callback endpoint (the minters, the resolvers) or an +// API event (the observers), and until then Host is only satisfied by the +// in-process implementation (crux/runtime.HostOf reports which). +// +// Every method here is called by cmd/hive during boot or on a configuration +// reload, after construction; none is on the dashboard's path. Methods are +// grouped by what the sidecar will replace them with. +type Host interface { + // Credential capabilities -> hive-side callback endpoint (stage 2). + SetAppAuth(auth AppTokenMinter) + SetAgentMint(issuer AgentMintIssuer) + SetLinearCredentialResolver(fn func() LinearCredential) + SetBobAPIKeyResolver(fn func() string) + SetBobKeySourceResolver(fn func() string) + SetSandboxPushMinter(minter pushbroker.TokenMinter) + SetSandboxPRClient(client PRCreator) + + // Launch-time resolvers -> definition fields or a callback endpoint. + SetBuildInfo(fn func() (version, commit string)) + SetGatewayBackendChecker(fn func(backend string) bool) + SetExplainModeDefaultResolver(fn func() string) + SetRepoAutoMergeEnabledResolver(fn func(repo string) bool) + // SetAdvisorEnabledResolver and SetAdvisorRolesResolver feed the advisor + // lane (#9722): whether the advisor reviews an agent, and the live + // model_roles map its OMP extension reads. Both resolve per launch. + SetAdvisorEnabledResolver(fn func(agentName string) bool) + SetAdvisorRolesResolver(fn func() map[string]config.ModelRole) + // SetRepeatFailureEnabledResolver feeds the neutral BASH_ENV route for + // repeat-failure capture (#14574): whether the monitor is on at launch. + SetRepeatFailureEnabledResolver(fn func() bool) + SetKickLogSourceResolver(fn func() string) + SetInferenceCallbacks(setRoute func(agentName, backend, model string), clearRoute func(agentName string)) + SetSandboxConfig(cfg config.AgentSandboxConfig) + SetSandboxMutationBoundary(boundary effects.Boundary) + SetDeadSessionRecoveryOwner(ownedElsewhere bool) + + // Observers -> /v1/events (stage 3). + SetKickOutcomeObserver(fn func(agentName string, outcome KickOutcome)) + SetPauseTransitionObserver(fn func(PauseTransitionEvent)) + SetRelaunchObserver(fn func(agentName string)) + SetPersistPauseCallback(fn func(name string, paused bool)) + SetRecordPromptCallback(fn func(agent, trigger, prompt string)) + SetSandboxAuditCallback(fn func(agent, action, detail string)) + SetAuditSink(sink AuditSink) + SetConditionReporter(r ConditionReporter) + SetSpanStarter(fn SpanStarter) +} diff --git a/src/pkg/inference/gateway_resolution_test.go b/src/crux/inference/gateway_resolution_test.go similarity index 98% rename from src/pkg/inference/gateway_resolution_test.go rename to src/crux/inference/gateway_resolution_test.go index 40c7d46f38..cad9338521 100644 --- a/src/pkg/inference/gateway_resolution_test.go +++ b/src/crux/inference/gateway_resolution_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // ResolveWatsonxGateway must prefer the gateway BOTH named and kinded diff --git a/src/pkg/inference/gateway_watsonx_auth_test.go b/src/crux/inference/gateway_watsonx_auth_test.go similarity index 98% rename from src/pkg/inference/gateway_watsonx_auth_test.go rename to src/crux/inference/gateway_watsonx_auth_test.go index 264eb10c5b..f2c568b9db 100644 --- a/src/pkg/inference/gateway_watsonx_auth_test.go +++ b/src/crux/inference/gateway_watsonx_auth_test.go @@ -5,8 +5,8 @@ import ( "net/http/httptest" "testing" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/watsonx" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/watsonx" ) // swapDefaultMinter points watsonx.DefaultMinter at endpoint for the duration diff --git a/src/pkg/inference/inference.go b/src/crux/inference/inference.go similarity index 98% rename from src/pkg/inference/inference.go rename to src/crux/inference/inference.go index d624739e53..91cb364d0b 100644 --- a/src/pkg/inference/inference.go +++ b/src/crux/inference/inference.go @@ -22,9 +22,9 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/logread" - "github.com/hivecommons/hive/pkg/watsonx" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/logread" + "github.com/hivecommons/hive/crux/watsonx" ) // ResolveLiteLLMRoute resolves the endpoint and model an agent's diff --git a/src/pkg/inference/litellm_output_test.go b/src/crux/inference/litellm_output_test.go similarity index 98% rename from src/pkg/inference/litellm_output_test.go rename to src/crux/inference/litellm_output_test.go index c2a59c4896..7c0700cea7 100644 --- a/src/pkg/inference/litellm_output_test.go +++ b/src/crux/inference/litellm_output_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/logread" + "github.com/hivecommons/hive/crux/logread" ) // TestMain keeps the litellm log source (#11944) out of the real diff --git a/src/pkg/inference/litellm_route_resolution_test.go b/src/crux/inference/litellm_route_resolution_test.go similarity index 99% rename from src/pkg/inference/litellm_route_resolution_test.go rename to src/crux/inference/litellm_route_resolution_test.go index 5d08574bde..1092f60e91 100644 --- a/src/pkg/inference/litellm_route_resolution_test.go +++ b/src/crux/inference/litellm_route_resolution_test.go @@ -3,7 +3,7 @@ package inference import ( "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // ResolveLiteLLMRoute is the route-install decision tree for the diff --git a/src/pkg/inference/litellm_supervision_test.go b/src/crux/inference/litellm_supervision_test.go similarity index 99% rename from src/pkg/inference/litellm_supervision_test.go rename to src/crux/inference/litellm_supervision_test.go index a6b8a8ab6e..53222f9041 100644 --- a/src/pkg/inference/litellm_supervision_test.go +++ b/src/crux/inference/litellm_supervision_test.go @@ -27,7 +27,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/internal/testutil" + "github.com/hivecommons/hive/crux/internal/testutil" ) // syncBuffer is a goroutine-safe log sink: the supervisor goroutine writes diff --git a/src/pkg/inference/port_collision_7238_test.go b/src/crux/inference/port_collision_7238_test.go similarity index 98% rename from src/pkg/inference/port_collision_7238_test.go rename to src/crux/inference/port_collision_7238_test.go index 5c543627cc..6ea659fcf1 100644 --- a/src/pkg/inference/port_collision_7238_test.go +++ b/src/crux/inference/port_collision_7238_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/proxy" + "github.com/hivecommons/hive/crux/proxy" ) // TestLocalLiteLLMPortDoesNotCollideWithTranslator pins an invariant that, diff --git a/src/crux/inferencehealth/inferencehealth.go b/src/crux/inferencehealth/inferencehealth.go new file mode 100644 index 0000000000..c9c24375a4 --- /dev/null +++ b/src/crux/inferencehealth/inferencehealth.go @@ -0,0 +1,227 @@ +// Package inferencehealth tracks the current failing precondition per +// inference gateway (DNS, connect, 5xx, auth, budget). It is a stdlib-only +// Crux ring leaf: the proxy records gateway faults into a Store and snapshots +// them for the runtime's status surface; pkg/inferencehealth re-exports every +// symbol for the hub and dashboard (hivecommons/hive#13536). +package inferencehealth + +import ( + "errors" + "net" + "net/url" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +const ( + ClassDNS = "dns" + ClassConnect = "connect" + Class5xx = "5xx" + ClassAuth = "auth" + ClassBudget = "budget" + ClassOther = "other" +) + +const detailLimit = 200 + +type GatewayStatus struct { + Name string `json:"name"` + Endpoint string `json:"endpoint,omitempty"` + Host string `json:"host,omitempty"` + ErrorClass string `json:"error_class,omitempty"` + HTTPStatus int `json:"http_status,omitempty"` + Detail string `json:"detail,omitempty"` + LastErrorAt string `json:"last_error_at,omitempty"` +} + +type Store struct { + mu sync.RWMutex + faults map[string]GatewayStatus +} + +func NewStore() *Store { return &Store{faults: map[string]GatewayStatus{}} } + +func (s *Store) RecordError(name string, err error, at time.Time) { + s.RecordEndpointError(name, "", err, at) +} + +func (s *Store) RecordEndpointError(name, endpoint string, err error, at time.Time) { + if s == nil || strings.TrimSpace(name) == "" || err == nil { + return + } + class, status := ClassifyError(err) + s.record(GatewayStatus{Name: strings.TrimSpace(name), Endpoint: safeDetail(endpoint), Host: endpointHost(endpoint, err), ErrorClass: class, HTTPStatus: status, Detail: safeDetail(err.Error()), LastErrorAt: at.UTC().Format(time.RFC3339)}) +} + +func (s *Store) RecordHTTPError(name string, status int, detail string, at time.Time) { + s.RecordEndpointHTTPError(name, "", status, detail, at) +} + +func (s *Store) RecordEndpointHTTPError(name, endpoint string, status int, detail string, at time.Time) { + if s == nil || strings.TrimSpace(name) == "" { + return + } + s.record(GatewayStatus{Name: strings.TrimSpace(name), Endpoint: safeDetail(endpoint), Host: endpointHost(endpoint, nil), ErrorClass: ClassifyHTTPStatus(status), HTTPStatus: status, Detail: safeDetail(detail), LastErrorAt: at.UTC().Format(time.RFC3339)}) +} + +func (s *Store) Clear(name string) { + if s == nil || strings.TrimSpace(name) == "" { + return + } + s.mu.Lock() + defer s.mu.Unlock() + key := strings.ToLower(strings.TrimSpace(name)) + delete(s.faults, key) +} + +func (s *Store) Snapshot() []GatewayStatus { + if s == nil { + return nil + } + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]GatewayStatus, 0, len(s.faults)) + for _, st := range s.faults { + out = append(out, st) + } + Sort(out) + return out +} + +func (s *Store) record(st GatewayStatus) { + if strings.TrimSpace(st.ErrorClass) == "" { + st.ErrorClass = ClassOther + } + s.mu.Lock() + defer s.mu.Unlock() + if s.faults == nil { + s.faults = map[string]GatewayStatus{} + } + s.faults[strings.ToLower(st.Name)] = st + // A 401/403 from the inference gateway is an operator/upstream + // condition, not a hive code defect, so it is deliberately not + // self-reported (#12284). +} + +func ClassifyHTTPStatus(status int) string { + switch { + case status == 401 || status == 403: + return ClassAuth + case status == 429: + return ClassBudget + case status >= 500 && status <= 599: + return Class5xx + default: + return ClassOther + } +} + +func ClassifyError(err error) (string, int) { + if err == nil { + return "", 0 + } + var urlErr *url.Error + if errors.As(err, &urlErr) && urlErr.Err != nil { + err = urlErr.Err + } + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + return ClassDNS, 0 + } + var opErr *net.OpError + if errors.As(err, &opErr) { + return ClassConnect, 0 + } + msg := strings.ToLower(err.Error()) + switch { + case strings.Contains(msg, "no such host") || strings.Contains(msg, "server misbehaving"): + return ClassDNS, 0 + case strings.Contains(msg, "connection refused") || strings.Contains(msg, "connect:") || strings.Contains(msg, "i/o timeout"): + return ClassConnect, 0 + case strings.Contains(msg, "http 401"): + return ClassAuth, 401 + case strings.Contains(msg, "http 403"): + return ClassAuth, 403 + case strings.Contains(msg, "http 429"): + return ClassBudget, 429 + case strings.Contains(msg, "http 5"): + return Class5xx, 0 + default: + return ClassOther, 0 + } +} + +func Reason(st GatewayStatus) string { + name := strings.TrimSpace(st.Name) + if name == "" { + name = "unknown" + } + switch st.ErrorClass { + case ClassDNS: + if host := strings.TrimSpace(st.Host); host != "" { + return name + " endpoint " + host + " not resolvable on this cluster — set inference." + name + ".endpoint or disable" + } + return "inference gateway '" + name + "' unreachable (dns)" + case ClassConnect: + return "inference gateway '" + name + "' unreachable (connect)" + case ClassAuth: + if st.HTTPStatus > 0 { + return "inference gateway '" + name + "' rejected key (" + strconv.Itoa(st.HTTPStatus) + ")" + } + return "inference gateway '" + name + "' rejected key" + case ClassBudget: + return "inference gateway '" + name + "' budget/rate limited (429)" + case Class5xx: + return "inference gateway '" + name + "' returned 5xx" + default: + return "inference gateway '" + name + "' failing" + } +} + +func Sort(st []GatewayStatus) { + sort.Slice(st, func(i, j int) bool { + return strings.ToLower(st[i].Name) < strings.ToLower(st[j].Name) + }) +} + +func MostRecent(st []GatewayStatus) (GatewayStatus, bool) { + var best GatewayStatus + var bestAt time.Time + for _, item := range st { + if strings.TrimSpace(item.Name) == "" || strings.TrimSpace(item.ErrorClass) == "" { + continue + } + at, _ := time.Parse(time.RFC3339, item.LastErrorAt) + if best.Name == "" || at.After(bestAt) { + best = item + bestAt = at + } + } + return best, best.Name != "" +} + +func safeDetail(s string) string { + s = strings.TrimSpace(s) + if len(s) > detailLimit { + return s[:detailLimit] + } + return s +} + +func endpointHost(endpoint string, err error) string { + if u, parseErr := url.Parse(strings.TrimSpace(endpoint)); parseErr == nil && u.Hostname() != "" { + return u.Hostname() + } + var urlErr *url.Error + if errors.As(err, &urlErr) && urlErr.Err != nil { + err = urlErr.Err + } + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) && strings.TrimSpace(dnsErr.Name) != "" { + return strings.TrimSpace(dnsErr.Name) + } + return "" +} diff --git a/src/pkg/inferencehealth/inferencehealth_test.go b/src/crux/inferencehealth/inferencehealth_test.go similarity index 92% rename from src/pkg/inferencehealth/inferencehealth_test.go rename to src/crux/inferencehealth/inferencehealth_test.go index 44a3fd9b71..6c38214d42 100644 --- a/src/pkg/inferencehealth/inferencehealth_test.go +++ b/src/crux/inferencehealth/inferencehealth_test.go @@ -7,8 +7,6 @@ import ( "strings" "testing" "time" - - "github.com/hivecommons/hive/pkg/fleetreport" ) func TestClassifyHTTPStatus(t *testing.T) { @@ -276,20 +274,3 @@ func TestStoreRecordEndpointErrorIncludesHost(t *testing.T) { t.Fatalf("Reason = %q", got) } } - -// A 401/403 is an operator/upstream condition: recording it must not file -// runtime-class hive-code evidence (#12284). -func TestStoreAuthFaultIsNotSelfReported(t *testing.T) { - fleetreport.SetDefaultTrackerForTest(t, fleetreport.NewRuntimeTracker(nil)) - at := time.Date(2026, 9, 2, 8, 30, 0, 0, time.UTC) - for _, status := range []int{401, 403} { - s := NewStore() - s.RecordHTTPError("gw", status, "unauthorized", at) - if snap := s.Snapshot(); len(snap) != 1 || snap[0].ErrorClass != ClassAuth { - t.Fatalf("status %d: snapshot = %#v, want one auth fault", status, snap) - } - if ev := fleetreport.DefaultTracker().Evidence(); len(ev) != 0 { - t.Fatalf("status %d: evidence = %#v, want none", status, ev) - } - } -} diff --git a/src/crux/internal/testutil/testutil.go b/src/crux/internal/testutil/testutil.go new file mode 100644 index 0000000000..9bee4d0201 --- /dev/null +++ b/src/crux/internal/testutil/testutil.go @@ -0,0 +1,115 @@ +// Package testutil holds the test helpers the Crux ring's own tests use: +// Eventually and its variants, and the HIVE_TEST_REQUIRE_BEHAVIOURAL skip +// guard. It is a ring-internal copy of the matching helpers in +// internal/testutil so that no ring package imports anything outside the +// ring, even in test code (hivecommons/hive#13536, Stage 2 step 4). At the +// module cut it becomes crux/internal/testutil; the two copies share the +// contract, and internal/testutil keeps the canonical documentation and the +// tests of that contract. +// +// It is test-support code only: nothing in a production package imports it. +package testutil + +import ( + "fmt" + "os" + "testing" + "time" +) + +// pollInterval is how often Eventually re-evaluates its condition. +const pollInterval = 10 * time.Millisecond + +// Eventually polls cond until it returns true or timeout elapses. On the +// deadline it fails the test via t.Fatalf with msg (a printf format) and args. +// See internal/testutil.Eventually for when to use it and when not to. +func Eventually(t testing.TB, timeout time.Duration, cond func() bool, msg string, args ...any) { + t.Helper() + EventuallyEvery(t, timeout, pollInterval, cond, msg, args...) +} + +// EventuallyEvery is Eventually with a caller-chosen sampling interval. +func EventuallyEvery(t testing.TB, timeout, interval time.Duration, cond func() bool, msg string, args ...any) { + t.Helper() + deadline := time.Now().Add(timeout) + for { + if cond() { + return + } + if time.Now().After(deadline) { + t.Fatalf("%s (condition not met within %s)", fmt.Sprintf(msg, args...), timeout) + return + } + time.Sleep(interval) + } +} + +// EventuallyValue polls get until it reports ok and returns the value it +// produced, failing the test via t.Fatalf on timeout exactly like Eventually. +func EventuallyValue[T any](t testing.TB, timeout time.Duration, get func() (T, bool), msg string, args ...any) T { + t.Helper() + var got T + Eventually(t, timeout, func() bool { + v, ok := get() + if ok { + got = v + } + return ok + }, msg, args...) + return got +} + +// EventuallyEveryFunc is EventuallyEvery with the failure message produced +// lazily, at the deadline, by describe. describe is called at most once. +func EventuallyEveryFunc(t testing.TB, timeout, interval time.Duration, cond func() bool, describe func() string) { + t.Helper() + deadline := time.Now().Add(timeout) + for { + if cond() { + return + } + if time.Now().After(deadline) { + t.Fatalf("%s (condition not met within %s)", describe(), timeout) + return + } + time.Sleep(interval) + } +} + +// RequireBehaviouralEnv is the environment variable that turns a permissive +// skip into a failure. It is the same variable internal/testutil and the +// shell suites under src/deploy/ honour, so one flag governs skip discipline +// across the whole test estate (#5388). +const RequireBehaviouralEnv = "HIVE_TEST_REQUIRE_BEHAVIOURAL" + +// failureTemplate mirrors internal/testutil's wording: it names the flag and +// states the diagnosis (BROKEN TEST, not unsuitable environment). +const failureTemplate = "%s\n\t%s=1 — the caller asserts this precondition holds here,\n" + + "\tso this is a BROKEN TEST, not an unsuitable environment (#5388)." + +// RequireBehavioural reports whether the caller has asserted that the +// preconditions for behavioural tests hold in this environment. +func RequireBehavioural() bool { + return os.Getenv(RequireBehaviouralEnv) == "1" +} + +// SkipUnlessRequired skips the test with reason, unless +// HIVE_TEST_REQUIRE_BEHAVIOURAL=1, in which case it fails the test instead. +// Like t.Skip and t.Fatal it does not return and must be called from the +// test goroutine. +func SkipUnlessRequired(t testing.TB, reason string) { + t.Helper() + if RequireBehavioural() { + t.Fatalf(failureTemplate, reason, RequireBehaviouralEnv) + } + t.Skip(reason) +} + +// SkipfUnlessRequired is SkipUnlessRequired with printf-style formatting. +func SkipfUnlessRequired(t testing.TB, format string, args ...any) { + t.Helper() + if RequireBehavioural() { + t.Fatalf(failureTemplate, fmt.Sprintf(format, args...), RequireBehaviouralEnv) + } + t.Skipf(format, args...) +} diff --git a/src/crux/internal/testutil/testutil_test.go b/src/crux/internal/testutil/testutil_test.go new file mode 100644 index 0000000000..0bb5a5baf8 --- /dev/null +++ b/src/crux/internal/testutil/testutil_test.go @@ -0,0 +1,79 @@ +package testutil + +import ( + "errors" + "fmt" + "strings" + "testing" + "time" +) + +// recorder is a testing.TB double that records Fatalf and Skip calls instead +// of ending the goroutine, so the guard's two branches can be asserted. +type recorder struct { + testing.TB + fatal string + skip string +} + +func (r *recorder) Helper() {} +func (r *recorder) Fatalf(format string, args ...any) { r.fatal = fmt.Sprintf(format, args...) } +func (r *recorder) Skip(args ...any) { r.skip = fmt.Sprint(args...) } +func (r *recorder) Skipf(format string, args ...any) { r.skip = fmt.Sprintf(format, args...) } + +func TestEventuallyReturnsOnceConditionHolds(t *testing.T) { + n := 0 + Eventually(t, 2*time.Second, func() bool { n++; return n >= 3 }, "counter never reached 3") + if n < 3 { + t.Fatalf("Eventually returned after %d polls", n) + } +} + +func TestEventuallyValueReturnsObservedValue(t *testing.T) { + n := 0 + got := EventuallyValue(t, 2*time.Second, func() (int, bool) { n++; return n, n >= 2 }, "never ok") + if got != 2 { + t.Fatalf("EventuallyValue = %d, want 2", got) + } +} + +func TestEventuallyFailsAtDeadline(t *testing.T) { + r := &recorder{TB: t} + EventuallyEvery(r, 20*time.Millisecond, time.Millisecond, func() bool { return false }, "never %s", "true") + if !strings.Contains(r.fatal, "never true") || !strings.Contains(r.fatal, "condition not met") { + t.Fatalf("Fatalf message = %q", r.fatal) + } +} + +func TestEventuallyEveryFuncDescribesLazily(t *testing.T) { + r := &recorder{TB: t} + calls := 0 + EventuallyEveryFunc(r, 20*time.Millisecond, time.Millisecond, func() bool { return false }, func() string { calls++; return "state" }) + if calls != 1 || !strings.HasPrefix(r.fatal, "state") { + t.Fatalf("describe called %d times, message %q", calls, r.fatal) + } +} + +func TestSkipUnlessRequiredSkipsByDefault(t *testing.T) { + t.Setenv(RequireBehaviouralEnv, "") + r := &recorder{TB: t} + SkipUnlessRequired(r, "no tmux") + SkipfUnlessRequired(r, "wrapped: %v", errors.New("boom")) + if r.fatal != "" || r.skip != "wrapped: boom" { + t.Fatalf("fatal=%q skip=%q", r.fatal, r.skip) + } +} + +func TestSkipUnlessRequiredFailsWhenArmed(t *testing.T) { + t.Setenv(RequireBehaviouralEnv, "1") + r := &recorder{TB: t} + // The double's Fatalf returns, so the Skipf after it also records; only + // the Fatalf branch is under test here. + SkipfUnlessRequired(r, "no %s", "tmux") + if !strings.Contains(r.fatal, "no tmux") || !strings.Contains(r.fatal, "BROKEN TEST") { + t.Fatalf("fatal=%q skip=%q", r.fatal, r.skip) + } + if !RequireBehavioural() { + t.Fatal("RequireBehavioural() = false with the flag set") + } +} diff --git a/src/crux/issueshape/issueshape.go b/src/crux/issueshape/issueshape.go new file mode 100644 index 0000000000..0063d357f1 --- /dev/null +++ b/src/crux/issueshape/issueshape.go @@ -0,0 +1,100 @@ +// Package issueshape holds the shared shape validators for agent-authored +// GitHub issue and comment content. The rules originated in the issue-request +// watcher (pkg/github), which validates the request files agents drop for the +// hive's own creation path. The proxy (pkg/proxy) enforces the same rules on +// the OTHER creation path — an agent's `gh issue create` becomes a raw +// POST /repos/{owner}/{repo}/issues that never touches the watcher — so the +// logic lives here as a single source of truth for both callers. It is a +// stdlib-only Crux ring leaf; pkg/issueshape re-exports it for the watcher +// and pkg/classify (hivecommons/hive#13536). +package issueshape + +import ( + "regexp" + "strings" +) + +var ( + angleBracketTokenRE = regexp.MustCompile(`<([^>\n]+)>`) + // unchosenOptionListRE matches a policy template's pick-one list left + // verbatim ("Impact: high/medium/low", "**Severity**: critical/high/medium/low"). + unchosenOptionListRE = regexp.MustCompile(`(?i)\b(?:critical/)?high/medium/low\b`) + templatePlaceholderContentRE = regexp.MustCompile(`^(?:[a-z][a-z0-9]*(?:[ -][a-z0-9]+)+|analysis|fix)$`) + markdownHTMLTagsWithAttributes = map[string]bool{ + "a": true, "br": true, "code": true, "dd": true, "del": true, "details": true, + "div": true, "dl": true, "dt": true, "em": true, "h1": true, "h2": true, + "h3": true, "h4": true, "h5": true, "h6": true, "hr": true, "img": true, + "ins": true, "kbd": true, "li": true, "ol": true, "p": true, "pre": true, + "rp": true, "rt": true, "ruby": true, "s": true, "samp": true, "source": true, + "span": true, "strong": true, "sub": true, "summary": true, "sup": true, + "table": true, "tbody": true, "td": true, "tfoot": true, "th": true, + "thead": true, "tr": true, "ul": true, "var": true, + } +) + +// UnsubstitutedTemplatePlaceholder reports whether text still contains an +// unfilled policy-template placeholder like , , or +// : lowercase words, with +// multi-word placeholders separated by spaces or hyphens. It deliberately +// skips URLs, generic type parameters, and recognized GitHub Markdown HTML +// tags (including attribute forms such as
) so those legitimate +// constructs are not mistaken for an unfilled template. The returned string is +// the full matched token (with angle brackets) for use in an error message. +func UnsubstitutedTemplatePlaceholder(text string) (string, bool) { + for _, match := range angleBracketTokenRE.FindAllStringSubmatch(text, -1) { + if len(match) != 2 { + continue + } + // A real placeholder is written tight: . Requiring no + // surrounding whitespace is what separates it from a sentence that + // merely contains two comparison operators -- "holds when a < b and + // c > d in practice" yields the span "< b and c >", which trimming + // would turn into the multi-word phrase "b and c" and refuse. That + // sentence is ordinary prose and must be allowed through. + if strings.TrimSpace(match[1]) != match[1] { + continue + } + content := match[1] + if content == "" || strings.Contains(content, "://") || strings.ContainsAny(content, "/=,") { + continue + } + firstField := content + if fields := strings.Fields(content); len(fields) > 0 { + firstField = fields[0] + } + if markdownHTMLTagsWithAttributes[strings.ToLower(firstField)] { + continue + } + if templatePlaceholderContentRE.MatchString(content) { + return match[0], true + } + } + return "", false +} + +// BodyHasMisEscapedNewlines reports whether body is a single physical line +// whose markdown structure was encoded as literal backslash-n escape +// sequences instead of real newlines (the classic `## X\n\n<...>\n\n## Y` +// specimen). A body with real line breaks may legitimately discuss "\n" in +// prose or a fenced code block, so it requires the combination — no real +// newline, multiple literal \n tokens, and \n\n / \n## structure — before +// flagging, to avoid blocking ordinary one-line text that mentions the escape. +func BodyHasMisEscapedNewlines(body string) bool { + return !strings.Contains(body, "\n") && + strings.Count(body, `\n`) >= 2 && + (strings.Contains(body, `\n\n`) || strings.Contains(body, `\n## `)) +} + +// UnchosenOptionList reports whether text still carries a policy template's +// pick-one option list verbatim — "Impact: high/medium/low", "**Severity**: +// critical/high/medium/low" — which means the author never chose one. The +// hivecommons/hive#7898 issue was the quality template filed untouched: +// prose placeholders ("Detailed explanation of what needs testing and why.") +// carry no angle brackets, so UnsubstitutedTemplatePlaceholder let it +// through; the unpicked priority list is the one template fragment that +// survives any rewording of the prose. The returned string is the matched +// list for use in an error message. +func UnchosenOptionList(text string) (string, bool) { + m := unchosenOptionListRE.FindString(text) + return m, m != "" +} diff --git a/src/pkg/issueshape/issueshape_test.go b/src/crux/issueshape/issueshape_test.go similarity index 100% rename from src/pkg/issueshape/issueshape_test.go rename to src/crux/issueshape/issueshape_test.go diff --git a/src/crux/jev/env.go b/src/crux/jev/env.go new file mode 100644 index 0000000000..d3f163e5cd --- /dev/null +++ b/src/crux/jev/env.go @@ -0,0 +1,16 @@ +// Package jev is the Crux ring's share of Hive's Jev typed-decision tool +// (hivecommons/hive#8939): the environment the agent runtime exports to an +// enabled agent and the jev-decide skill it installs into the agent's CLI +// home. It is stdlib-only. The decision endpoint, the provider client and +// the `hive jev` CLI stay in pkg/jev, which aliases these symbols. +package jev + +// Environment the agent manager exports alongside HIVE_CAVEMAN_MODE. +const ( + // ModeEnvVar carries the agent's jev_mode ("assist" when the tool is on). + ModeEnvVar = "HIVE_JEV_MODE" + // EndpointEnvVar carries the decision endpoint base URL. + EndpointEnvVar = "HIVE_JEV_ENDPOINT" + // DefaultEndpoint is what EndpointEnvVar carries to agents. + DefaultEndpoint = "http://127.0.0.1:18446" +) diff --git a/src/crux/jev/jev_test.go b/src/crux/jev/jev_test.go new file mode 100644 index 0000000000..53dd889497 --- /dev/null +++ b/src/crux/jev/jev_test.go @@ -0,0 +1,33 @@ +package jev + +import ( + "strings" + "testing" +) + +func TestSkillRelDir(t *testing.T) { + for backend, want := range map[string]string{ + "claude": ".claude/skills/jev-decide", "codex": "skills/jev-decide", + "copilot": ".copilot/skills/jev-decide", "gemini": ".gemini/skills/jev-decide", + "goose": ".config/goose/skills/jev-decide", "aider": "", "bob": "", + } { + if got := SkillRelDir(backend); got != want { + t.Errorf("%s: %q, want %q", backend, got, want) + } + } + if !strings.Contains(string(SkillMarkdown()), "hive jev decide") || !strings.HasPrefix(string(SkillMarkdown()), "---\nname: jev-decide") { + t.Error("embedded skill must be the jev-decide SKILL.md with front matter") + } +} + +func TestEnvironmentContract(t *testing.T) { + if ModeEnvVar != "HIVE_JEV_MODE" || EndpointEnvVar != "HIVE_JEV_ENDPOINT" { + t.Fatalf("env var names changed: %q %q", ModeEnvVar, EndpointEnvVar) + } + if !strings.HasPrefix(DefaultEndpoint, "http://127.0.0.1:") { + t.Fatalf("DefaultEndpoint must be loopback, got %q", DefaultEndpoint) + } + if SkillFile != "SKILL.md" || !strings.HasSuffix(SkillRelDir("claude"), SkillName) { + t.Fatalf("skill layout changed: %q %q", SkillFile, SkillRelDir("claude")) + } +} diff --git a/src/pkg/jev/skill.go b/src/crux/jev/skill.go similarity index 100% rename from src/pkg/jev/skill.go rename to src/crux/jev/skill.go diff --git a/src/pkg/jev/skill/SKILL.md b/src/crux/jev/skill/SKILL.md similarity index 100% rename from src/pkg/jev/skill/SKILL.md rename to src/crux/jev/skill/SKILL.md diff --git a/src/crux/jsonextract/jsonextract.go b/src/crux/jsonextract/jsonextract.go new file mode 100644 index 0000000000..eeea869c63 --- /dev/null +++ b/src/crux/jsonextract/jsonextract.go @@ -0,0 +1,23 @@ +// Package jsonextract provides a small shared helper for pulling a JSON +// object out of surrounding free text, typically an LLM chat response that +// wraps the requested JSON in prose or code fences. It was previously +// hand-copied byte-identically into pkg/intent, pkg/ioscan, pkg/retro, and +// pkg/trajectory. It lives in the Crux ring (stdlib-only) because +// pkg/trajectory parses model output with it; pkg/jsonextract aliases it +// for the governor-side callers. +package jsonextract + +import "strings" + +// Object returns the substring of s spanning the first '{' through the last +// '}', or "" when no such span exists. It is a deliberate heuristic — it does +// not validate the JSON — matching the behavior callers rely on for parsing +// model output. +func Object(s string) string { + start := strings.IndexByte(s, '{') + end := strings.LastIndexByte(s, '}') + if start < 0 || end < 0 || end < start { + return "" + } + return s[start : end+1] +} diff --git a/src/pkg/jsonextract/jsonextract_test.go b/src/crux/jsonextract/jsonextract_test.go similarity index 100% rename from src/pkg/jsonextract/jsonextract_test.go rename to src/crux/jsonextract/jsonextract_test.go diff --git a/src/pkg/crux/types/kick.go b/src/crux/kick.go similarity index 92% rename from src/pkg/crux/types/kick.go rename to src/crux/kick.go index db99221c9d..945066850e 100644 --- a/src/pkg/crux/types/kick.go +++ b/src/crux/kick.go @@ -1,4 +1,4 @@ -package types +package crux import "time" @@ -95,3 +95,9 @@ type KickLogInfo struct { // for archives written before the header carried one. Source string `json:"source,omitempty"` } + +// GovernorCadenceKickSource is the kick source the governor eval loop passes +// through Kicker.SendKickWithSource for a cadence (scheduled) kick, as +// opposed to a mention or an operator kick. The runtime treats it as the +// trigger that drains deferred mention kicks. +const GovernorCadenceKickSource = "governor-eval" diff --git a/src/pkg/crux/types/kick_test.go b/src/crux/kick_test.go similarity index 98% rename from src/pkg/crux/types/kick_test.go rename to src/crux/kick_test.go index 4069852bba..69fcc45588 100644 --- a/src/pkg/crux/types/kick_test.go +++ b/src/crux/kick_test.go @@ -1,4 +1,4 @@ -package types +package crux import ( "testing" diff --git a/src/pkg/kubejob/kubejob.go b/src/crux/kubejob/kubejob.go similarity index 99% rename from src/pkg/kubejob/kubejob.go rename to src/crux/kubejob/kubejob.go index 2b0218857e..e317eb8a5c 100644 --- a/src/pkg/kubejob/kubejob.go +++ b/src/crux/kubejob/kubejob.go @@ -41,7 +41,7 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/sandbox" + "github.com/hivecommons/hive/crux/sandbox" ) const ( diff --git a/src/pkg/kubejob/kubejob_test.go b/src/crux/kubejob/kubejob_test.go similarity index 97% rename from src/pkg/kubejob/kubejob_test.go rename to src/crux/kubejob/kubejob_test.go index 35bd07e6c7..8f2d92dce8 100644 --- a/src/pkg/kubejob/kubejob_test.go +++ b/src/crux/kubejob/kubejob_test.go @@ -17,7 +17,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/sandbox" + "github.com/hivecommons/hive/crux/sandbox" ) // fakeAPI is a minimal in-cluster API server: it records the Job manifest it @@ -36,6 +36,7 @@ type fakeAPI struct { podStatus string // "terminated" or "running" failCode int // when >0, every request fails with this status createErr int // when >0, POST fails with this status + onJobGet func() // called (under mu) on every GET job, after it is counted } func newFakeAPI() *fakeAPI { @@ -72,6 +73,9 @@ func (f *fakeAPI) handler(t *testing.T) http.HandlerFunc { i = len(f.statuses) - 1 } f.getCalls++ + if f.onJobGet != nil { + f.onJobGet() + } _, _ = w.Write([]byte(f.statuses[i])) case r.Method == http.MethodDelete: f.deleted = true @@ -186,11 +190,14 @@ func TestRunContextTimeoutDeletesTheJob(t *testing.T) { defer srv.Close() l := newLauncher(srv, Options{}) - ctx, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond) + // End the context only once Run is polling the created Job, so a slow + // runner cannot expire it before the Job exists. + ctx, cancel := context.WithCancel(context.Background()) defer cancel() + api.onJobGet = cancel res, err := l.Run(ctx, specFor("/data/agents/sandbox/deps-123")) - if !errors.Is(err, context.DeadlineExceeded) { - t.Fatalf("Run error = %v, want deadline exceeded", err) + if !errors.Is(err, context.Canceled) { + t.Fatalf("Run error = %v, want the ended context", err) } if !api.deleted { t.Error("a timed-out Job must be deleted so the workload stops") diff --git a/src/pkg/logread/cursor.go b/src/crux/logread/cursor.go similarity index 100% rename from src/pkg/logread/cursor.go rename to src/crux/logread/cursor.go diff --git a/src/pkg/logread/files.go b/src/crux/logread/files.go similarity index 100% rename from src/pkg/logread/files.go rename to src/crux/logread/files.go diff --git a/src/pkg/logread/logread.go b/src/crux/logread/logread.go similarity index 100% rename from src/pkg/logread/logread.go rename to src/crux/logread/logread.go diff --git a/src/pkg/logread/logread_test.go b/src/crux/logread/logread_test.go similarity index 100% rename from src/pkg/logread/logread_test.go rename to src/crux/logread/logread_test.go diff --git a/src/pkg/logread/output.go b/src/crux/logread/output.go similarity index 100% rename from src/pkg/logread/output.go rename to src/crux/logread/output.go diff --git a/src/pkg/logread/output_test.go b/src/crux/logread/output_test.go similarity index 100% rename from src/pkg/logread/output_test.go rename to src/crux/logread/output_test.go diff --git a/src/pkg/logread/parse.go b/src/crux/logread/parse.go similarity index 100% rename from src/pkg/logread/parse.go rename to src/crux/logread/parse.go diff --git a/src/pkg/logread/scan.go b/src/crux/logread/scan.go similarity index 100% rename from src/pkg/logread/scan.go rename to src/crux/logread/scan.go diff --git a/src/pkg/logread/scrub.go b/src/crux/logread/scrub.go similarity index 98% rename from src/pkg/logread/scrub.go rename to src/crux/logread/scrub.go index 48f1e139f0..9e3f44660c 100644 --- a/src/pkg/logread/scrub.go +++ b/src/crux/logread/scrub.go @@ -4,7 +4,7 @@ import ( "regexp" "strings" - "github.com/hivecommons/hive/pkg/logscrub" + "github.com/hivecommons/hive/crux/logscrub" ) // PrivateKeyKind is the redaction kind for private-key material, matching diff --git a/src/pkg/logread/secrets.go b/src/crux/logread/secrets.go similarity index 100% rename from src/pkg/logread/secrets.go rename to src/crux/logread/secrets.go diff --git a/src/pkg/logread/sources_output.go b/src/crux/logread/sources_output.go similarity index 100% rename from src/pkg/logread/sources_output.go rename to src/crux/logread/sources_output.go diff --git a/src/pkg/logread/sources_starts_test.go b/src/crux/logread/sources_starts_test.go similarity index 100% rename from src/pkg/logread/sources_starts_test.go rename to src/crux/logread/sources_starts_test.go diff --git a/src/pkg/logread/textblock.go b/src/crux/logread/textblock.go similarity index 100% rename from src/pkg/logread/textblock.go rename to src/crux/logread/textblock.go diff --git a/src/pkg/logread/textblock_test.go b/src/crux/logread/textblock_test.go similarity index 100% rename from src/pkg/logread/textblock_test.go rename to src/crux/logread/textblock_test.go diff --git a/src/pkg/logscrub/basic_auth_test.go b/src/crux/logscrub/basic_auth_test.go similarity index 100% rename from src/pkg/logscrub/basic_auth_test.go rename to src/crux/logscrub/basic_auth_test.go diff --git a/src/pkg/logscrub/handler.go b/src/crux/logscrub/handler.go similarity index 100% rename from src/pkg/logscrub/handler.go rename to src/crux/logscrub/handler.go diff --git a/src/pkg/logscrub/handler_coverage_test.go b/src/crux/logscrub/handler_coverage_test.go similarity index 100% rename from src/pkg/logscrub/handler_coverage_test.go rename to src/crux/logscrub/handler_coverage_test.go diff --git a/src/pkg/logscrub/handler_test.go b/src/crux/logscrub/handler_test.go similarity index 100% rename from src/pkg/logscrub/handler_test.go rename to src/crux/logscrub/handler_test.go diff --git a/src/pkg/logscrub/pane.go b/src/crux/logscrub/pane.go similarity index 100% rename from src/pkg/logscrub/pane.go rename to src/crux/logscrub/pane.go diff --git a/src/pkg/logscrub/pane_test.go b/src/crux/logscrub/pane_test.go similarity index 100% rename from src/pkg/logscrub/pane_test.go rename to src/crux/logscrub/pane_test.go diff --git a/src/pkg/logscrub/redaction.go b/src/crux/logscrub/redaction.go similarity index 100% rename from src/pkg/logscrub/redaction.go rename to src/crux/logscrub/redaction.go diff --git a/src/pkg/logscrub/redaction_test.go b/src/crux/logscrub/redaction_test.go similarity index 100% rename from src/pkg/logscrub/redaction_test.go rename to src/crux/logscrub/redaction_test.go diff --git a/src/pkg/logscrub/scrub_test.go b/src/crux/logscrub/scrub_test.go similarity index 93% rename from src/pkg/logscrub/scrub_test.go rename to src/crux/logscrub/scrub_test.go index dcd1ff1613..1b36c4ff47 100644 --- a/src/pkg/logscrub/scrub_test.go +++ b/src/crux/logscrub/scrub_test.go @@ -1,11 +1,15 @@ package logscrub import ( + "errors" + "io/fs" "os" "regexp" "sort" "strings" "testing" + + "github.com/hivecommons/hive/crux/internal/testutil" ) // Fixture values are assembled at runtime so the literal strings never @@ -210,8 +214,16 @@ func TestScrubStringLeavesOrdinaryHyphenatedWords(t *testing.T) { } func TestRelayAndGoSecretPatternCategoriesAgree(t *testing.T) { + // The relay lives in the hive root module, outside src/crux. When this + // package is built standalone (the published hivecommons/crux copy, + // #14138) the file is absent and the parity it asserts is hive's to keep, + // so skip rather than fail there; inside hive the behavioural flag turns + // a missing file back into a failure. const relayPath = "../../../bin/contributor-relay.js" body, err := os.ReadFile(relayPath) + if errors.Is(err, fs.ErrNotExist) { + testutil.SkipUnlessRequired(t, "relay script "+relayPath+" is not part of the crux module (standalone build)") + } if err != nil { t.Fatalf("read relay secret patterns from %s: %v", relayPath, err) } diff --git a/src/crux/observation.go b/src/crux/observation.go new file mode 100644 index 0000000000..9d14d583e1 --- /dev/null +++ b/src/crux/observation.go @@ -0,0 +1,76 @@ +package crux + +import ( + "encoding/json" + "net/http" + "strings" + "time" + + "github.com/hivecommons/hive/crux/logscrub" +) + +// Copilot token sources, as reported by Runtime.CopilotTokenSource. They are +// operator-facing phrases naming which credential GitHub saw: they are +// spliced verbatim into the model-picker notice and the "rejected by +// upstream" log line, so each one has to read as an answer to "which +// credential did GitHub just reject?". crux/agent aliases them. +const ( + // CopilotTokenSourceEnv: the COPILOT_GITHUB_TOKEN the hive process was + // started with (a provisioned/machine token, typically). + CopilotTokenSourceEnv = "the COPILOT_GITHUB_TOKEN environment variable" + // CopilotTokenSourceDashboardLogin: a device-flow login completed on the + // dashboard during this process's lifetime. + CopilotTokenSourceDashboardLogin = "the dashboard Copilot login" + // CopilotTokenSourceDurableFile: a dashboard login persisted by an earlier + // hive process and read back at startup. + CopilotTokenSourceDurableFile = "a dashboard Copilot login persisted at " + CopilotUserTokenPath + // CopilotTokenSourceCLIConfig: an in-agent `/login` the reconciler promoted + // out of the shared Copilot CLI config. + CopilotTokenSourceCLIConfig = "an in-agent /login promoted from the shared Copilot CLI config" +) + +// CopilotUserTokenPath is where the dashboard's device-flow login persists +// the Copilot OAuth token; injected into agents as COPILOT_GITHUB_TOKEN. +// Dashboard: Copilot auth status and login. +const CopilotUserTokenPath = "/data/copilot-user-token" + +// GitHubTokenLogin resolves the GitHub login that owns token via GET /user, or +// "" on any failure. Short-timeout, one call. Dashboard: the model-discovery +// notice naming the account behind a rejected Copilot credential; crux/agent: +// the Copilot CLI config seed path. +func GitHubTokenLogin(token string) string { + req, err := http.NewRequest("GET", gitHubUserURL, nil) + if err != nil { + return "" + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Accept", "application/vnd.github+json") + resp, err := gitHubUserClient.Do(req) + if err != nil { + return "" + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return "" + } + var body struct { + Login string `json:"login"` + } + if json.NewDecoder(resp.Body).Decode(&body) != nil { + return "" + } + return strings.TrimSpace(body.Login) +} + +// gitHubUserURL and gitHubUserClient are the endpoint and client +// GitHubTokenLogin uses; tests point them at a local server. +var ( + gitHubUserURL = "https://api.github.com/user" + gitHubUserClient = &http.Client{Timeout: 5 * time.Second} +) + +// RedactPaneTokens masks GitHub tokens, sk- API keys and device-login lines in +// pane-derived text. Dashboard: activity summaries. +func RedactPaneTokens(s string) string { + return logscrub.RedactPaneTokens(s) +} diff --git a/src/crux/observation_login_test.go b/src/crux/observation_login_test.go new file mode 100644 index 0000000000..5d69b17a1b --- /dev/null +++ b/src/crux/observation_login_test.go @@ -0,0 +1,75 @@ +package crux + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func pointGitHubUserAt(t *testing.T, handler http.HandlerFunc) *httptest.Server { + t.Helper() + srv := httptest.NewServer(handler) + origURL, origClient := gitHubUserURL, gitHubUserClient + gitHubUserURL, gitHubUserClient = srv.URL+"/user", srv.Client() + t.Cleanup(func() { + gitHubUserURL, gitHubUserClient = origURL, origClient + srv.Close() + }) + return srv +} + +func TestGitHubTokenLogin_ResolvesLogin(t *testing.T) { + var gotAuth, gotAccept string + pointGitHubUserAt(t, func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + gotAccept = r.Header.Get("Accept") + _, _ = w.Write([]byte(`{"login":" octocat \n"}`)) + }) + if got := GitHubTokenLogin("tok-123"); got != "octocat" { + t.Fatalf("GitHubTokenLogin = %q, want octocat", got) + } + if gotAuth != "Bearer tok-123" { + t.Fatalf("Authorization = %q, want Bearer tok-123", gotAuth) + } + if gotAccept != "application/vnd.github+json" { + t.Fatalf("Accept = %q", gotAccept) + } +} + +func TestGitHubTokenLogin_NonOKStatusIsEmpty(t *testing.T) { + pointGitHubUserAt(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"login":"octocat"}`)) + }) + if got := GitHubTokenLogin("bad"); got != "" { + t.Fatalf("GitHubTokenLogin on 401 = %q, want empty", got) + } +} + +func TestGitHubTokenLogin_MalformedBodyIsEmpty(t *testing.T) { + pointGitHubUserAt(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"login":`)) + }) + if got := GitHubTokenLogin("tok"); got != "" { + t.Fatalf("GitHubTokenLogin on malformed JSON = %q, want empty", got) + } +} + +func TestGitHubTokenLogin_TransportErrorIsEmpty(t *testing.T) { + srv := pointGitHubUserAt(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"login":"octocat"}`)) + }) + srv.Close() + if got := GitHubTokenLogin("tok"); got != "" { + t.Fatalf("GitHubTokenLogin against a closed server = %q, want empty", got) + } +} + +func TestGitHubTokenLogin_BadURLIsEmpty(t *testing.T) { + orig := gitHubUserURL + t.Cleanup(func() { gitHubUserURL = orig }) + gitHubUserURL = "://not a url" + if got := GitHubTokenLogin("tok"); got != "" { + t.Fatalf("GitHubTokenLogin with an unparsable URL = %q, want empty", got) + } +} diff --git a/src/crux/observation_test.go b/src/crux/observation_test.go new file mode 100644 index 0000000000..a4114d471d --- /dev/null +++ b/src/crux/observation_test.go @@ -0,0 +1,37 @@ +package crux_test + +import ( + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" +) + +func TestCopilotTokenConstants_MatchAgentPackage(t *testing.T) { + cases := map[string]string{ + crux.CopilotTokenSourceEnv: agent.CopilotTokenSourceEnv, + crux.CopilotTokenSourceDashboardLogin: agent.CopilotTokenSourceDashboardLogin, + crux.CopilotTokenSourceDurableFile: agent.CopilotTokenSourceDurableFile, + crux.CopilotTokenSourceCLIConfig: agent.CopilotTokenSourceCLIConfig, + crux.CopilotUserTokenPath: agent.CopilotUserTokenPath, + } + for got, want := range cases { + if got != want { + t.Errorf("crux constant %q != agent constant %q", got, want) + } + } +} + +func TestRedactPaneTokens_MatchesAgentPackage(t *testing.T) { + panes := []string{ + "", + "Welcome\n\x1b[1mhello\x1b[0m\n> ", + "Please visit https://github.com/login/device and enter code ABCD-1234", + "token ghp_0123456789abcdefghijABCDEFGHIJ0123456789 leaked", + } + for _, p := range panes { + if got, want := crux.RedactPaneTokens(p), agent.RedactPaneTokens(p); got != want { + t.Errorf("RedactPaneTokens(%q) = %q, want %q", p, got, want) + } + } +} diff --git a/src/pkg/openrouter/openrouter.go b/src/crux/openrouter/openrouter.go similarity index 100% rename from src/pkg/openrouter/openrouter.go rename to src/crux/openrouter/openrouter.go diff --git a/src/pkg/openrouter/openrouter_http_test.go b/src/crux/openrouter/openrouter_http_test.go similarity index 100% rename from src/pkg/openrouter/openrouter_http_test.go rename to src/crux/openrouter/openrouter_http_test.go diff --git a/src/pkg/openrouter/openrouter_test.go b/src/crux/openrouter/openrouter_test.go similarity index 100% rename from src/pkg/openrouter/openrouter_test.go rename to src/crux/openrouter/openrouter_test.go diff --git a/src/crux/outputschema/outputschema.go b/src/crux/outputschema/outputschema.go new file mode 100644 index 0000000000..afdaaf666b --- /dev/null +++ b/src/crux/outputschema/outputschema.go @@ -0,0 +1,440 @@ +// Package outputschema is the structured agent-report contract: the JSON +// object an agent writes at the end of a sandboxed kick and the validator +// the runtime runs over it before the governor consumes it. It lives in the +// Crux ring and imports only the ring's own leaves (pkg/crux/effects for the +// artifact digest, the crux root for the receipt provenance handle); the +// host-side report directory and path helpers stay in pkg/outputschema, +// which aliases everything here (hivecommons/hive#13536). +package outputschema + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "regexp" + "sort" + "strings" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/effects" +) + +const ( + // MaxValidationRetries bounds corrective re-prompt attempts after invalid output. + MaxValidationRetries = 3 + + AgentReportFilePrefix = "agent-report-" + AgentReportFileSuffix = ".json" + + MaxLaneLength = 64 + MaxKindLength = 64 + MaxSummaryLength = 2000 + MaxFindings = 100 + MaxPRsOpened = 50 + MaxBeadsFiled = 50 + MaxArtifacts = 100 + MaxTitleLength = 200 + MaxFindingBodyLength = 2000 + MaxRepoLength = 200 + MaxURLLength = 500 + MaxBeadIDLength = 120 + MaxArtifactPathLength = 500 + MaxReceiptFieldLength = 500 + MaxReceiptProvenanceCheckRunIDs = 100 +) + +const StageReceiptSchemaVersion = "stage-receipt/v1" + +type ReportKind string + +const ( + KindFindings ReportKind = "findings" + KindFix ReportKind = "fix" + KindReview ReportKind = "review" + KindAdvisory ReportKind = "advisory" + KindSummary ReportKind = "summary" + KindInstrument ReportKind = "instrument" + KindStageReceipt ReportKind = "stage_receipt" +) + +type Severity string + +const ( + SeverityInfo Severity = "info" + SeverityLow Severity = "low" + SeverityMedium Severity = "medium" + SeverityHigh Severity = "high" + SeverityCritical Severity = "critical" +) + +type BeadType string + +const ( + BeadTypeTask BeadType = "task" + BeadTypeBug BeadType = "bug" + BeadTypeAdvisory BeadType = "advisory" + BeadTypeFeature BeadType = "feature" +) + +type AgentReport struct { + Lane string `json:"lane"` + Kind ReportKind `json:"kind"` + Findings []Finding `json:"findings"` + PRsOpened []PROpened `json:"prs_opened"` + BeadsFiled []BeadFiled `json:"beads_filed"` + Artifacts []Artifact `json:"artifacts,omitempty"` + Summary string `json:"summary"` + Receipt *StageReceipt `json:"stage_receipt,omitempty"` +} + +// Artifact records a repository file produced or materially updated by an agent. +// It lets artifact-producing lanes report their primary output independently of +// the pull request that happens to carry it. +type Artifact struct { + Repo string `json:"repo"` + Path string `json:"path"` + Description string `json:"description"` +} + +type StageReceiptResultClass string + +const ( + ReceiptResultCompleted StageReceiptResultClass = "completed" + ReceiptResultNoChange StageReceiptResultClass = "no_change" + ReceiptResultBlocked StageReceiptResultClass = "blocked" + ReceiptResultFailed StageReceiptResultClass = "failed" + ReceiptResultUnknown StageReceiptResultClass = "unknown" +) + +type StageReceipt struct { + SchemaVersion string `json:"schema_version"` + WorkKey string `json:"work_key"` + AssignmentID string `json:"assignment_id"` + Generation uint64 `json:"generation"` + Stage string `json:"stage"` + ContractRevision string `json:"contract_revision"` + ExecutionKey string `json:"execution_key"` + Engine *StageReceiptEngine `json:"engine"` + RemoteRunID string `json:"remote_run_id,omitempty"` + RemoteIncarnation string `json:"remote_incarnation,omitempty"` + InputRevision string `json:"input_revision"` + OutputDigest string `json:"output_digest"` + ResultClass StageReceiptResultClass `json:"result_class"` + StartedAt string `json:"started_at"` + EndedAt string `json:"ended_at"` + Provenance *crux.ProofProvenance `json:"provenance"` + Artifacts []Artifact `json:"artifacts"` +} + +type StageReceiptEngine struct { + Name string `json:"name"` + Version string `json:"version"` +} + +type Finding struct { + Title string `json:"title"` + Severity Severity `json:"severity"` + Summary string `json:"summary"` + File string `json:"file,omitempty"` + Line int `json:"line,omitempty"` + ReviewScope string `json:"review_scope,omitempty"` +} + +type PROpened struct { + Repo string `json:"repo"` + Number int `json:"number"` + URL string `json:"url,omitempty"` + Title string `json:"title"` +} + +type BeadFiled struct { + ID string `json:"id"` + Type BeadType `json:"type"` + Title string `json:"title"` + URL string `json:"url,omitempty"` +} + +type Violation struct { + Field string `json:"field"` + Message string `json:"message"` +} + +type ValidationError struct { + Violations []Violation `json:"violations"` +} + +func (e *ValidationError) Error() string { + if e == nil || len(e.Violations) == 0 { + return "agent report validation failed" + } + parts := make([]string, 0, len(e.Violations)) + for _, v := range e.Violations { + parts = append(parts, fmt.Sprintf("%s: %s", v.Field, v.Message)) + } + return "agent report validation failed: " + strings.Join(parts, "; ") +} + +func Validate(raw []byte) (*AgentReport, error) { + if len(bytes.TrimSpace(raw)) == 0 { + return nil, &ValidationError{Violations: []Violation{{Field: "$", Message: "must be a non-empty JSON object"}}} + } + + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + var report AgentReport + if err := dec.Decode(&report); err != nil { + return nil, fmt.Errorf("decode agent report: %w", err) + } + if err := dec.Decode(&struct{}{}); err != io.EOF { + return nil, &ValidationError{Violations: []Violation{{Field: "$", Message: "must contain exactly one JSON object"}}} + } + + if violations := validateReport(report); len(violations) > 0 { + return nil, &ValidationError{Violations: violations} + } + return &report, nil +} + +func validateReport(report AgentReport) []Violation { + var violations []Violation + violations = append(violations, requireBounded("lane", report.Lane, MaxLaneLength)...) + violations = append(violations, requireBounded("kind", string(report.Kind), MaxKindLength)...) + if report.Kind != "" && !validKind(report.Kind) { + violations = append(violations, Violation{Field: "kind", Message: "must be one of findings, fix, review, advisory, summary, instrument, stage_receipt"}) + } + if report.Findings == nil { + violations = append(violations, Violation{Field: "findings", Message: "is required; use [] when there are no findings"}) + } else if len(report.Findings) > MaxFindings { + violations = append(violations, Violation{Field: "findings", Message: fmt.Sprintf("must contain at most %d items", MaxFindings)}) + } + if report.PRsOpened == nil { + violations = append(violations, Violation{Field: "prs_opened", Message: "is required; use [] when no PRs were opened"}) + } else if len(report.PRsOpened) > MaxPRsOpened { + violations = append(violations, Violation{Field: "prs_opened", Message: fmt.Sprintf("must contain at most %d items", MaxPRsOpened)}) + } + if report.BeadsFiled == nil { + violations = append(violations, Violation{Field: "beads_filed", Message: "is required; use [] when no beads were filed"}) + } else if len(report.BeadsFiled) > MaxBeadsFiled { + violations = append(violations, Violation{Field: "beads_filed", Message: fmt.Sprintf("must contain at most %d items", MaxBeadsFiled)}) + } + if len(report.Artifacts) > MaxArtifacts { + violations = append(violations, Violation{Field: "artifacts", Message: fmt.Sprintf("must contain at most %d items", MaxArtifacts)}) + } + violations = append(violations, requireBounded("summary", report.Summary, MaxSummaryLength)...) + + for i, finding := range report.Findings { + prefix := fmt.Sprintf("findings[%d]", i) + violations = append(violations, requireBounded(prefix+".title", finding.Title, MaxTitleLength)...) + violations = append(violations, requireBounded(prefix+".severity", string(finding.Severity), MaxKindLength)...) + if finding.Severity != "" && !validSeverity(finding.Severity) { + violations = append(violations, Violation{Field: prefix + ".severity", Message: "must be one of info, low, medium, high, critical"}) + } + violations = append(violations, requireBounded(prefix+".summary", finding.Summary, MaxFindingBodyLength)...) + violations = append(violations, optionalBounded(prefix+".review_scope", finding.ReviewScope, MaxKindLength)...) + if finding.ReviewScope != "" && finding.ReviewScope != "in-scope" && finding.ReviewScope != "out-of-scope" { + violations = append(violations, Violation{Field: prefix + ".review_scope", Message: "must be in-scope or out-of-scope"}) + } + if finding.Line < 0 { + violations = append(violations, Violation{Field: prefix + ".line", Message: "must be zero or positive"}) + } + } + for i, pr := range report.PRsOpened { + prefix := fmt.Sprintf("prs_opened[%d]", i) + violations = append(violations, requireBounded(prefix+".repo", pr.Repo, MaxRepoLength)...) + violations = append(violations, requireBounded(prefix+".title", pr.Title, MaxTitleLength)...) + if pr.Number <= 0 { + violations = append(violations, Violation{Field: prefix + ".number", Message: "must be greater than zero"}) + } + violations = append(violations, optionalBounded(prefix+".url", pr.URL, MaxURLLength)...) + } + for i, bead := range report.BeadsFiled { + prefix := fmt.Sprintf("beads_filed[%d]", i) + violations = append(violations, requireBounded(prefix+".id", bead.ID, MaxBeadIDLength)...) + violations = append(violations, requireBounded(prefix+".type", string(bead.Type), MaxKindLength)...) + if bead.Type != "" && !validBeadType(bead.Type) { + violations = append(violations, Violation{Field: prefix + ".type", Message: "must be one of task, bug, advisory, feature"}) + } + violations = append(violations, requireBounded(prefix+".title", bead.Title, MaxTitleLength)...) + violations = append(violations, optionalBounded(prefix+".url", bead.URL, MaxURLLength)...) + } + violations = append(violations, validateArtifacts("artifacts", report.Artifacts, report.Artifacts != nil)...) + if report.Kind == KindStageReceipt { + violations = append(violations, validateStageReceipt(report.Receipt)...) + } + return violations +} + +func CorrectivePrompt(err error) string { + if err == nil { + return "" + } + return fmt.Sprintf("Your previous structured agent report was invalid and could not be consumed by Hive. Return exactly one JSON object matching the AgentReport contract: required fields lane, kind, findings, prs_opened, beads_filed, summary; artifacts is optional. For kind stage_receipt, include stage_receipt with schema_version, work_key, assignment_id, generation, stage, contract_revision, execution_key, engine.name, engine.version, input_revision, output_digest, result_class, started_at, ended_at, provenance, and artifacts. Use [] for empty arrays. Allowed kind values: findings, fix, review, advisory, summary, instrument, stage_receipt. Fix these validation errors: %s. Hive will retry validation at most %d times.", err.Error(), MaxValidationRetries) +} + +func requireBounded(field, value string, max int) []Violation { + value = strings.TrimSpace(value) + if value == "" { + return []Violation{{Field: field, Message: "is required"}} + } + return optionalBounded(field, value, max) +} + +func optionalBounded(field, value string, max int) []Violation { + if value == "" { + return nil + } + if len([]rune(value)) > max { + return []Violation{{Field: field, Message: fmt.Sprintf("must be at most %d characters", max)}} + } + return nil +} + +var ( + gitSHAPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + artifactRevisionPat = regexp.MustCompile(`^[^@\s]+@(sha256:)?[0-9a-f]{40,64}$`) +) + +func validateStageReceipt(receipt *StageReceipt) []Violation { + if receipt == nil { + return []Violation{{Field: "stage_receipt", Message: "is required for kind stage_receipt"}} + } + var violations []Violation + if receipt.SchemaVersion == "" { + violations = append(violations, Violation{Field: "stage_receipt.schema_version", Message: "is required"}) + } else if receipt.SchemaVersion != StageReceiptSchemaVersion { + violations = append(violations, Violation{Field: "stage_receipt.schema_version", Message: "must be stage-receipt/v1"}) + } + violations = append(violations, requireBounded("stage_receipt.work_key", receipt.WorkKey, MaxReceiptFieldLength)...) + violations = append(violations, requireBounded("stage_receipt.assignment_id", receipt.AssignmentID, MaxReceiptFieldLength)...) + if receipt.Generation == 0 { + violations = append(violations, Violation{Field: "stage_receipt.generation", Message: "is required and must be greater than zero"}) + } + violations = append(violations, requireBounded("stage_receipt.stage", receipt.Stage, MaxReceiptFieldLength)...) + violations = append(violations, requireBounded("stage_receipt.contract_revision", receipt.ContractRevision, MaxReceiptFieldLength)...) + violations = append(violations, requireBounded("stage_receipt.execution_key", receipt.ExecutionKey, MaxReceiptFieldLength)...) + if receipt.Engine == nil { + violations = append(violations, Violation{Field: "stage_receipt.engine", Message: "is required"}) + } else { + violations = append(violations, requireBounded("stage_receipt.engine.name", receipt.Engine.Name, MaxReceiptFieldLength)...) + violations = append(violations, requireBounded("stage_receipt.engine.version", receipt.Engine.Version, MaxReceiptFieldLength)...) + } + violations = append(violations, optionalBounded("stage_receipt.remote_run_id", receipt.RemoteRunID, MaxReceiptFieldLength)...) + violations = append(violations, optionalBounded("stage_receipt.remote_incarnation", receipt.RemoteIncarnation, MaxReceiptFieldLength)...) + violations = append(violations, requireBounded("stage_receipt.input_revision", receipt.InputRevision, MaxReceiptFieldLength)...) + if receipt.InputRevision != "" && !validInputRevision(receipt.InputRevision) { + violations = append(violations, Violation{Field: "stage_receipt.input_revision", Message: "must be a 40-hex git SHA or artifact@hash"}) + } + violations = append(violations, requireBounded("stage_receipt.output_digest", receipt.OutputDigest, MaxReceiptFieldLength)...) + if receipt.ResultClass == "" { + violations = append(violations, Violation{Field: "stage_receipt.result_class", Message: "is required"}) + } else if !validStageReceiptResultClass(receipt.ResultClass) { + violations = append(violations, Violation{Field: "stage_receipt.result_class", Message: "must be one of completed, no_change, blocked, failed, unknown"}) + } + startedAt, startedOK := validateReceiptTime("stage_receipt.started_at", receipt.StartedAt, &violations) + endedAt, endedOK := validateReceiptTime("stage_receipt.ended_at", receipt.EndedAt, &violations) + if startedOK && endedOK && endedAt.Before(startedAt) { + violations = append(violations, Violation{Field: "stage_receipt.ended_at", Message: "must be at or after started_at"}) + } + if receipt.Provenance == nil { + violations = append(violations, Violation{Field: "stage_receipt.provenance", Message: "is required"}) + } else { + violations = append(violations, requireBounded("stage_receipt.provenance.query", receipt.Provenance.Query, MaxReceiptFieldLength)...) + if len(receipt.Provenance.CheckRunIDs) > MaxReceiptProvenanceCheckRunIDs { + violations = append(violations, Violation{Field: "stage_receipt.provenance.check_run_ids", Message: fmt.Sprintf("must contain at most %d items", MaxReceiptProvenanceCheckRunIDs)}) + } + } + if receipt.Artifacts == nil { + violations = append(violations, Violation{Field: "stage_receipt.artifacts", Message: "is required; use [] when there are no artifacts"}) + } else { + artifactViolations := validateArtifacts("stage_receipt.artifacts", receipt.Artifacts, true) + violations = append(violations, artifactViolations...) + if receipt.ResultClass == ReceiptResultCompleted && len(receipt.Artifacts) == 0 { + violations = append(violations, Violation{Field: "stage_receipt.artifacts", Message: "completed receipts require at least one artifact"}) + } + if receipt.OutputDigest != "" && len(artifactViolations) == 0 { + want := stageReceiptArtifactDigest(receipt.Artifacts) + if receipt.OutputDigest != want { + violations = append(violations, Violation{Field: "stage_receipt.output_digest", Message: "does not match artifacts digest"}) + } + } + } + return violations +} + +func validateReceiptTime(field, value string, violations *[]Violation) (time.Time, bool) { + if strings.TrimSpace(value) == "" { + *violations = append(*violations, Violation{Field: field, Message: "is required"}) + return time.Time{}, false + } + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + *violations = append(*violations, Violation{Field: field, Message: "must be RFC3339"}) + return time.Time{}, false + } + return parsed, true +} + +func validateArtifacts(field string, artifacts []Artifact, present bool) []Violation { + if !present { + return nil + } + var violations []Violation + for i, artifact := range artifacts { + prefix := fmt.Sprintf("%s[%d]", field, i) + violations = append(violations, requireBounded(prefix+".repo", artifact.Repo, MaxRepoLength)...) + violations = append(violations, requireBounded(prefix+".path", artifact.Path, MaxArtifactPathLength)...) + violations = append(violations, requireBounded(prefix+".description", artifact.Description, MaxFindingBodyLength)...) + } + return violations +} + +func stageReceiptArtifactDigest(artifacts []Artifact) string { + parts := make([]string, 0, len(artifacts)) + for _, artifact := range artifacts { + parts = append(parts, strings.Join([]string{artifact.Repo, artifact.Path, artifact.Description}, "\x00")) + } + sort.Strings(parts) + return effects.StableDigest(parts...) +} + +func validInputRevision(revision string) bool { + return gitSHAPattern.MatchString(revision) || artifactRevisionPat.MatchString(revision) +} + +func validStageReceiptResultClass(result StageReceiptResultClass) bool { + switch result { + case ReceiptResultCompleted, ReceiptResultNoChange, ReceiptResultBlocked, ReceiptResultFailed, ReceiptResultUnknown: + return true + default: + return false + } +} + +func validKind(kind ReportKind) bool { + switch kind { + case KindFindings, KindFix, KindReview, KindAdvisory, KindSummary, KindInstrument, KindStageReceipt: + return true + default: + return false + } +} + +func validSeverity(severity Severity) bool { + switch severity { + case SeverityInfo, SeverityLow, SeverityMedium, SeverityHigh, SeverityCritical: + return true + default: + return false + } +} + +func validBeadType(beadType BeadType) bool { + switch beadType { + case BeadTypeTask, BeadTypeBug, BeadTypeAdvisory, BeadTypeFeature: + return true + default: + return false + } +} diff --git a/src/crux/outputschema/outputschema_test.go b/src/crux/outputschema/outputschema_test.go new file mode 100644 index 0000000000..743fda571c --- /dev/null +++ b/src/crux/outputschema/outputschema_test.go @@ -0,0 +1,365 @@ +package outputschema + +import ( + "encoding/json" + "errors" + "os" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/jsonextract" +) + +func TestValidate(t *testing.T) { + valid := `{ + "lane":"scanner", + "kind":"findings", + "findings":[{"title":"racy write","severity":"high","summary":"shared map is written without a lock","file":"pkg/x.go","line":42}], + "prs_opened":[{"repo":"hivecommons/hive","number":2806,"title":"structured reports","url":"https://github.com/hivecommons/hive/pull/1"}], + "beads_filed":[{"id":"bead-1","type":"task","title":"follow-up"}], + "artifacts":[{"repo":"hivecommons/hive","path":"grafana/dashboards/api.json","description":"API service dashboard"}], + "summary":"one finding, one PR, one bead" + }` + + longSummary := strings.Repeat("x", MaxSummaryLength+1) + tests := []struct { + name string + raw string + wantErr []string + }{ + { + name: "valid full report", + raw: valid, + }, + { + name: "valid empty arrays", + raw: `{"lane":"fixer","kind":"fix","findings":[],"prs_opened":[],"beads_filed":[],"summary":"nothing to report"}`, + }, + { + name: "valid instrument report", + raw: `{"lane":"telemetry","kind":"instrument","findings":[],"prs_opened":[],"beads_filed":[],"artifacts":[{"repo":"hivecommons/hive","path":"deploy/servicemonitor.yaml","description":"Prometheus scrape target"}],"summary":"added monitoring"}`, + }, + { + name: "invalid artifact", + raw: `{"lane":"telemetry","kind":"instrument","findings":[],"prs_opened":[],"beads_filed":[],"artifacts":[{"repo":"","path":"","description":""}],"summary":"bad artifact"}`, + wantErr: []string{"artifacts[0].repo", "artifacts[0].path", "artifacts[0].description"}, + }, + { + name: "missing required arrays", + raw: `{"lane":"scanner","kind":"findings","summary":"missing arrays"}`, + wantErr: []string{"findings", "prs_opened", "beads_filed"}, + }, + { + name: "bad enum values", + raw: `{"lane":"scanner","kind":"other","findings":[{"title":"x","severity":"severe","summary":"y"}],"prs_opened":[],"beads_filed":[{"id":"b","type":"unknown","title":"t"}],"summary":"bad enums"}`, + wantErr: []string{"kind: must be one of findings, fix, review, advisory, summary, instrument", "findings[0].severity", "beads_filed[0].type"}, + }, + { + name: "bounded lengths", + raw: `{"lane":"scanner","kind":"summary","findings":[],"prs_opened":[],"beads_filed":[],"summary":"` + longSummary + `"}`, + wantErr: []string{"summary", "at most"}, + }, + { + name: "invalid nested values", + raw: `{"lane":"scanner","kind":"fix","findings":[{"title":"x","severity":"low","summary":"y","line":-1}],"prs_opened":[{"repo":"hivecommons/hive","number":0,"title":"bad"}],"beads_filed":[],"summary":"bad nested"}`, + wantErr: []string{"findings[0].line", "prs_opened[0].number"}, + }, + { + name: "invalid review scope", + raw: `{"lane":"review-swarm","kind":"review","findings":[{"title":"x","severity":"low","summary":"y","review_scope":"later"}],"prs_opened":[],"beads_filed":[],"summary":"bad scope"}`, + wantErr: []string{"findings[0].review_scope", "must be in-scope or out-of-scope"}, + }, + { + name: "unknown field rejected", + raw: `{"lane":"scanner","kind":"summary","findings":[],"prs_opened":[],"beads_filed":[],"summary":"ok","extra":true}`, + wantErr: []string{"unknown field"}, + }, + { + name: "empty input", + raw: ` `, + wantErr: []string{"non-empty JSON object"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + report, err := Validate([]byte(tt.raw)) + if len(tt.wantErr) == 0 { + if err != nil { + t.Fatalf("Validate() error = %v", err) + } + if report == nil || report.Lane == "" { + t.Fatalf("Validate() returned empty report: %#v", report) + } + return + } + if err == nil { + t.Fatal("Validate() error = nil, want error") + } + got := err.Error() + for _, want := range tt.wantErr { + if !strings.Contains(got, want) { + t.Fatalf("Validate() error %q missing %q", got, want) + } + } + }) + } +} + +func TestCorrectivePrompt(t *testing.T) { + prompt := CorrectivePrompt(errors.New("kind: must be one of findings, fix")) + for _, want := range []string{ + "structured agent report was invalid", + "lane, kind, findings, prs_opened, beads_filed, summary", + "instrument", + "kind: must be one of findings, fix", + "3 times", + } { + if !strings.Contains(prompt, want) { + t.Fatalf("CorrectivePrompt() = %q, missing %q", prompt, want) + } + } + if got := CorrectivePrompt(nil); got != "" { + t.Fatalf("CorrectivePrompt(nil) = %q, want empty", got) + } +} + +// A nil or violation-free ValidationError still yields the generic message — +// callers wrap it blindly, so the fallback branch must not panic or go blank. +func TestValidationErrorEmptyAndNil(t *testing.T) { + var nilErr *ValidationError + if got := nilErr.Error(); got != "agent report validation failed" { + t.Errorf("nil ValidationError.Error() = %q", got) + } + if got := (&ValidationError{}).Error(); got != "agent report validation failed" { + t.Errorf("empty ValidationError.Error() = %q", got) + } +} + +// Exactly one JSON object: trailing content after the report is rejected. +func TestValidateRejectsTrailingJSON(t *testing.T) { + raw := `{"lane":"scanner","kind":"summary","findings":[],"prs_opened":[],"beads_filed":[],"summary":"ok"}{"lane":"x"}` + _, err := Validate([]byte(raw)) + if err == nil { + t.Fatal("Validate() accepted two JSON objects") + } + if !strings.Contains(err.Error(), "exactly one JSON object") { + t.Errorf("Validate() error = %q, want the exactly-one-object violation", err) + } +} + +// Every collection cap has an over-limit branch; each must fire independently. +func TestValidateReportOverMaxCollections(t *testing.T) { + report := AgentReport{ + Lane: "scanner", + Kind: KindSummary, + Findings: make([]Finding, MaxFindings+1), + PRsOpened: make([]PROpened, MaxPRsOpened+1), + BeadsFiled: make([]BeadFiled, MaxBeadsFiled+1), + Artifacts: make([]Artifact, MaxArtifacts+1), + Summary: "too much of everything", + } + violations := validateReport(report) + for _, field := range []string{"findings", "prs_opened", "beads_filed", "artifacts"} { + found := false + for _, v := range violations { + if v.Field == field && strings.Contains(v.Message, "must contain at most") { + found = true + break + } + } + if !found { + t.Errorf("no over-max violation for %q in %v", field, violations) + } + } +} + +func TestValidateStageReceipt(t *testing.T) { + raw := mustReadStageReceipt(t) + report, err := Validate(raw) + if err != nil { + t.Fatalf("Validate() stage receipt error = %v", err) + } + if report.Receipt == nil || report.Receipt.SchemaVersion != StageReceiptSchemaVersion { + t.Fatalf("Validate() returned receipt %#v", report.Receipt) + } +} + +func TestValidateStageReceiptMissingRequiredFields(t *testing.T) { + tests := []struct { + name string + field string + edit func(map[string]any) + }{ + {name: "receipt", field: "stage_receipt", edit: func(root map[string]any) { delete(root, "stage_receipt") }}, + {name: "schema version", field: "stage_receipt.schema_version", edit: deleteReceiptField("schema_version")}, + {name: "work key", field: "stage_receipt.work_key", edit: deleteReceiptField("work_key")}, + {name: "assignment id", field: "stage_receipt.assignment_id", edit: deleteReceiptField("assignment_id")}, + {name: "generation", field: "stage_receipt.generation", edit: deleteReceiptField("generation")}, + {name: "stage", field: "stage_receipt.stage", edit: deleteReceiptField("stage")}, + {name: "contract revision", field: "stage_receipt.contract_revision", edit: deleteReceiptField("contract_revision")}, + {name: "execution key", field: "stage_receipt.execution_key", edit: deleteReceiptField("execution_key")}, + {name: "engine", field: "stage_receipt.engine", edit: deleteReceiptField("engine")}, + {name: "engine name", field: "stage_receipt.engine.name", edit: deleteEngineField("name")}, + {name: "engine version", field: "stage_receipt.engine.version", edit: deleteEngineField("version")}, + {name: "input revision", field: "stage_receipt.input_revision", edit: deleteReceiptField("input_revision")}, + {name: "output digest", field: "stage_receipt.output_digest", edit: deleteReceiptField("output_digest")}, + {name: "result class", field: "stage_receipt.result_class", edit: deleteReceiptField("result_class")}, + {name: "started at", field: "stage_receipt.started_at", edit: deleteReceiptField("started_at")}, + {name: "ended at", field: "stage_receipt.ended_at", edit: deleteReceiptField("ended_at")}, + {name: "provenance", field: "stage_receipt.provenance", edit: deleteReceiptField("provenance")}, + {name: "provenance query", field: "stage_receipt.provenance.query", edit: deleteProvenanceField("query")}, + {name: "artifacts", field: "stage_receipt.artifacts", edit: deleteReceiptField("artifacts")}, + {name: "artifact repo", field: "stage_receipt.artifacts[0].repo", edit: deleteArtifactField("repo")}, + {name: "artifact path", field: "stage_receipt.artifacts[0].path", edit: deleteArtifactField("path")}, + {name: "artifact description", field: "stage_receipt.artifacts[0].description", edit: deleteArtifactField("description")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + raw := mutateStageReceipt(t, tt.edit) + _, err := Validate(raw) + if err == nil { + t.Fatal("Validate() error = nil, want missing field violation") + } + validationErr := requireValidationError(t, err) + if countViolations(validationErr, tt.field) != 1 { + t.Fatalf("violations for %q = %v, want exactly one", tt.field, validationErr.Violations) + } + }) + } +} + +func TestValidateStageReceiptRules(t *testing.T) { + tests := []struct { + name string + field string + edit func(map[string]any) + }{ + {name: "schema version", field: "stage_receipt.schema_version", edit: setReceiptField("schema_version", "stage-receipt/v2")}, + {name: "input revision", field: "stage_receipt.input_revision", edit: setReceiptField("input_revision", "not-a-revision")}, + {name: "output digest", field: "stage_receipt.output_digest", edit: setReceiptField("output_digest", "bad-digest")}, + {name: "result class", field: "stage_receipt.result_class", edit: setReceiptField("result_class", "success")}, + {name: "time ordering", field: "stage_receipt.ended_at", edit: setReceiptField("ended_at", "2026-09-22T16:59:59Z")}, + {name: "started RFC3339", field: "stage_receipt.started_at", edit: setReceiptField("started_at", "2026-09-22 17:00:00")}, + {name: "provenance check run cap", field: "stage_receipt.provenance.check_run_ids", edit: func(root map[string]any) { + ids := make([]any, MaxReceiptProvenanceCheckRunIDs+1) + for i := range ids { + ids[i] = float64(i + 1) + } + receiptMap(root)["provenance"].(map[string]any)["check_run_ids"] = ids + }}, + {name: "completed requires artifact", field: "stage_receipt.artifacts", edit: func(root map[string]any) { + receiptMap(root)["artifacts"] = []any{} + receiptMap(root)["output_digest"] = stageReceiptArtifactDigest(nil) + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := Validate(mutateStageReceipt(t, tt.edit)) + if err == nil { + t.Fatal("Validate() error = nil, want rule violation") + } + if countViolations(requireValidationError(t, err), tt.field) == 0 { + t.Fatalf("Validate() error %q missing field %q", err, tt.field) + } + }) + } +} + +func TestValidateStageReceiptNoChangeAllowsNoArtifacts(t *testing.T) { + raw := mutateStageReceipt(t, func(root map[string]any) { + receipt := receiptMap(root) + receipt["result_class"] = "no_change" + receipt["artifacts"] = []any{} + receipt["output_digest"] = stageReceiptArtifactDigest(nil) + }) + if _, err := Validate(raw); err != nil { + t.Fatalf("Validate() no_change empty artifacts error = %v", err) + } +} + +func TestValidateStageReceiptExtractedFromProse(t *testing.T) { + prose := "stage finished; receipt follows:\n```json\n" + string(mustReadStageReceipt(t)) + "\n```\nready for handoff" + extracted := jsonextract.Object(prose) + if extracted == "" { + t.Fatal("jsonextract.Object() returned empty") + } + if _, err := Validate([]byte(extracted)); err != nil { + t.Fatalf("Validate(jsonextract.Object(prose)) error = %v", err) + } +} + +func TestCorrectivePromptStageReceipt(t *testing.T) { + prompt := CorrectivePrompt(&ValidationError{Violations: []Violation{{Field: "stage_receipt.output_digest", Message: "does not match artifacts digest"}}}) + for _, want := range []string{"stage_receipt", "schema_version", "work_key", "assignment_id", "generation", "contract_revision", "execution_key", "engine.name", "engine.version", "input_revision", "output_digest", "result_class", "started_at", "ended_at", "provenance", "artifacts"} { + if !strings.Contains(prompt, want) { + t.Fatalf("CorrectivePrompt() = %q, missing %q", prompt, want) + } + } +} + +func mustReadStageReceipt(t *testing.T) []byte { + t.Helper() + raw, err := os.ReadFile("testdata/stage_receipt_v1.json") + if err != nil { + t.Fatal(err) + } + return raw +} + +func mutateStageReceipt(t *testing.T, edit func(map[string]any)) []byte { + t.Helper() + var root map[string]any + if err := json.Unmarshal(mustReadStageReceipt(t), &root); err != nil { + t.Fatal(err) + } + edit(root) + raw, err := json.Marshal(root) + if err != nil { + t.Fatal(err) + } + return raw +} + +func receiptMap(root map[string]any) map[string]any { + return root["stage_receipt"].(map[string]any) +} + +func deleteReceiptField(field string) func(map[string]any) { + return func(root map[string]any) { delete(receiptMap(root), field) } +} + +func deleteEngineField(field string) func(map[string]any) { + return func(root map[string]any) { delete(receiptMap(root)["engine"].(map[string]any), field) } +} + +func deleteProvenanceField(field string) func(map[string]any) { + return func(root map[string]any) { delete(receiptMap(root)["provenance"].(map[string]any), field) } +} + +func deleteArtifactField(field string) func(map[string]any) { + return func(root map[string]any) { delete(receiptMap(root)["artifacts"].([]any)[0].(map[string]any), field) } +} + +func setReceiptField(field string, value any) func(map[string]any) { + return func(root map[string]any) { receiptMap(root)[field] = value } +} + +func requireValidationError(t *testing.T, err error) *ValidationError { + t.Helper() + validationErr, ok := err.(*ValidationError) + if !ok { + t.Fatalf("Validate() error = %T %v, want *ValidationError", err, err) + } + return validationErr +} + +func countViolations(err *ValidationError, field string) int { + count := 0 + for _, violation := range err.Violations { + if violation.Field == field { + count++ + } + } + return count +} diff --git a/src/pkg/outputschema/testdata/stage_receipt_v1.json b/src/crux/outputschema/testdata/stage_receipt_v1.json similarity index 100% rename from src/pkg/outputschema/testdata/stage_receipt_v1.json rename to src/crux/outputschema/testdata/stage_receipt_v1.json diff --git a/src/crux/paneclass/paneclass.go b/src/crux/paneclass/paneclass.go new file mode 100644 index 0000000000..b29d93e77e --- /dev/null +++ b/src/crux/paneclass/paneclass.go @@ -0,0 +1,33 @@ +// Package paneclass is the runtime's pane classification surface for +// consumers: the readers that turn a captured agent pane into display text or +// a login/auth verdict. The classifiers are implemented in crux/agent, next to +// the pane poller that uses the same tables; this leaf re-exports them so the +// crux root package does not have to import crux/agent (#13962, #13963) and +// consumers still do not name crux/agent. +package paneclass + +import "github.com/hivecommons/hive/crux/agent" + +// AuthPaneState is what ClassifyAuthPane reads off an agent pane: a login +// prompt (and its URL) or a backend authentication error. +type AuthPaneState = agent.AuthPaneState + +// ClassifyAuthPane classifies captured pane lines for login prompts and +// backend auth errors. Loginscan: the pane login detector. +func ClassifyAuthPane(lines []string) AuthPaneState { + return agent.ClassifyAuthPane(lines) +} + +// PaneShowsBlockingPrompt reports whether the pane sits on a known +// startup-blocking modal (folder trust, update prompt, …) for backend, which +// is not a login problem. Loginscan: the login detector stands down on it. +func PaneShowsBlockingPrompt(backend, pane string) bool { + return agent.PaneShowsBlockingPrompt(backend, pane) +} + +// SanitizePaneText removes terminal/TUI chrome from pane-derived text before +// it is displayed, keeping at most maxLines lines. Dashboard: live and detail +// activity summaries; discord: agent status summaries. +func SanitizePaneText(text string, maxLines int) string { + return agent.SanitizePaneText(text, maxLines) +} diff --git a/src/crux/paneclass/paneclass_test.go b/src/crux/paneclass/paneclass_test.go new file mode 100644 index 0000000000..a571f21b45 --- /dev/null +++ b/src/crux/paneclass/paneclass_test.go @@ -0,0 +1,31 @@ +package paneclass + +import ( + "testing" + + "github.com/hivecommons/hive/crux/agent" +) + +func TestPaneHelpers_MatchAgentPackage(t *testing.T) { + panes := []string{ + "", + "Welcome\n\x1b[1mhello\x1b[0m\n> ", + "Please visit https://github.com/login/device and enter code ABCD-1234", + "token ghp_0123456789abcdefghijABCDEFGHIJ0123456789 leaked", + "Do you trust the files in this folder?\n1. Yes, proceed\n2. No, exit", + } + for _, p := range panes { + if got, want := SanitizePaneText(p, 5), agent.SanitizePaneText(p, 5); got != want { + t.Errorf("SanitizePaneText(%q) = %q, want %q", p, got, want) + } + for _, backend := range []string{"claude", "copilot", "codex"} { + if got, want := PaneShowsBlockingPrompt(backend, p), agent.PaneShowsBlockingPrompt(backend, p); got != want { + t.Errorf("PaneShowsBlockingPrompt(%q, %q) = %v, want %v", backend, p, got, want) + } + } + lines := []string{p} + if got, want := ClassifyAuthPane(lines), agent.ClassifyAuthPane(lines); got != want { + t.Errorf("ClassifyAuthPane(%q) = %+v, want %+v", p, got, want) + } + } +} diff --git a/src/crux/pause.go b/src/crux/pause.go new file mode 100644 index 0000000000..5960e84189 --- /dev/null +++ b/src/crux/pause.go @@ -0,0 +1,32 @@ +package crux + +import "time" + +// BreakerTrigger is the distinct PausedTrigger stamped on every pause the fleet +// breaker performs. It serves two jobs: the audit log attributes the pause to +// the breaker, and the runtime's breaker release uses it as the guard that +// distinguishes a pause the breaker still owns (safe to auto-resume) from one +// an operator re-applied during the breaker window (must stay paused). +// Anything whose current Process.PausedTrigger != BreakerTrigger was last +// paused by something else. +const BreakerTrigger = "fleet-breaker" + +// PauseCausation is hook causation metadata carried in agent pause/resume +// events without making the runtime import the host's hook package. +type PauseCausation struct { + Depth int + HookName string + OriginTransition string +} + +// PauseTransitionEvent describes a durable agent pause/resume transition, as +// delivered to the observer the host registers with the runtime. +type PauseTransitionEvent struct { + Agent string + Paused bool + Trigger string + Reason string + By string + Causation PauseCausation + At time.Time +} diff --git a/src/crux/process.go b/src/crux/process.go new file mode 100644 index 0000000000..b1ea412bba --- /dev/null +++ b/src/crux/process.go @@ -0,0 +1,108 @@ +package crux + +import ( + "time" + + "github.com/hivecommons/hive/crux/config" +) + +// Process is the observable snapshot of one agent: state, pins, overrides, +// kick history, pane-derived conditions. It is what Observer.GetStatus and +// Observer.AllStatuses return. +// +// It is its own type, not the runtime's internal agent record: crux/agent +// copies the exported fields of its AgentProcess into a Process when it takes +// a snapshot, so this package does not import crux/agent (#13962, #13963). +// The field meanings are documented on agent.AgentProcess. A snapshot is a +// read-only display value; writing to one does not change the runtime. +type Process struct { + Name string + ID string + Config config.AgentConfig + State ProcessState + PID int + UID int + StartedAt *time.Time + LastKick *time.Time + KicksUndeliverable int + BusySince time.Time + LastTranscriptActivity time.Time + BusyCondition string + BusyConditionMessage string + Paused bool + PausedAt time.Time + PausedReason string + PausedTrigger string + PausedBy string + OperatorStopped bool + PinnedCLI string + PinnedModel string + ModelOverride string + BackendOverride string + RestartCount int + RestartEvents []RestartEvent + LastRestartReason string + OutputBuffer *RingBuffer + SubAgentModels []SubAgentModel + KickHistory []KickRecord + LastKickMessage string + KickRefused bool + KickRefusalReason string + LaunchedMode AgentMode + HasLaunched bool + BootstrapOverride string + LastError string + NeedsLogin bool + LoginURL string + Starting bool + QuotaExhausted bool + WatchdogConditions []Condition + LastPaneChange time.Time + StallNudges int + StallRecoveries int + CIPollNudges int + AwaitingCI bool + TransientNudges int + ProviderErrorClass string + ProviderErrorLine string + ProviderErrorBackoffUntil time.Time + BackendAuth BackendAuthState + TurnLoss TurnLoss + ActionNudges int + KickOutcome KickOutcome + StartFailureClass string + StartFailureReason string + StartFailureCount int + StartFailureLastAt time.Time + StartFailureExitCode *int + StartFailureSignal string + StartBlocked bool + StartBackoffUntil time.Time + + paneCapture []string + paneFilter func(lines []string, n int) []string +} + +// SetPaneCapture records the pane capture a snapshot was taken with and the +// runtime's filter for it, which PaneLines and FilteredPaneLines apply. The +// runtime calls it when it builds a snapshot; lines must not be shared with +// the live capture. +func (p *Process) SetPaneCapture(lines []string, filter func(lines []string, n int) []string) { + p.paneCapture = lines + p.paneFilter = filter +} + +// PaneLines returns the last n lines from the most recent tmux pane capture, +// preferring content from the current CLI session (after the last ❯ prompt). +// Falls back to showing the full tail if the current session has too few lines. +func (p *Process) PaneLines(n int) []string { + if len(p.paneCapture) == 0 || p.paneFilter == nil { + return nil + } + return p.paneFilter(p.paneCapture, n) +} + +// FilteredPaneLines is PaneLines; both names are kept for their callers. +func (p *Process) FilteredPaneLines(n int) []string { + return p.PaneLines(n) +} diff --git a/src/crux/process_test.go b/src/crux/process_test.go new file mode 100644 index 0000000000..c3deb48f4a --- /dev/null +++ b/src/crux/process_test.go @@ -0,0 +1,41 @@ +package crux + +import ( + "reflect" + "testing" +) + +func TestProcess_PaneLinesWithoutCapture(t *testing.T) { + var p Process + if got := p.PaneLines(10); got != nil { + t.Fatalf("PaneLines on an empty snapshot = %v, want nil", got) + } + // A capture without a filter is unanswerable too: the runtime owns the + // prompt-aware filtering, and the snapshot never guesses at it. + p.SetPaneCapture([]string{"a", "b"}, nil) + if got := p.FilteredPaneLines(1); got != nil { + t.Fatalf("FilteredPaneLines with no filter = %v, want nil", got) + } +} + +func TestProcess_PaneLinesApplyRuntimeFilter(t *testing.T) { + var p Process + var sawN int + p.SetPaneCapture([]string{"one", "two", "three"}, func(lines []string, n int) []string { + sawN = n + if n >= len(lines) { + return lines + } + return lines[len(lines)-n:] + }) + + if got := p.PaneLines(2); !reflect.DeepEqual(got, []string{"two", "three"}) { + t.Fatalf("PaneLines(2) = %v, want [two three]", got) + } + if sawN != 2 { + t.Fatalf("filter saw n=%d, want 2", sawN) + } + if got := p.FilteredPaneLines(10); !reflect.DeepEqual(got, []string{"one", "two", "three"}) { + t.Fatalf("FilteredPaneLines(10) = %v, want the whole capture", got) + } +} diff --git a/src/pkg/crux/types/process.go b/src/crux/process_types.go similarity index 99% rename from src/pkg/crux/types/process.go rename to src/crux/process_types.go index 8fc209960e..73612dd6b8 100644 --- a/src/pkg/crux/types/process.go +++ b/src/crux/process_types.go @@ -1,4 +1,4 @@ -package types +package crux import "time" diff --git a/src/crux/project.go b/src/crux/project.go new file mode 100644 index 0000000000..fd05022ee6 --- /dev/null +++ b/src/crux/project.go @@ -0,0 +1,161 @@ +package crux + +import ( + "strings" + + "github.com/hivecommons/hive/crux/taskmcp" +) + +// ProjectContext holds the host-derived project facts (org, repos, policy +// dir, pause state) a runtime is built with and injects into agent boot +// prompts. It is on the Crux surface so a host such as cmd/crux can build one +// without naming crux/agent. +type ProjectContext struct { + Org string + Repos []string + PrimaryRepoName string + ACMMLevel int + PRsAllowed bool + PolicyDir string + // GHHost is the bare hostname of the source forge when it is NOT public + // github.com (e.g. "github.ibm.com" for a GHE spoke), derived from the + // configured github.api_url. Exported to agents as GH_HOST so the gh CLI + // targets the right host — without it every agent gh call went to + // api.github.com where the project's repos do not exist (root-caused live + // 2026-08-20: the security agent's issue/PR creation failed silently on + // every GHE-hosted hive). Empty ⇒ public github.com, nothing exported. + GHHost string + // AppAuthoredPRs mirrors config github.app_authored_prs: when true, push- + // capable agents get the App installation token as GITHUB_TOKEN so the GitHub + // MCP server authors PRs/commits as the App bot. Default false → no token is + // injected and behavior is unchanged (opt-in per hive). + AppAuthoredPRs bool + // AppBotLogin and AppBotUserID identify the installed GitHub App's bot + // account. When AppAuthoredPRs is on, push-capable agents use this bot's + // noreply address ("+@users.noreply.github.com") for git + // author/committer email so GitHub links their fixer commits to the App + // while the commit name and sign-off still name the lane. AppBotUserID + // resolves the bot's GitHub USER id (not the App id, which GitHub never + // links) and is called at each launch so a failed lookup (0) only costs + // that launch the linkage; nil means unknown. + AppBotLogin string + AppBotUserID func() int64 + // BobSessionPrefix mirrors governor.bob.session_prefix. Empty is + // default-off; bob launch commands remain byte-identical unless it or a + // per-agent bob.session_label is configured. + BobSessionPrefix string + // RepoPaused reports whether a repo carries an operator pause (#6203). It is + // a live predicate rather than a snapshot list because ProjectContext is + // built once at boot and a pause is taken mid-run — a repo frozen for a + // release must drop out of the next agent's scope without a hive restart. + // Nil means nothing is paused, which is every hive that does not use the + // feature and every test that constructs a ProjectContext literal. + RepoPaused func(repo string) bool + // AgentRepos returns the repositories one agent serves (#6204), and + // AgentPrimaryRepo the repo its prompts should treat as "the" repo. Both + // are live functions rather than a snapshot map because ProjectContext is + // built once at boot while a scope is edited mid-run — a specialist + // re-scoped in the dashboard must reach the next agent launch without a + // hive restart. + // + // Nil means no scoping: every agent gets Repos and PrimaryRepo, which is + // what every hive did before this field and what every test that builds a + // ProjectContext literal still gets. + AgentRepos func(agent string) []string + AgentPrimaryRepo func(agent string) string + // TaskMCPURL is the hub-served, read-only task context endpoint for + // hub-launched agents. Empty keeps launches byte-for-byte unchanged. It + // carries NO credential: the dashboard bearer must never reach an agent + // (#8348), so the per-launch token comes from TaskMCPLaunchToken instead. + TaskMCPURL string + // TaskMCPLaunchToken mints the credential one launch presents to the task + // MCP endpoint, bound to that launch's scope (task id, repo, agent, + // generation) and revoked when the launch ends. Nil means the URL is + // handed over without a token; the hub then refuses the agent's calls + // rather than falling back to a wider credential. + TaskMCPLaunchToken func(scope taskmcp.LaunchScope) string + // ExperimentEnv is the hive-wide experimentation environment (RFC + // #11585 part 5, pkg/nousengine.HiveEnv): the Nous dispatcher and data + // root as the hive resolved them (env > yaml > default). It is exported + // ONLY to agents whose role is config.RoleExperimenter - alongside that + // agent's own engine name - so the strategist and every other role + // launch byte-for-byte as before. Nil exports only the engine name. + ExperimentEnv map[string]string +} + +func (p ProjectContext) PrimaryRepo() string { + if strings.TrimSpace(p.PrimaryRepoName) != "" { + return strings.TrimPrefix(p.PrimaryRepoName, p.Org+"/") + } + if len(p.Repos) > 0 { + return strings.TrimPrefix(p.Repos[0], p.Org+"/") + } + return "" +} + +// ActiveRepos is Repos minus the operator-paused entries: the repos an agent +// should be told to work. +// +// PrimaryRepo deliberately does NOT filter. The primary repo is the hive's +// identity — what $HIVE_REPO names and what an agent's workdir is a checkout +// of — and re-pointing it at a different repository because the operator +// froze the first one would be a far larger surprise than the pause itself. +// A paused primary simply stops appearing in the work scope. +func (p ProjectContext) ActiveRepos() []string { + return p.unpaused(p.Repos) +} + +// ReposFor returns the repositories the named agent serves, falling back to the +// hive-wide list when the agent is unscoped or no scope function is wired. +func (p ProjectContext) ReposFor(agent string) []string { + if p.AgentRepos == nil { + return p.Repos + } + if repos := p.AgentRepos(agent); repos != nil { + return repos + } + return p.Repos +} + +// ActiveReposFor is ReposFor minus the operator-paused entries: the repos the +// named agent should actually be handed work on. +// +// The two narrowings compose rather than override. A scope says which repos an +// agent is FOR (#6204); a pause says which repos are open for work at all +// (#6203). An agent scoped to a repo the operator has since paused is handed +// nothing for it, and unpausing restores it without touching the scope. +func (p ProjectContext) ActiveReposFor(agent string) []string { + return p.unpaused(p.ReposFor(agent)) +} + +// unpaused drops the operator-paused entries from repos, preserving order. +func (p ProjectContext) unpaused(repos []string) []string { + if p.RepoPaused == nil { + return repos + } + out := make([]string, 0, len(repos)) + for _, r := range repos { + if p.RepoPaused(r) { + continue + } + out = append(out, r) + } + return out +} + +// PrimaryRepoFor returns the repo the named agent's prompts should treat as +// "the" repo — what $HIVE_REPO names. +// +// Unlike PrimaryRepo() this is per-agent, and it has to be: the shipped +// templates say `gh issue create --repo "$HIVE_REPO"`, so handing a specialist +// scoped away from the hive primary the hive primary would aim every example it +// has at a repository the proxy then refuses. An unscoped agent is unaffected. +// Like PrimaryRepo it does not filter paused repos — see ActiveRepos. +func (p ProjectContext) PrimaryRepoFor(agent string) string { + if p.AgentPrimaryRepo != nil { + if repo := strings.TrimPrefix(strings.TrimSpace(p.AgentPrimaryRepo(agent)), p.Org+"/"); repo != "" { + return repo + } + } + return p.PrimaryRepo() +} diff --git a/src/crux/project_test.go b/src/crux/project_test.go new file mode 100644 index 0000000000..2477e7bfdb --- /dev/null +++ b/src/crux/project_test.go @@ -0,0 +1,140 @@ +package crux + +import ( + "reflect" + "testing" +) + +func scopedProject(scopes map[string][]string) ProjectContext { + p := ProjectContext{ + Org: "testorg", + Repos: []string{"console", "dashboard", "docs"}, + PrimaryRepoName: "console", + ACMMLevel: 4, + PRsAllowed: true, + } + if len(scopes) == 0 { + return p + } + p.AgentRepos = func(agent string) []string { + scope, ok := scopes[agent] + if !ok { + return nil + } + out := make([]string, 0, len(p.Repos)) + for _, r := range p.Repos { + for _, s := range scope { + if r == s { + out = append(out, r) + } + } + } + return out + } + p.AgentPrimaryRepo = func(agent string) string { + if repos := p.AgentRepos(agent); len(repos) > 0 { + return repos[0] + } + return "" + } + return p +} + +func TestProjectContext_PrimaryRepo(t *testing.T) { + tests := []struct { + name string + p ProjectContext + want string + }{ + {"named primary wins", ProjectContext{Org: "o", Repos: []string{"a", "b"}, PrimaryRepoName: "b"}, "b"}, + {"org prefix is stripped", ProjectContext{Org: "o", PrimaryRepoName: "o/b"}, "b"}, + {"blank primary falls back to first repo", ProjectContext{Org: "o", Repos: []string{"o/a", "b"}, PrimaryRepoName: " "}, "a"}, + {"nothing configured", ProjectContext{Org: "o"}, ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := tc.p.PrimaryRepo(); got != tc.want { + t.Fatalf("PrimaryRepo() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestProjectContext_ReposForFallsBackWhenUnwired(t *testing.T) { + p := scopedProject(nil) + if got := p.ReposFor("anything"); !reflect.DeepEqual(got, p.Repos) { + t.Fatalf("ReposFor with no scope function = %v, want the hive list", got) + } + if got := p.PrimaryRepoFor("anything"); got != "console" { + t.Errorf("PrimaryRepoFor with no scope function = %q, want console", got) + } + if got := p.ActiveRepos(); !reflect.DeepEqual(got, p.Repos) { + t.Errorf("ActiveRepos with no pause predicate = %v, want the hive list", got) + } + if got := p.ActiveReposFor("anything"); !reflect.DeepEqual(got, p.Repos) { + t.Errorf("ActiveReposFor with no pause predicate = %v, want the hive list", got) + } +} + +func TestProjectContext_ReposForScopedAgent(t *testing.T) { + p := scopedProject(map[string][]string{"schema": {"dashboard"}}) + + if got := p.ReposFor("schema"); !reflect.DeepEqual(got, []string{"dashboard"}) { + t.Errorf("ReposFor(schema) = %v, want [dashboard]", got) + } + // A scope function answering nil means "unscoped": the hive list applies. + if got := p.ReposFor("scanner"); !reflect.DeepEqual(got, p.Repos) { + t.Errorf("ReposFor(unscoped) = %v, want all three", got) + } + if got := p.PrimaryRepoFor("schema"); got != "dashboard" { + t.Errorf("PrimaryRepoFor(schema) = %q, want dashboard", got) + } + if got := p.PrimaryRepoFor("scanner"); got != "console" { + t.Errorf("PrimaryRepoFor(unscoped) = %q, want console", got) + } + if len(p.Repos) != 3 { + t.Errorf("Repos was mutated to %v", p.Repos) + } +} + +func TestProjectContext_PrimaryRepoForStripsOrgAndFallsBack(t *testing.T) { + p := scopedProject(nil) + p.AgentPrimaryRepo = func(agent string) string { + switch agent { + case "qualified": + return " testorg/docs " + default: + return "" + } + } + if got := p.PrimaryRepoFor("qualified"); got != "docs" { + t.Errorf("PrimaryRepoFor(qualified) = %q, want docs", got) + } + // A blank per-agent answer falls through to the hive primary. + if got := p.PrimaryRepoFor("blank"); got != "console" { + t.Errorf("PrimaryRepoFor(blank) = %q, want console", got) + } +} + +func TestProjectContext_PausedReposDropOutOfActiveScope(t *testing.T) { + p := scopedProject(map[string][]string{"schema": {"dashboard", "docs"}}) + p.RepoPaused = func(repo string) bool { return repo == "dashboard" } + + if got := p.ActiveRepos(); !reflect.DeepEqual(got, []string{"console", "docs"}) { + t.Errorf("ActiveRepos = %v, want [console docs]", got) + } + if got := p.ActiveReposFor("schema"); !reflect.DeepEqual(got, []string{"docs"}) { + t.Errorf("ActiveReposFor(schema) = %v, want [docs]", got) + } + // The pause narrows the work scope only: the scope and the primary are + // untouched, so unpausing restores them without a re-scope. + if got := p.ReposFor("schema"); !reflect.DeepEqual(got, []string{"dashboard", "docs"}) { + t.Errorf("ReposFor(schema) = %v, want the unfiltered scope", got) + } + if got := p.PrimaryRepoFor("schema"); got != "dashboard" { + t.Errorf("PrimaryRepoFor(schema) = %q, want dashboard even while paused", got) + } + if got := p.PrimaryRepo(); got != "console" { + t.Errorf("PrimaryRepo = %q, want console", got) + } +} diff --git a/src/crux/provenance.go b/src/crux/provenance.go new file mode 100644 index 0000000000..2379a5a50a --- /dev/null +++ b/src/crux/provenance.go @@ -0,0 +1,15 @@ +package crux + +// ProofProvenance is the bounded replay handle a convergence proof record +// and a stage receipt carry: enough to re-issue the exact observation and +// audit which runs were counted — deliberately NOT a log archive. It lives +// here because the agent-report schema the runtime validates +// (pkg/crux/outputschema) embeds it; pkg/convergence/proof aliases it as +// proof.Provenance. +type ProofProvenance struct { + // CheckRunIDs are the check-run IDs counted toward the result, sorted. + CheckRunIDs []int64 `json:"check_run_ids,omitempty"` + // Query names the observation call shape, e.g. + // "ListCheckRunsForRef@". + Query string `json:"query"` +} diff --git a/src/pkg/crux/types/provider_status.go b/src/crux/provider_status.go similarity index 92% rename from src/pkg/crux/types/provider_status.go rename to src/crux/provider_status.go index e872e5bb9e..2d2fbf8229 100644 --- a/src/pkg/crux/types/provider_status.go +++ b/src/crux/provider_status.go @@ -1,15 +1,4 @@ -// Package types holds the value types of the Crux runtime surface -// (hivecommons/hive#13536, src/docs/design/crux.md): plain data the runtime -// publishes and its consumers read. pkg/agent imports this package and -// re-exports each type as an alias, so a *agent.Manager keeps returning the -// same types a consumer compares against; pkg/crux re-exports them too, so -// consumers program against crux. and never see this package. -// -// It is a separate leaf package, not pkg/crux itself, because pkg/crux still -// aliases agent.AgentProcess (crux.Process) and so imports pkg/agent; a -// package pkg/agent imports cannot be one that imports pkg/agent back. The -// leaf folds into pkg/crux once Process is its own type. -package types +package crux import ( "sort" diff --git a/src/pkg/crux/types/provider_status_test.go b/src/crux/provider_status_test.go similarity index 99% rename from src/pkg/crux/types/provider_status_test.go rename to src/crux/provider_status_test.go index d31050b7c9..3be0cda40b 100644 --- a/src/pkg/crux/types/provider_status_test.go +++ b/src/crux/provider_status_test.go @@ -1,4 +1,4 @@ -package types +package crux import ( "strings" diff --git a/src/pkg/proxy/agent_github_budget.go b/src/crux/proxy/agent_github_budget.go similarity index 94% rename from src/pkg/proxy/agent_github_budget.go rename to src/crux/proxy/agent_github_budget.go index fa4069716a..3f00143cc0 100644 --- a/src/pkg/proxy/agent_github_budget.go +++ b/src/crux/proxy/agent_github_budget.go @@ -1,8 +1,10 @@ package proxy import ( + "context" "encoding/json" "io" + "log/slog" "math" "net/http" "strconv" @@ -10,7 +12,7 @@ import ( "sync" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) const ( @@ -192,11 +194,17 @@ func (p *GitHubProxy) writeAgentGitHubBudget429(client netWriterConn, st agentGi if st.RetryAfter <= 0 { st.RetryAfter = 1 } - if p.githubBudget != nil && p.githubBudget.markNudged(st.Agent) && p.githubBudgetNudge != nil { + first := p.githubBudget != nil && p.githubBudget.markNudged(st.Agent) + if first && p.githubBudgetNudge != nil { p.githubBudgetNudge(st.Agent, "Stop polling GitHub. gh returned 429 hourly cap reached; stop all GitHub reads for this session and finish with local work.") } if p.logger != nil { - p.logger.Warn("agent GitHub API hourly cap reached", + // Warn once per agent per cap window; repeats are expected and self-limiting. + level := slog.LevelDebug + if first { + level = slog.LevelWarn + } + p.logger.Log(context.Background(), level, "agent GitHub API hourly cap reached", "agent", st.Agent, "count", st.Count, "cap", st.Cap, "retry_after", st.RetryAfter, "reason", st.Reason) } payload, _ := json.Marshal(map[string]string{ diff --git a/src/pkg/proxy/agent_github_budget_test.go b/src/crux/proxy/agent_github_budget_test.go similarity index 78% rename from src/pkg/proxy/agent_github_budget_test.go rename to src/crux/proxy/agent_github_budget_test.go index 913f55d756..102409c4d8 100644 --- a/src/pkg/proxy/agent_github_budget_test.go +++ b/src/crux/proxy/agent_github_budget_test.go @@ -2,7 +2,9 @@ package proxy import ( "bufio" + "bytes" "encoding/json" + "io" "log/slog" "net" "net/http" @@ -115,3 +117,34 @@ func TestAgentGitHubBudget429ShapeAndNudge(t *testing.T) { t.Fatalf("nudge repeated in same window: %d", nudges) } } + +func TestAgentGitHubBudgetCapWarnOncePerWindow(t *testing.T) { + now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + var buf bytes.Buffer + p := &GitHubProxy{ + logger: slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})), + githubBudget: newAgentGitHubBudget(func() time.Time { return now }), + } + refuse := func() { + server, client := net.Pipe() + defer server.Close() + defer client.Close() + go func() { _, _ = io.Copy(io.Discard, client) }() + p.writeAgentGitHubBudget429(server, agentGitHubBudgetStatus{Agent: "scanner", Count: 3, Cap: 3, RetryAfter: 5, Reason: "cap"}) + } + warns := func() int { return strings.Count(buf.String(), "level=WARN") } + for i := 0; i < 5; i++ { + refuse() + } + if got := warns(); got != 1 { + t.Fatalf("WARNs in first window = %d, want 1\n%s", got, buf.String()) + } + if got := strings.Count(buf.String(), "level=DEBUG"); got != 4 { + t.Fatalf("DEBUG repeats = %d, want 4", got) + } + now = now.Add(time.Hour + time.Second) + refuse() + if got := warns(); got != 2 { + t.Fatalf("WARNs after window reset = %d, want 2", got) + } +} diff --git a/src/pkg/proxy/agent_identification_test.go b/src/crux/proxy/agent_identification_test.go similarity index 99% rename from src/pkg/proxy/agent_identification_test.go rename to src/crux/proxy/agent_identification_test.go index 869a165cdc..944c8f331f 100644 --- a/src/pkg/proxy/agent_identification_test.go +++ b/src/crux/proxy/agent_identification_test.go @@ -9,7 +9,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // agentUIDFixture is the UID the fake /proc/net/tcp table attributes the diff --git a/src/pkg/proxy/agent_repo_scope_test.go b/src/crux/proxy/agent_repo_scope_test.go similarity index 99% rename from src/pkg/proxy/agent_repo_scope_test.go rename to src/crux/proxy/agent_repo_scope_test.go index 9f5df92569..64ab593f22 100644 --- a/src/pkg/proxy/agent_repo_scope_test.go +++ b/src/crux/proxy/agent_repo_scope_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // scopeOf builds the predicate the proxy takes from a simple agent → repos map, diff --git a/src/pkg/proxy/anthropic_translate.go b/src/crux/proxy/anthropic_translate.go similarity index 100% rename from src/pkg/proxy/anthropic_translate.go rename to src/crux/proxy/anthropic_translate.go diff --git a/src/pkg/proxy/anthropic_translate_test.go b/src/crux/proxy/anthropic_translate_test.go similarity index 100% rename from src/pkg/proxy/anthropic_translate_test.go rename to src/crux/proxy/anthropic_translate_test.go diff --git a/src/pkg/proxy/auth_inject_copilot_test.go b/src/crux/proxy/auth_inject_copilot_test.go similarity index 99% rename from src/pkg/proxy/auth_inject_copilot_test.go rename to src/crux/proxy/auth_inject_copilot_test.go index ddc25414d9..c294e1dd40 100644 --- a/src/pkg/proxy/auth_inject_copilot_test.go +++ b/src/crux/proxy/auth_inject_copilot_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // testCopilotUserToken stands in for the user's Copilot OAuth token diff --git a/src/pkg/proxy/auth_inject_test.go b/src/crux/proxy/auth_inject_test.go similarity index 99% rename from src/pkg/proxy/auth_inject_test.go rename to src/crux/proxy/auth_inject_test.go index 7c20c2b442..cc7ab6d6cb 100644 --- a/src/pkg/proxy/auth_inject_test.go +++ b/src/crux/proxy/auth_inject_test.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) const ( diff --git a/src/pkg/proxy/autonomy_enforcement_test.go b/src/crux/proxy/autonomy_enforcement_test.go similarity index 96% rename from src/pkg/proxy/autonomy_enforcement_test.go rename to src/crux/proxy/autonomy_enforcement_test.go index effec929cc..f874460f6a 100644 --- a/src/pkg/proxy/autonomy_enforcement_test.go +++ b/src/crux/proxy/autonomy_enforcement_test.go @@ -3,7 +3,7 @@ package proxy import ( "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) func TestAutonomyDemotedRepoPRWriteRefused(t *testing.T) { diff --git a/src/pkg/proxy/autonomy_levels.go b/src/crux/proxy/autonomy_levels.go similarity index 97% rename from src/pkg/proxy/autonomy_levels.go rename to src/crux/proxy/autonomy_levels.go index f0cf91913b..e74c046ca1 100644 --- a/src/pkg/proxy/autonomy_levels.go +++ b/src/crux/proxy/autonomy_levels.go @@ -6,7 +6,7 @@ import ( "strings" "sync" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) var autonomyRepoLevels sync.Map diff --git a/src/pkg/proxy/canary_test.go b/src/crux/proxy/canary_test.go similarity index 51% rename from src/pkg/proxy/canary_test.go rename to src/crux/proxy/canary_test.go index 1791ab7bd0..98909d297c 100644 --- a/src/pkg/proxy/canary_test.go +++ b/src/crux/proxy/canary_test.go @@ -1,85 +1,95 @@ package proxy import ( - "encoding/base64" - "encoding/hex" "io" "log/slog" "net/http" "strings" "testing" - "github.com/hivecommons/hive/pkg/ioscan" + "github.com/hivecommons/hive/crux" ) +// fakeCanaryScanner stands in for the host's canary registry (pkg/ioscan): +// a planted token per agent, matched by plain substring. Encodings are the +// registry's job and are tested there; what the proxy owes the registry is +// the raw write body, verbatim, and this fake records what it was handed. +type fakeCanaryScanner struct { + tokens map[string]string // agent -> token + seen []string +} + +func newFakeCanaryScanner() *fakeCanaryScanner { + return &fakeCanaryScanner{tokens: map[string]string{}} +} + +func (f *fakeCanaryScanner) plant(agent string) string { + tok := "CANARY-" + agent + "-7f3a9c" + f.tokens[agent] = tok + return tok +} + +func (f *fakeCanaryScanner) Scan(agentName, text, source string) (crux.CanaryLeak, bool) { + f.seen = append(f.seen, text) + for agent, tok := range f.tokens { + if strings.Contains(text, tok) { + return crux.CanaryLeak{Agent: agent, Token: tok, Source: source}, true + } + } + return crux.CanaryLeak{}, false +} + func TestInspectCanaryEgressDetectsGitHubWrites(t *testing.T) { - r := ioscan.NewCanaryRegistry("") - c, err := r.Add("scanner") - if err != nil { - t.Fatalf("Add: %v", err) - } - var got ioscan.CanaryLeak + r := newFakeCanaryScanner() + tok := r.plant("scanner") + var got crux.CanaryLeak p := &GitHubProxy{logger: slog.Default()} - p.SetCanaryScanner(true, true, r, func(leak ioscan.CanaryLeak) { got = leak }) - req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/issues/1/comments", strings.NewReader(`{"body":"`+c.Token+`"}`)) + p.SetCanaryScanner(true, true, r, func(leak crux.CanaryLeak) { got = leak }) + req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/issues/1/comments", strings.NewReader(`{"body":"`+tok+`"}`)) reason, deny, detected := p.inspectCanaryEgress("scanner", req) if !detected || !deny || reason == "" { t.Fatalf("got detected=%v deny=%v reason=%q, want detected deny", detected, deny, reason) } - if got.Agent != "scanner" || got.Source == "" { + if got.Agent != "scanner" || got.Token != tok || got.Source != "github:POST /repos/org/repo/issues/1/comments" { t.Fatalf("callback leak = %+v", got) } body, _ := io.ReadAll(req.Body) - if !strings.Contains(string(body), c.Token) { + if !strings.Contains(string(body), tok) { t.Fatalf("request body was not restored: %q", body) } } // The canary exists to catch deliberate exfiltration, and a deliberate -// exfiltrator encodes, so the proxy path — not just the registry — has to see -// through the encodings a GitHub write body can carry. -func TestInspectCanaryEgressDetectsEncodedCanaries(t *testing.T) { - r := ioscan.NewCanaryRegistry("") - c, err := r.Add("scanner") - if err != nil { - t.Fatalf("Add: %v", err) - } - bodies := map[string]string{ - "base64": base64.StdEncoding.EncodeToString([]byte(c.Token)), - "hex": hex.EncodeToString([]byte(c.Token)), - "split": strings.Join(strings.Split(c.Token, ""), ", "), - } - for name, encoded := range bodies { - t.Run(name, func(t *testing.T) { - var got ioscan.CanaryLeak - p := &GitHubProxy{logger: slog.Default()} - p.SetCanaryScanner(true, true, r, func(leak ioscan.CanaryLeak) { got = leak }) - req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/issues/1/comments", strings.NewReader(`{"body":"`+encoded+`"}`)) - _, deny, detected := p.inspectCanaryEgress("scanner", req) - if !detected || !deny { - t.Fatalf("got detected=%v deny=%v, want fail-closed block", detected, deny) - } - if got.Token != c.Token { - t.Fatalf("callback leak = %+v, want token %s", got, c.Token) - } - body, _ := io.ReadAll(req.Body) - if !strings.Contains(string(body), encoded) { - t.Fatalf("request body was not restored: %q", body) - } - }) +// exfiltrator encodes; the registry sees through encodings only if the proxy +// hands it the body exactly as the agent wrote it, so this pins that the +// scanner receives the raw body — not a parsed field, not a truncation. +func TestInspectCanaryEgressHandsScannerTheRawBody(t *testing.T) { + r := newFakeCanaryScanner() + r.plant("scanner") + p := &GitHubProxy{logger: slog.Default()} + p.SetCanaryScanner(true, true, r, nil) + raw := `{"body":"Q0FOQVJZLXNjYW5uZXI=","title":"x"}` + req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/issues", strings.NewReader(raw)) + _, deny, detected := p.inspectCanaryEgress("scanner", req) + if detected || deny { + t.Fatalf("got detected=%v deny=%v for a body the fake does not match", detected, deny) + } + if len(r.seen) != 1 || r.seen[0] != raw { + t.Fatalf("scanner saw %q, want the raw body %q", r.seen, raw) + } + body, _ := io.ReadAll(req.Body) + if string(body) != raw || req.ContentLength != int64(len(raw)) { + t.Fatalf("request body was not restored: %q (len %d)", body, req.ContentLength) } } func TestInspectCanaryEgressAuditOnlyWhenFailOpen(t *testing.T) { - r := ioscan.NewCanaryRegistry("") - c, err := r.Add("scanner") - if err != nil { - t.Fatalf("Add: %v", err) - } + r := newFakeCanaryScanner() + tok := r.plant("scanner") called := false p := &GitHubProxy{logger: slog.Default()} - p.SetCanaryScanner(true, false, r, func(ioscan.CanaryLeak) { called = true }) - req, _ := http.NewRequest(http.MethodPatch, "https://api.github.com/repos/org/repo/pulls/2", strings.NewReader(`{"body":"`+c.Token+`"}`)) + p.SetCanaryScanner(true, false, r, func(crux.CanaryLeak) { called = true }) + req, _ := http.NewRequest(http.MethodPatch, "https://api.github.com/repos/org/repo/pulls/2", strings.NewReader(`{"body":"`+tok+`"}`)) _, deny, detected := p.inspectCanaryEgress("scanner", req) if !detected || deny || !called { t.Fatalf("got detected=%v deny=%v called=%v, want detected audit-only", detected, deny, called) @@ -87,10 +97,8 @@ func TestInspectCanaryEgressAuditOnlyWhenFailOpen(t *testing.T) { } func TestInspectCanaryEgressIgnoresBenignAndDisabled(t *testing.T) { - r := ioscan.NewCanaryRegistry("") - if _, err := r.Add("scanner"); err != nil { - t.Fatalf("Add: %v", err) - } + r := newFakeCanaryScanner() + r.plant("scanner") p := &GitHubProxy{logger: slog.Default()} p.SetCanaryScanner(true, true, r, nil) req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/issues/1/comments", strings.NewReader(`{"body":"hello"}`)) @@ -104,10 +112,22 @@ func TestInspectCanaryEgressIgnoresBenignAndDisabled(t *testing.T) { if detected { t.Fatal("disabled scanner detected leak") } + if len(r.seen) != 1 { + t.Fatalf("disabled scanner was still consulted: %d scans", len(r.seen)) + } +} + +func TestInspectCanaryEgressSkipsWithoutScanner(t *testing.T) { + p := &GitHubProxy{logger: slog.Default()} + p.SetCanaryScanner(true, true, nil, nil) + req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/issues/1/comments", strings.NewReader(`{"body":"hello"}`)) + if _, deny, detected := p.inspectCanaryEgress("scanner", req); detected || deny { + t.Fatalf("got detected=%v deny=%v with no scanner installed, want pass-through", detected, deny) + } } func TestInspectCanaryEgressBlocksOversizedBodies(t *testing.T) { - r := ioscan.NewCanaryRegistry("") + r := newFakeCanaryScanner() p := &GitHubProxy{logger: slog.Default()} p.SetCanaryScanner(true, false, r, nil) req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/issues/1/comments", strings.NewReader(strings.Repeat("x", maxGitHubWriteBodyScan+1))) @@ -115,17 +135,17 @@ func TestInspectCanaryEgressBlocksOversizedBodies(t *testing.T) { if !detected || !deny || !strings.Contains(reason, "too large") { t.Fatalf("got detected=%v deny=%v reason=%q, want oversized block", detected, deny, reason) } + if len(r.seen) != 0 { + t.Fatalf("oversized body was handed to the scanner") + } } func TestInspectCanaryEgressCoversDirectPullCreatePath(t *testing.T) { - r := ioscan.NewCanaryRegistry("") - c, err := r.Add("quality") - if err != nil { - t.Fatalf("Add: %v", err) - } + r := newFakeCanaryScanner() + tok := r.plant("quality") p := &GitHubProxy{logger: slog.Default()} p.SetCanaryScanner(true, false, r, nil) - req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/pulls", strings.NewReader(`{"body":"`+c.Token+`"}`)) + req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/pulls", strings.NewReader(`{"body":"`+tok+`"}`)) _, deny, detected := p.inspectCanaryEgress("quality", req) if !detected || deny { t.Fatalf("got detected=%v deny=%v, want audit-only detection on hard-denied path", detected, deny) @@ -133,14 +153,11 @@ func TestInspectCanaryEgressCoversDirectPullCreatePath(t *testing.T) { } func TestInspectCanaryEgressCoversGitCommitRESTAndBlocksOpaquePush(t *testing.T) { - r := ioscan.NewCanaryRegistry("") - c, err := r.Add("quality") - if err != nil { - t.Fatalf("Add: %v", err) - } + r := newFakeCanaryScanner() + tok := r.plant("quality") p := &GitHubProxy{logger: slog.Default()} p.SetCanaryScanner(true, true, r, nil) - req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/git/commits", strings.NewReader(`{"message":"`+c.Token+`"}`)) + req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/git/commits", strings.NewReader(`{"message":"`+tok+`"}`)) _, deny, detected := p.inspectCanaryEgress("quality", req) if !detected || !deny { t.Fatalf("REST git commit got detected=%v deny=%v, want fail-closed leak", detected, deny) diff --git a/src/pkg/proxy/clienthello_record_test.go b/src/crux/proxy/clienthello_record_test.go similarity index 100% rename from src/pkg/proxy/clienthello_record_test.go rename to src/crux/proxy/clienthello_record_test.go diff --git a/src/pkg/proxy/close_wait_leak_test.go b/src/crux/proxy/close_wait_leak_test.go similarity index 99% rename from src/pkg/proxy/close_wait_leak_test.go rename to src/crux/proxy/close_wait_leak_test.go index 0b1b270039..359eb8cb72 100644 --- a/src/pkg/proxy/close_wait_leak_test.go +++ b/src/crux/proxy/close_wait_leak_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // Regressions for issue #3875: the hive process leaked CLOSE_WAIT sockets to diff --git a/src/pkg/proxy/converse_capability_test.go b/src/crux/proxy/converse_capability_test.go similarity index 99% rename from src/pkg/proxy/converse_capability_test.go rename to src/crux/proxy/converse_capability_test.go index 976fbab233..e3e32823eb 100644 --- a/src/pkg/proxy/converse_capability_test.go +++ b/src/crux/proxy/converse_capability_test.go @@ -4,7 +4,7 @@ import ( "encoding/json" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // The `converse` capability (#4492) is an ORTHOGONAL grant: it widens the two diff --git a/src/pkg/proxy/converse_review_parity_test.go b/src/crux/proxy/converse_review_parity_test.go similarity index 97% rename from src/pkg/proxy/converse_review_parity_test.go rename to src/crux/proxy/converse_review_parity_test.go index 85d415b89d..9db6dbde7a 100644 --- a/src/pkg/proxy/converse_review_parity_test.go +++ b/src/crux/proxy/converse_review_parity_test.go @@ -3,7 +3,7 @@ package proxy import ( "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // TestProxyGrantsPRReviewOnConverse pins the premise that diff --git a/src/pkg/proxy/copilot_anthropic_usage.go b/src/crux/proxy/copilot_anthropic_usage.go similarity index 100% rename from src/pkg/proxy/copilot_anthropic_usage.go rename to src/crux/proxy/copilot_anthropic_usage.go diff --git a/src/pkg/proxy/copilot_anthropic_usage_test.go b/src/crux/proxy/copilot_anthropic_usage_test.go similarity index 88% rename from src/pkg/proxy/copilot_anthropic_usage_test.go rename to src/crux/proxy/copilot_anthropic_usage_test.go index de95f3633a..b7b3a93cf3 100644 --- a/src/pkg/proxy/copilot_anthropic_usage_test.go +++ b/src/crux/proxy/copilot_anthropic_usage_test.go @@ -5,18 +5,13 @@ import ( "bytes" "crypto/tls" "crypto/x509" - "encoding/json" "fmt" "io" "net" "net/http" - "os" - "path/filepath" "strings" "testing" "testing/iotest" - - "github.com/hivecommons/hive/pkg/tokens" ) // anthropicSSEFixture mirrors the event shapes the Copilot CLI receives from @@ -207,25 +202,8 @@ func runAnthropicCopilotSniff(t *testing.T, p *GitHubProxy, agentName, reqBody, return clientGot, <-sawCh } -func readCopilotLiveEntries(t *testing.T, dir, agent string) []tokens.SessionEntry { - t.Helper() - data, err := os.ReadFile(filepath.Join(dir, "copilot-live-"+agent+".jsonl")) - if err != nil { - t.Fatalf("reading usage file: %v", err) - } - var out []tokens.SessionEntry - for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { - var e tokens.SessionEntry - if err := json.Unmarshal([]byte(line), &e); err != nil { - t.Fatalf("bad usage line %q: %v", line, err) - } - out = append(out, e) - } - return out -} - func TestProxyCopilotHTTP_AnthropicStreaming(t *testing.T) { - p, dir := newTestProxyWithCA(t) + p, sink := newTestProxyWithCA(t) reqBody := `{"model":"claude-sonnet-4.5","stream":true,"max_tokens":1024,"messages":[{"role":"user","content":"hi"}]}` got, saw := runAnthropicCopilotSniff(t, p, "worker1", reqBody, "text/event-stream", anthropicSSEFixture) @@ -239,19 +217,22 @@ func TestProxyCopilotHTTP_AnthropicStreaming(t *testing.T) { t.Errorf("OpenAI stream_options injected into Anthropic request: %s", saw) } - entries := readCopilotLiveEntries(t, dir, "worker1") + recs := sink.forAgent("worker1") + if len(recs) == 0 { + t.Fatal("no usage recorded for worker1") + } var in, out, cr, cc int64 - for _, e := range entries { - if e.Backend != tokens.BackendCopilot { - t.Errorf("entry backend = %q, want %q", e.Backend, tokens.BackendCopilot) + for _, r := range recs { + if !r.copilot { + t.Errorf("usage recorded through Record, want the Copilot path: %+v", r) } - if e.Role == "assistant" && e.Model != "claude-sonnet-4.5" { - t.Errorf("assistant entry model = %q, want claude-sonnet-4.5", e.Model) + if r.model != "claude-sonnet-4.5" { + t.Errorf("recorded model = %q, want claude-sonnet-4.5", r.model) } - in += e.InputTokens - out += e.OutputTokens - cr += e.CacheRead - cc += e.CacheCreation + in += r.in + out += r.out + cr += r.cacheRead + cc += r.cacheCreation } if in != 12 || out != 57 || cr != 4000 || cc != 300 { t.Errorf("recorded usage in=%d out=%d cache_read=%d cache_creation=%d, want 12/57/4000/300", in, out, cr, cc) @@ -259,14 +240,14 @@ func TestProxyCopilotHTTP_AnthropicStreaming(t *testing.T) { } func TestProxyCopilotHTTP_AnthropicJSONAutoModel(t *testing.T) { - p, dir := newTestProxyWithCA(t) + p, sink := newTestProxyWithCA(t) reqBody := `{"model":"auto","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}` got, _ := runAnthropicCopilotSniff(t, p, "worker2", reqBody, "application/json", anthropicJSONFixture) if string(got) != anthropicJSONFixture { t.Errorf("client did not receive the JSON body verbatim: %s", got) } - model, in, out := readInferenceUsage(t, dir, "worker2") + model, in, out := readInferenceUsage(t, sink, "worker2") if model != "claude-opus-4.5" || in != 25 || out != 9 { t.Errorf("recorded usage = (%q,%d,%d), want (claude-opus-4.5,25,9)", model, in, out) } diff --git a/src/pkg/proxy/copilot_sniff_test.go b/src/crux/proxy/copilot_sniff_test.go similarity index 87% rename from src/pkg/proxy/copilot_sniff_test.go rename to src/crux/proxy/copilot_sniff_test.go index 5ea7438b7f..946b3f7787 100644 --- a/src/pkg/proxy/copilot_sniff_test.go +++ b/src/crux/proxy/copilot_sniff_test.go @@ -5,64 +5,101 @@ import ( "bytes" "crypto/tls" "crypto/x509" - "encoding/json" "fmt" "io" "log/slog" "net" "net/http" - "os" - "path/filepath" "strings" + "sync" "testing" "time" - - "github.com/hivecommons/hive/pkg/tokens" ) +// usageRecord is one UsageSink call the test sink captured. +type usageRecord struct { + agent, model string + in, out, cacheRead, cacheCreation int64 + copilot bool +} + +// memUsageSink stands in for the host's token sink (pkg/tokens): it keeps +// every record in memory so a test reads back exactly what the proxy +// attributed, with no file format in between. +type memUsageSink struct { + mu sync.Mutex + records []usageRecord +} + +func (m *memUsageSink) add(r usageRecord) { + m.mu.Lock() + defer m.mu.Unlock() + m.records = append(m.records, r) +} + +func (m *memUsageSink) Record(agent, model string, in, out int64) { + m.add(usageRecord{agent: agent, model: model, in: in, out: out}) +} + +func (m *memUsageSink) RecordCopilot(agent, model string, in, out int64) { + m.add(usageRecord{agent: agent, model: model, in: in, out: out, copilot: true}) +} + +func (m *memUsageSink) RecordCopilotWithCache(agent, model string, in, out, cacheRead, cacheCreation int64) { + m.add(usageRecord{agent: agent, model: model, in: in, out: out, cacheRead: cacheRead, cacheCreation: cacheCreation, copilot: true}) +} + +// forAgent returns the records attributed to agent. +func (m *memUsageSink) forAgent(agent string) []usageRecord { + m.mu.Lock() + defer m.mu.Unlock() + var out []usageRecord + for _, r := range m.records { + if r.agent == agent { + out = append(out, r) + } + } + return out +} + // newTestProxyWithCA builds a GitHubProxy with a fresh in-memory CA and cert -// cache, plus a token sink writing to a temp dir. Returns the proxy and the -// metrics dir for reading back recorded usage. -func newTestProxyWithCA(t *testing.T) (*GitHubProxy, string) { +// cache, plus an in-memory usage sink. Returns the proxy and the sink for +// reading back recorded usage. +func newTestProxyWithCA(t *testing.T) (*GitHubProxy, *memUsageSink) { t.Helper() caCert, caX509, err := generateCA() if err != nil { t.Fatal(err) } - dir := t.TempDir() + sink := &memUsageSink{} p := &GitHubProxy{ caCert: caCert, caX509: caX509, logger: slog.New(slog.NewTextHandler(io.Discard, nil)), certCache: make(map[string]cachedCert), - tokenSink: tokens.NewInferenceSink(dir, slog.Default()), + tokenSink: sink, } - return p, dir + return p, sink } -// readInferenceUsage reads back the cumulative usage the sink wrote for an -// agent (copilot-live-.jsonl) and returns the model plus total in/out -// tokens. -func readInferenceUsage(t *testing.T, dir, agent string) (model string, in, out int64) { +// readInferenceUsage reads back the cumulative usage the proxy recorded for +// an agent on the Copilot path and returns the model plus total in/out +// tokens; it fails the test when nothing was recorded. +func readInferenceUsage(t *testing.T, sink *memUsageSink, agent string) (model string, in, out int64) { t.Helper() - path := filepath.Join(dir, "copilot-live-"+agent+".jsonl") - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("reading usage file %s: %v", path, err) + recs := sink.forAgent(agent) + if len(recs) == 0 { + t.Fatalf("no usage recorded for agent %q", agent) } - for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { - if line == "" { - continue - } - var e tokens.SessionEntry - if err := json.Unmarshal([]byte(line), &e); err != nil { - t.Fatalf("bad usage line: %v", err) + for _, r := range recs { + if !r.copilot { + t.Errorf("agent %q: usage recorded through Record, want the Copilot path", agent) } - if e.Model != "" { - model = e.Model + if r.model != "" { + model = r.model } - in += e.InputTokens - out += e.OutputTokens + in += r.in + out += r.out } return model, in, out } @@ -75,7 +112,7 @@ func readInferenceUsage(t *testing.T, dir, agent string) (model string, in, out // We use two net.Pipe pairs: one for client<->proxy (TLS on the client-facing // side), one for proxy<->upstream (plain, upstream speaks HTTP/1.1 directly). func TestProxyCopilotHTTP_NonStreaming(t *testing.T) { - p, dir := newTestProxyWithCA(t) + p, sink := newTestProxyWithCA(t) // Upstream: a plain TCP listener speaking HTTP/1.1 with a JSON completion. upstreamLn, err := net.Listen("tcp", "127.0.0.1:0") @@ -166,14 +203,14 @@ func TestProxyCopilotHTTP_NonStreaming(t *testing.T) { <-proxyDone - model, in, out := readInferenceUsage(t, dir, "worker1") + model, in, out := readInferenceUsage(t, sink, "worker1") if model != "gpt-5" || in != 120 || out != 34 { t.Errorf("recorded usage = (%q,%d,%d), want (gpt-5,120,34)", model, in, out) } } func TestProxyCopilotHTTP_Streaming(t *testing.T) { - p, dir := newTestProxyWithCA(t) + p, sink := newTestProxyWithCA(t) upstreamLn, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -264,7 +301,7 @@ func TestProxyCopilotHTTP_Streaming(t *testing.T) { <-proxyDone - model, in, out := readInferenceUsage(t, dir, "worker2") + model, in, out := readInferenceUsage(t, sink, "worker2") if model != "claude-sonnet-4.5" || in != 200 || out != 88 { t.Errorf("recorded usage = (%q,%d,%d), want (claude-sonnet-4.5,200,88)", model, in, out) } @@ -273,7 +310,7 @@ func TestProxyCopilotHTTP_Streaming(t *testing.T) { // TestProxyCopilotHTTP_NonCompletionPassthrough verifies non-completion requests // (e.g. /models) pass through without recording usage. func TestProxyCopilotHTTP_NonCompletionPassthrough(t *testing.T) { - p, dir := newTestProxyWithCA(t) + p, sink := newTestProxyWithCA(t) upstreamLn, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -343,8 +380,8 @@ func TestProxyCopilotHTTP_NonCompletionPassthrough(t *testing.T) { <-proxyDone // No usage file should have been written. - if _, err := os.Stat(filepath.Join(dir, "copilot-live-worker3.jsonl")); !os.IsNotExist(err) { - t.Errorf("non-completion request should not record usage; file exists: %v", err) + if recs := sink.forAgent("worker3"); len(recs) != 0 { + t.Errorf("non-completion request should not record usage; got %+v", recs) } } @@ -368,7 +405,7 @@ func TestCopilotSniffRoutingGuard(t *testing.T) { // (faked) upstream via the test seam, and records usage. This covers // handleCopilotSniff, which in production dials the real Copilot host by name. func TestHandleCopilotSniff_EndToEnd(t *testing.T) { - p, dir := newTestProxyWithCA(t) + p, sink := newTestProxyWithCA(t) // Fake upstream: a plain TCP server answering a completion. upstreamLn, err := net.Listen("tcp", "127.0.0.1:0") @@ -458,7 +495,7 @@ func TestHandleCopilotSniff_EndToEnd(t *testing.T) { <-done - model, in, out := readInferenceUsage(t, dir, "worker9") + model, in, out := readInferenceUsage(t, sink, "worker9") if model != "gpt-5" || in != 11 || out != 4 { t.Errorf("recorded (%q,%d,%d), want (gpt-5,11,4)", model, in, out) } @@ -551,7 +588,7 @@ func TestDialCopilotUpstream_DefaultErrors(t *testing.T) { // TestProxyCopilotHTTP_NonSuccessCompletion covers the branch where a completion // request gets a non-2xx response — it must pass through without recording usage. func TestProxyCopilotHTTP_NonSuccessCompletion(t *testing.T) { - p, dir := newTestProxyWithCA(t) + p, sink := newTestProxyWithCA(t) upstreamLn, _ := net.Listen("tcp", "127.0.0.1:0") defer upstreamLn.Close() @@ -613,8 +650,8 @@ func TestProxyCopilotHTTP_NonSuccessCompletion(t *testing.T) { resp.Body.Close() <-done - if _, err := os.Stat(filepath.Join(dir, "copilot-live-w4.jsonl")); !os.IsNotExist(err) { - t.Errorf("non-2xx completion must not record usage") + if recs := sink.forAgent("w4"); len(recs) != 0 { + t.Errorf("non-2xx completion must not record usage; got %+v", recs) } } @@ -622,7 +659,7 @@ func TestProxyCopilotHTTP_NonSuccessCompletion(t *testing.T) { // continuation (req.Close/resp.Close false) and accumulation across two // completions on one connection. func TestProxyCopilotHTTP_KeepAliveTwoRequests(t *testing.T) { - p, dir := newTestProxyWithCA(t) + p, sink := newTestProxyWithCA(t) upstreamLn, _ := net.Listen("tcp", "127.0.0.1:0") defer upstreamLn.Close() @@ -690,7 +727,7 @@ func TestProxyCopilotHTTP_KeepAliveTwoRequests(t *testing.T) { clientConn.Close() <-done - _, in, out := readInferenceUsage(t, dir, "w5") + _, in, out := readInferenceUsage(t, sink, "w5") if in != 20 || out != 10 { t.Errorf("keep-alive accumulated usage = (%d,%d), want (20,10)", in, out) } @@ -699,7 +736,7 @@ func TestProxyCopilotHTTP_KeepAliveTwoRequests(t *testing.T) { // TestProxyCopilotHTTP_ZeroUsageNotRecorded covers the branch where a completion // response carries no usage — nothing is recorded. func TestProxyCopilotHTTP_ZeroUsageNotRecorded(t *testing.T) { - p, dir := newTestProxyWithCA(t) + p, sink := newTestProxyWithCA(t) upstreamLn, _ := net.Listen("tcp", "127.0.0.1:0") defer upstreamLn.Close() go func() { @@ -754,8 +791,8 @@ func TestProxyCopilotHTTP_ZeroUsageNotRecorded(t *testing.T) { resp.Body.Close() } <-done - if _, err := os.Stat(filepath.Join(dir, "copilot-live-w6.jsonl")); !os.IsNotExist(err) { - t.Errorf("zero-usage completion must not record") + if recs := sink.forAgent("w6"); len(recs) != 0 { + t.Errorf("zero-usage completion must not record; got %+v", recs) } } diff --git a/src/pkg/proxy/copilot_sniff_write_error_test.go b/src/crux/proxy/copilot_sniff_write_error_test.go similarity index 100% rename from src/pkg/proxy/copilot_sniff_write_error_test.go rename to src/crux/proxy/copilot_sniff_write_error_test.go diff --git a/src/pkg/proxy/copilot_usage.go b/src/crux/proxy/copilot_usage.go similarity index 100% rename from src/pkg/proxy/copilot_usage.go rename to src/crux/proxy/copilot_usage.go diff --git a/src/pkg/proxy/copilot_usage_test.go b/src/crux/proxy/copilot_usage_test.go similarity index 100% rename from src/pkg/proxy/copilot_usage_test.go rename to src/crux/proxy/copilot_usage_test.go diff --git a/src/pkg/proxy/denial_message.go b/src/crux/proxy/denial_message.go similarity index 100% rename from src/pkg/proxy/denial_message.go rename to src/crux/proxy/denial_message.go diff --git a/src/pkg/proxy/denial_message_test.go b/src/crux/proxy/denial_message_test.go similarity index 98% rename from src/pkg/proxy/denial_message_test.go rename to src/crux/proxy/denial_message_test.go index 3d0be06ec6..5ec5a4291e 100644 --- a/src/pkg/proxy/denial_message_test.go +++ b/src/crux/proxy/denial_message_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // #12127: every proxy 403 is ONE line that ends in a machine-readable diff --git a/src/pkg/proxy/entitlement.go b/src/crux/proxy/entitlement.go similarity index 100% rename from src/pkg/proxy/entitlement.go rename to src/crux/proxy/entitlement.go diff --git a/src/pkg/proxy/entitlement_default_model_test.go b/src/crux/proxy/entitlement_default_model_test.go similarity index 100% rename from src/pkg/proxy/entitlement_default_model_test.go rename to src/crux/proxy/entitlement_default_model_test.go diff --git a/src/pkg/proxy/entitlement_test.go b/src/crux/proxy/entitlement_test.go similarity index 100% rename from src/pkg/proxy/entitlement_test.go rename to src/crux/proxy/entitlement_test.go diff --git a/src/crux/proxy/forge_contract_test.go b/src/crux/proxy/forge_contract_test.go new file mode 100644 index 0000000000..d334813316 --- /dev/null +++ b/src/crux/proxy/forge_contract_test.go @@ -0,0 +1,290 @@ +package proxy + +import ( + "fmt" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/agent" +) + +// ForgeEndpoint specifies the HTTP method and path for a forge operation. +type ForgeEndpoint struct { + Method string + Path string +} + +// ForgeRuleContract defines the expected write-gate behavior across all supported +// forges. Every rule in this contract MUST provide the corresponding endpoint for +// GitHub, GitLab, and Gitea so that any rule added for one forge must be answered +// for all three. +type ForgeRuleContract struct { + Name string + MinMode agent.AgentMode + Capability func(agent.AgentCapabilities) bool + HardDeny bool + + GitHub ForgeEndpoint + GitLab ForgeEndpoint + Gitea ForgeEndpoint +} + +// ForgeRulesContract returns the canonical table-driven rules contract shared +// across GitHub, GitLab, and Gitea. +var ForgeRulesContract = []ForgeRuleContract{ + { + Name: "ReadIssues", + MinMode: agent.ModeAdvisory, + GitHub: ForgeEndpoint{Method: "GET", Path: "/repos/org/repo/issues"}, + GitLab: ForgeEndpoint{Method: "GET", Path: "/api/v4/projects/org%2Frepo/issues"}, + Gitea: ForgeEndpoint{Method: "GET", Path: "/api/v1/repos/org/repo/issues"}, + }, + { + Name: "ReadChangeRequests", + MinMode: agent.ModeAdvisory, + GitHub: ForgeEndpoint{Method: "GET", Path: "/repos/org/repo/pulls"}, + GitLab: ForgeEndpoint{Method: "GET", Path: "/api/v4/projects/org%2Frepo/merge_requests"}, + Gitea: ForgeEndpoint{Method: "GET", Path: "/api/v1/repos/org/repo/pulls"}, + }, + { + Name: "CreateIssue", + MinMode: agent.ModeIssuesOnly, + GitHub: ForgeEndpoint{Method: "POST", Path: "/repos/org/repo/issues"}, + GitLab: ForgeEndpoint{Method: "POST", Path: "/api/v4/projects/org%2Frepo/issues"}, + Gitea: ForgeEndpoint{Method: "POST", Path: "/api/v1/repos/org/repo/issues"}, + }, + { + Name: "EditIssue", + MinMode: agent.ModeIssuesOnly, + GitHub: ForgeEndpoint{Method: "PATCH", Path: "/repos/org/repo/issues/42"}, + GitLab: ForgeEndpoint{Method: "PUT", Path: "/api/v4/projects/org%2Frepo/issues/42"}, + Gitea: ForgeEndpoint{Method: "PATCH", Path: "/api/v1/repos/org/repo/issues/42"}, + }, + { + Name: "CommentIssue", + MinMode: agent.ModeIssuesOnly, + Capability: agent.AgentCapabilities.CanConverse, + GitHub: ForgeEndpoint{Method: "POST", Path: "/repos/org/repo/issues/42/comments"}, + GitLab: ForgeEndpoint{Method: "POST", Path: "/api/v4/projects/org%2Frepo/issues/42/notes"}, + Gitea: ForgeEndpoint{Method: "POST", Path: "/api/v1/repos/org/repo/issues/42/comments"}, + }, + { + Name: "AddLabels", + MinMode: agent.ModeIssuesOnly, + GitHub: ForgeEndpoint{Method: "POST", Path: "/repos/org/repo/issues/42/labels"}, + GitLab: ForgeEndpoint{Method: "POST", Path: "/api/v4/projects/org%2Frepo/labels"}, + Gitea: ForgeEndpoint{Method: "POST", Path: "/api/v1/repos/org/repo/issues/42/labels"}, + }, + { + Name: "RemoveLabelHoldGate", + MinMode: agent.ModeIssuesOnly, + GitHub: ForgeEndpoint{Method: "DELETE", Path: "/repos/org/repo/issues/42/labels/hold"}, + GitLab: ForgeEndpoint{Method: "PUT", Path: "/api/v4/projects/org%2Frepo/issues/42"}, + Gitea: ForgeEndpoint{Method: "DELETE", Path: "/api/v1/repos/org/repo/issues/42/labels/hold"}, + }, + { + Name: "CreateChangeRequestHardDeny", + MinMode: agent.ModeIssuesAndPRs, + HardDeny: true, + GitHub: ForgeEndpoint{Method: "POST", Path: "/repos/org/repo/pulls"}, + GitLab: ForgeEndpoint{Method: "POST", Path: "/api/v4/projects/org%2Frepo/merge_requests"}, + Gitea: ForgeEndpoint{Method: "POST", Path: "/api/v1/repos/org/repo/pulls"}, + }, + { + Name: "MergeChangeRequestHardDeny", + MinMode: agent.ModeIssuesPRsMerge, + HardDeny: true, + GitHub: ForgeEndpoint{Method: "PUT", Path: "/repos/org/repo/pulls/42/merge"}, + GitLab: ForgeEndpoint{Method: "PUT", Path: "/api/v4/projects/org%2Frepo/merge_requests/42/merge"}, + Gitea: ForgeEndpoint{Method: "POST", Path: "/api/v1/repos/org/repo/pulls/42/merge"}, + }, + { + Name: "EditChangeRequest", + MinMode: agent.ModeIssuesAndPRs, + GitHub: ForgeEndpoint{Method: "PATCH", Path: "/repos/org/repo/pulls/42"}, + GitLab: ForgeEndpoint{Method: "PUT", Path: "/api/v4/projects/org%2Frepo/merge_requests/42"}, + Gitea: ForgeEndpoint{Method: "PATCH", Path: "/api/v1/repos/org/repo/pulls/42"}, + }, + { + Name: "ReviewChangeRequest", + MinMode: agent.ModeIssuesAndPRs, + Capability: agent.AgentCapabilities.CanConverse, + GitHub: ForgeEndpoint{Method: "POST", Path: "/repos/org/repo/pulls/42/reviews"}, + GitLab: ForgeEndpoint{Method: "POST", Path: "/api/v4/projects/org%2Frepo/merge_requests/42/notes"}, + Gitea: ForgeEndpoint{Method: "POST", Path: "/api/v1/repos/org/repo/pulls/42/reviews"}, + }, + { + Name: "GitPush", + MinMode: agent.ModeIssuesAndPRs, + GitHub: ForgeEndpoint{Method: "POST", Path: "/repos/org/repo/git/refs"}, + GitLab: ForgeEndpoint{Method: "POST", Path: "/api/v4/projects/org%2Frepo/repository/branches"}, + Gitea: ForgeEndpoint{Method: "POST", Path: "/api/v1/repos/org/repo/git/refs"}, + }, + { + Name: "UpdateBranchRebase", + MinMode: agent.ModeIssuesPRsMerge, + GitHub: ForgeEndpoint{Method: "PUT", Path: "/repos/org/repo/pulls/42/update-branch"}, + GitLab: ForgeEndpoint{Method: "PUT", Path: "/api/v4/projects/org%2Frepo/merge_requests/42/rebase"}, + Gitea: ForgeEndpoint{Method: "POST", Path: "/api/v1/repos/org/repo/pulls/42/update"}, + }, +} + +// RunForgeRulesContractTest executes the shared write-gate contract test across +// GitHub, GitLab, and Gitea. +func RunForgeRulesContractTest(t *testing.T) { + t.Helper() + + forges := []struct { + name string + get func(ForgeRuleContract) ForgeEndpoint + }{ + {"GitHub", func(r ForgeRuleContract) ForgeEndpoint { return r.GitHub }}, + {"GitLab", func(r ForgeRuleContract) ForgeEndpoint { return r.GitLab }}, + {"Gitea", func(r ForgeRuleContract) ForgeEndpoint { return r.Gitea }}, + } + + for _, rule := range ForgeRulesContract { + rule := rule + // Enforce contract completeness: every rule must have non-empty endpoints for all forges. + if rule.GitHub.Method == "" || rule.GitHub.Path == "" { + t.Fatalf("contract rule %q must define GitHub endpoint", rule.Name) + } + if rule.GitLab.Method == "" || rule.GitLab.Path == "" { + t.Fatalf("contract rule %q must define GitLab endpoint", rule.Name) + } + if rule.Gitea.Method == "" || rule.Gitea.Path == "" { + t.Fatalf("contract rule %q must define Gitea endpoint", rule.Name) + } + + t.Run(rule.Name, func(t *testing.T) { + for _, f := range forges { + ep := f.get(rule) + t.Run(f.name, func(t *testing.T) { + if rule.HardDeny { + // Hard-denied operations MUST be refused at EVERY agent mode, + // and MUST return a directive message pointing to the request-file path. + for _, mode := range allModes { + if AllowedByMode(mode, ep.Method, ep.Path) { + t.Errorf("%s %s: hard-deny %s must be refused at mode %v", + f.name, rule.Name, ep.Path, mode) + } + } + msg, denied := DeniedMessage(ep.Method, ep.Path) + if !denied || msg == "" { + t.Errorf("%s %s: expected DeniedMessage for hard-deny %s %s, got denied=%v msg=%q", + f.name, rule.Name, ep.Method, ep.Path, denied, msg) + } + return + } + + // Standard mode-ladder enforcement. + for _, mode := range allModes { + allowed := AllowedByMode(mode, ep.Method, ep.Path) + want := mode >= rule.MinMode + if allowed != want { + t.Errorf("%s %s at %v: got allowed=%v, want %v (%s %s)", + f.name, rule.Name, mode, allowed, want, ep.Method, ep.Path) + } + } + + // Capability checks (e.g. converse widening conversation routes). + if rule.Capability != nil { + converseCaps := agent.AgentCapabilities{Converse: true} + noCaps := agent.AgentCapabilities{} + + // Below MinMode, holding converse MUST grant access. + if !AllowedByModeCaps(agent.ModeAdvisory, converseCaps, ep.Method, ep.Path) { + t.Errorf("%s %s: converse capability must grant access at ModeAdvisory for %s %s", + f.name, rule.Name, ep.Method, ep.Path) + } + // Below MinMode, zero capabilities MUST still be denied. + if AllowedByModeCaps(agent.ModeAdvisory, noCaps, ep.Method, ep.Path) { + t.Errorf("%s %s: zero capabilities must NOT grant access at ModeAdvisory for %s %s", + f.name, rule.Name, ep.Method, ep.Path) + } + } + }) + } + }) + } + + // Default-deny (fail-closed) contract test for all three forges: + // Unmatched or unknown paths MUST be denied for all agent modes. + t.Run("DefaultDenyUnmatchedRoutes", func(t *testing.T) { + unmatchedPaths := []struct { + forge string + method string + path string + }{ + {"GitHub", "POST", "/repos/org/repo/unknown_endpoint"}, + {"GitHub", "DELETE", "/repos/org/repo/unknown_endpoint"}, + {"GitLab", "POST", "/api/v4/projects/org%2Frepo/unknown_endpoint"}, + {"GitLab", "PUT", "/api/v4/projects/org%2Frepo/unknown_endpoint"}, + {"Gitea", "POST", "/api/v1/repos/org/repo/unknown_endpoint"}, + {"Gitea", "PATCH", "/api/v1/repos/org/repo/unknown_endpoint"}, + } + + for _, tc := range unmatchedPaths { + name := fmt.Sprintf("%s_%s_%s", tc.forge, tc.method, strings.TrimPrefix(tc.path, "/")) + t.Run(name, func(t *testing.T) { + for _, mode := range allModes { + if AllowedByMode(mode, tc.method, tc.path) { + t.Errorf("%s: unmatched route %s %s must be denied at mode %v", + tc.forge, tc.method, tc.path, mode) + } + } + }) + } + }) + + // Variations and path equivalences across forges. + t.Run("ForgePathVariations", func(t *testing.T) { + variations := []struct { + name string + method string + path string + minMode agent.AgentMode + hardDeny bool + }{ + // Gitea without /api/v1 prefix + {"Gitea_Pulls_Bare_Deny", "POST", "/repos/org/repo/pulls", agent.ModeIssuesAndPRs, true}, + {"Gitea_Merge_Bare_Deny", "POST", "/repos/org/repo/pulls/1/merge", agent.ModeIssuesPRsMerge, true}, + {"Gitea_Update_Bare", "POST", "/repos/org/repo/pulls/1/update", agent.ModeIssuesPRsMerge, false}, + {"Gitea_UpdateBranch_Bare", "POST", "/repos/org/repo/pulls/1/update-branch", agent.ModeIssuesPRsMerge, false}, + + // GitLab with numeric project ID + {"GitLab_NumericID_MR_Deny", "POST", "/api/v4/projects/12345/merge_requests", agent.ModeIssuesAndPRs, true}, + {"GitLab_NumericID_Merge_PUT_Deny", "PUT", "/api/v4/projects/12345/merge_requests/1/merge", agent.ModeIssuesPRsMerge, true}, + {"GitLab_NumericID_Merge_POST_Deny", "POST", "/api/v4/projects/12345/merge_requests/1/merge", agent.ModeIssuesPRsMerge, true}, + {"GitLab_NumericID_Rebase", "PUT", "/api/v4/projects/12345/merge_requests/1/rebase", agent.ModeIssuesPRsMerge, false}, + + // GitLab discussions + {"GitLab_Issue_Discussions", "POST", "/api/v4/projects/org%2Frepo/issues/1/discussions", agent.ModeIssuesOnly, false}, + {"GitLab_MR_Discussions", "POST", "/api/v4/projects/org%2Frepo/merge_requests/1/discussions", agent.ModeIssuesAndPRs, false}, + } + + for _, v := range variations { + t.Run(v.name, func(t *testing.T) { + if v.hardDeny { + for _, mode := range allModes { + if AllowedByMode(mode, v.method, v.path) { + t.Errorf("%s: hard deny failed for %s %s at mode %v", v.name, v.method, v.path, mode) + } + } + return + } + for _, mode := range allModes { + allowed := AllowedByMode(mode, v.method, v.path) + want := mode >= v.minMode + if allowed != want { + t.Errorf("%s at %v: got %v, want %v (%s %s)", v.name, mode, allowed, want, v.method, v.path) + } + } + }) + } + }) +} + +func TestForgeRulesContract(t *testing.T) { + RunForgeRulesContractTest(t) +} diff --git a/src/pkg/proxy/gateway_health_signal_test.go b/src/crux/proxy/gateway_health_signal_test.go similarity index 100% rename from src/pkg/proxy/gateway_health_signal_test.go rename to src/crux/proxy/gateway_health_signal_test.go diff --git a/src/pkg/proxy/gateway_path_canary_test.go b/src/crux/proxy/gateway_path_canary_test.go similarity index 99% rename from src/pkg/proxy/gateway_path_canary_test.go rename to src/crux/proxy/gateway_path_canary_test.go index 9dd9224f22..a9a1443c22 100644 --- a/src/pkg/proxy/gateway_path_canary_test.go +++ b/src/crux/proxy/gateway_path_canary_test.go @@ -10,7 +10,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // This file is the hermetic canary for #6515: the OpenAI-compatible gateway diff --git a/src/pkg/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go similarity index 92% rename from src/pkg/proxy/github_proxy.go rename to src/crux/proxy/github_proxy.go index 8e9b55427d..ea526f744e 100644 --- a/src/pkg/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -30,13 +30,11 @@ import ( "syscall" "time" - "github.com/hivecommons/hive/pkg/agent" - "github.com/hivecommons/hive/pkg/crux/config" - hgithub "github.com/hivecommons/hive/pkg/github" - "github.com/hivecommons/hive/pkg/inferencehealth" - "github.com/hivecommons/hive/pkg/ioscan" - "github.com/hivecommons/hive/pkg/issueshape" - "github.com/hivecommons/hive/pkg/tokens" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/inferencehealth" + "github.com/hivecommons/hive/crux/issueshape" ) const ( @@ -208,6 +206,13 @@ type GitHubProxy struct { // header as an agent's identity. proxyAdvisoryOK bool + // allowPrivateForward snapshots HIVE_PROXY_ALLOW_PRIVATE_FORWARD at + // construction (a boot-time deployment choice like proxyAdvisoryOK). The + // zero value keeps the plain-HTTP forward guard ON: non-routable + // destinations (loopback, link-local/metadata, RFC 1918, CGNAT, multicast) + // are refused on the plain path. See plain_forward_guard.go. + allowPrivateForward bool + // injectGHAuth snapshots config.ProxyInjectGHAuth() at construction (#1861, // opt-in, default off — a boot-time deployment choice like proxyAdvisoryOK). // When true, the proxy is the ONLY holder of usable GitHub credentials on @@ -229,6 +234,9 @@ type GitHubProxy struct { mu sync.RWMutex violations map[string]int // agent name -> blocked request count + // blockedSeen holds the agent|method|path keys already logged at WARN so + // a repeated refusal logs once and then at Debug. Bounded by maxViolationLog. + blockedSeen map[string]struct{} writes writeCounters @@ -264,8 +272,13 @@ type GitHubProxy struct { // tokenSink records per-agent token usage for bare-mode inference // agents, whose usage the file-scanning token collector cannot see - // otherwise. May be nil (Record is a safe no-op on a nil sink). - tokenSink *tokens.InferenceSink + // otherwise. May be nil; the host installs it with SetTokenSink. + tokenSink UsageSink + + // restAccounting receives one record per relayed GitHub REST response + // and supplies the shared core-bucket snapshot; nil until the host + // installs it with SetGitHubRESTAccounting. + restAccounting GitHubRESTAccounting githubBudget *agentGitHubBudget githubBudgetCap func(agentName string) int @@ -287,8 +300,8 @@ type GitHubProxy struct { canariesEnabled bool canaryFailClosed bool - canaryRegistry *ioscan.CanaryRegistry - canaryLeakFunc func(ioscan.CanaryLeak) + canaryRegistry CanaryScanner + canaryLeakFunc func(crux.CanaryLeak) // linearCredential resolves the hive's CURRENT Linear write credential // for injection into agent requests to api.linear.app — see @@ -373,7 +386,7 @@ func (p *GitHubProxy) dialCopilotUpstream(host string) (net.Conn, error) { // SetTokenSink wires the inference token sink so the translator can record // per-agent usage from upstream OpenAI responses into the shared token store. -func (p *GitHubProxy) SetTokenSink(sink *tokens.InferenceSink) { +func (p *GitHubProxy) SetTokenSink(sink UsageSink) { p.tokenSink = sink } @@ -419,7 +432,10 @@ func (p *GitHubProxy) hostNeedsMITM(host string) bool { return p.injectGHAuth && IsGitHubHost(host) } -func (p *GitHubProxy) SetCanaryScanner(enabled, failClosed bool, reg *ioscan.CanaryRegistry, onLeak func(ioscan.CanaryLeak)) { +// SetCanaryScanner installs the host's canary registry and leak handler. +// enabled gates the scan; failClosed turns a detection (or an unscannable +// body) into a refusal rather than an audit-only event. +func (p *GitHubProxy) SetCanaryScanner(enabled, failClosed bool, reg CanaryScanner, onLeak func(crux.CanaryLeak)) { p.canariesEnabled = enabled p.canaryFailClosed = failClosed p.canaryRegistry = reg @@ -486,27 +502,36 @@ func newGitHubProxyWithCA(caCert tls.Certificate, caX509 *x509.Certificate, logg "env", config.ProxyInjectGHAuthEnv) } + // Plain-HTTP forward guard: on by default, lifted only by an explicit + // operator opt-in, and announced when lifted so the log says the proxy may + // dial the private estate on an agent's behalf. + allowPrivateForward := plainForwardAllowPrivate() + if allowPrivateForward { + logger.Warn("proxy: plain-HTTP forwards to non-routable destinations are ALLOWED — agents can reach loopback, link-local (cloud metadata) and private-network services through this proxy", + "env", plainForwardAllowPrivateEnv) + } + p := &GitHubProxy{ - listenAddr: fmt.Sprintf("127.0.0.1:%d", proxyListenPort), - caCert: caCert, - caX509: caX509, - logger: logger, - uidMap: uidMap, - proxyAdvisoryOK: advisoryOK, - injectGHAuth: injectGHAuth, - allowedRepos: allowed, - violations: make(map[string]int), - certCache: make(map[string]cachedCert), - inference: newInferenceRouter(), - entitlements: newEntitlementStore(), - inferenceAuth: &inferenceAuthState{}, - inferenceLastErrors: newInferenceLastErrorState(), - inferenceBudget: &inferenceBudgetState{}, - gatewayHealth: inferencehealth.NewStore(), - githubBudget: newAgentGitHubBudget(time.Now), - githubBudgetCap: func(string) int { return config.DefaultAgentsGitHubAPIHourlyCap }, - githubReserveFloor: func() int { return config.DefaultGitHubAgentReserveFloor }, - githubReserveSnapshot: hgithub.CoreRateLimitSnapshot, + listenAddr: fmt.Sprintf("127.0.0.1:%d", proxyListenPort), + caCert: caCert, + caX509: caX509, + logger: logger, + uidMap: uidMap, + proxyAdvisoryOK: advisoryOK, + allowPrivateForward: allowPrivateForward, + injectGHAuth: injectGHAuth, + allowedRepos: allowed, + violations: make(map[string]int), + certCache: make(map[string]cachedCert), + inference: newInferenceRouter(), + entitlements: newEntitlementStore(), + inferenceAuth: &inferenceAuthState{}, + inferenceLastErrors: newInferenceLastErrorState(), + inferenceBudget: &inferenceBudgetState{}, + gatewayHealth: inferencehealth.NewStore(), + githubBudget: newAgentGitHubBudget(time.Now), + githubBudgetCap: func(string) int { return config.DefaultAgentsGitHubAPIHourlyCap }, + githubReserveFloor: func() int { return config.DefaultGitHubAgentReserveFloor }, } // Pre-warm cert cache for known GitHub hosts to avoid startup burst @@ -532,6 +557,18 @@ func (p *GitHubProxy) SetRepoPausedFunc(fn func(repo string) bool) { p.pausedMu.Unlock() } +// repoScopePath is the request path handed to the per-repo write gates (pause, +// agent scope, auto-merge, per-repo ACMM mode, repo filter). It starts from +// the client's own escaping rather than the decoded req.URL.Path so a GitLab +// "namespace%2Fproject" stays one segment and ExtractRepo resolves the exact +// project — including nested groups — instead of guessing where the project +// path ends from the sub-resource that follows it. CanonicalGatePath then +// decodes every OTHER escape so an escaped structural segment (/r%65pos/, +// repo%2Egit, pro%6Aects, merg%65) cannot hide the repo from the gates. +func repoScopePath(req *http.Request) string { + return CanonicalGatePath(req.URL.EscapedPath()) +} + // repoPauseRefusal is RepoPauseRefusal against this proxy's current predicate. func (p *GitHubProxy) repoPauseRefusal(method, path string) (string, bool) { p.pausedMu.RLock() @@ -1140,26 +1177,86 @@ func (p *GitHubProxy) fallbackAgentName(r *http.Request) string { // identifyAgentByUID reads /proc/net/tcp to find the UID of the process // that owns the socket connected to the proxy, then looks up the agent name. func (p *GitHubProxy) identifyAgentByUID(remoteAddr string) string { - _, portStr, err := net.SplitHostPort(remoteAddr) + uid, err := uidForRemoteAddr(remoteAddr) if err != nil { return "" } + return p.uidMap.LookupByUID(uid) +} + +// uidForRemoteAddr resolves the UID of the process that owns the local socket +// behind remoteAddr ("host:port" as http.Server and net.Conn report it) via +// /proc/net/tcp. It is the shared first half of every UID-attribution path. +func uidForRemoteAddr(remoteAddr string) (int, error) { + _, portStr, err := net.SplitHostPort(remoteAddr) + if err != nil { + return -1, err + } port := 0 const maxPort = 65535 for _, c := range portStr { if c < '0' || c > '9' { - return "" + return -1, fmt.Errorf("remote port %q is not numeric", portStr) } port = port*10 + int(c-'0') if port > maxPort { - return "" + return -1, fmt.Errorf("remote port %q out of range", portStr) } } - uid, err := LookupUIDByLocalPort(port) - if err != nil { - return "" + return LookupUIDByLocalPort(port) +} + +// inferenceAPIKeyPrefix is what the manager prepends to the agent's name to +// form the ANTHROPIC_API_KEY it hands each inference-routed agent +// (agent/manager_env.go). The suffix is the agent's NAME, not a secret. +const inferenceAPIKeyPrefix = "sk-hive-" + +// translatorAgentName names the caller of the inference translator. +// +// The x-api-key header the Claude CLI sends is "sk-hive-", and the +// agent name is public inside the pod: every agent knows every other agent's +// name. Trusting the header alone lets any process on the loopback borrow +// another agent's inference route — its backend, model, upstream credential +// and usage attribution — which is exactly the self-asserted-identity hole +// closed for the GitHub path in fallbackAgentName and for the decision +// endpoint in IdentifyAgentByUID. So the socket owner's UID decides: +// +// - a UID mapped to an agent is authoritative; a header naming someone else +// is logged and overridden; +// - the root control plane keeps the header name — the hive is the party +// that issues these keys; the proxy UID cannot assert a name because its +// upstream sockets can relay agent-controlled requests; +// - an unmapped UID, or no UID map at all, accepts the header only under +// HIVE_PROXY_ADVISORY_OK, the same operator escape hatch the GitHub path +// honours for deployments without per-agent UID separation. +// +// The second return is false when the caller could not be identified and the +// request must be refused. +func (p *GitHubProxy) translatorAgentName(r *http.Request) (string, bool) { + claimed := strings.TrimPrefix(r.Header.Get("x-api-key"), inferenceAPIKeyPrefix) + if p.uidMap == nil { + if p.proxyAdvisoryOK { + return claimed, true + } + return "", false } - return p.uidMap.LookupByUID(uid) + uid, err := uidForRemoteAddr(r.RemoteAddr) + if err == nil { + if name := p.uidMap.LookupByUID(uid); name != "" { + if claimed != "" && claimed != name { + p.logger.Warn("inference identity: x-api-key names another agent; using the socket owner", + "agent", name, "claimed", claimed, "uid", uid) + } + return name, true + } + if uid == 0 && p.uidMap.IsInternalUID(uid) { + return claimed, true + } + } + if p.proxyAdvisoryOK { + return claimed, true + } + return "", false } // handleConnectDirect handles CONNECT requests on a raw connection (no http.Server). @@ -1357,15 +1454,15 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str // scope so their more specific reasons win when both apply. writeSurfaceKind := "" if agentName != internalCallerName { - if reason, paused := p.repoPauseRefusal(req.Method, req.URL.Path); paused { + if reason, paused := p.repoPauseRefusal(req.Method, repoScopePath(req)); paused { blocked = true blockReason = reason blockCode = DenyReasonRepoPaused - } else if reason, refused := p.agentRepoScopeRefusal(agentName, req.Method, req.URL.Path); refused { + } else if reason, refused := p.agentRepoScopeRefusal(agentName, req.Method, repoScopePath(req)); refused { blocked = true blockReason = reason blockCode = DenyReasonAgentRepoScope - } else if reason, refused := p.autoMergeRefusal(req.Method, req.URL.Path); refused { + } else if reason, refused := p.autoMergeRefusal(req.Method, repoScopePath(req)); refused { blocked = true blockReason = reason blockCode = DenyReasonRepoAutoMergeOff @@ -1444,7 +1541,7 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str // autonomy. Its control-plane calls (App token mint, heartbeat) // must not be gated as if they were agent writes. Repo-filter and // canary-egress checks below still apply. - } else if !AllowedByModeCaps(autonomyModeForRepo(agentName, ExtractRepo(req.URL.Path), mode), caps, req.Method, req.URL.Path) { + } else if !AllowedByModeCaps(autonomyModeForRepo(agentName, ExtractRepo(repoScopePath(req)), mode), caps, req.Method, req.URL.Path) { blocked = true blockCode = DenyReasonMode // An unidentified agent is silently treated as ADVISORY, which turns @@ -1462,7 +1559,7 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str blockReason = msg blockCode = DenyReasonHardDeny } - } else if len(p.allowedRepos) > 0 && !RepoFilterAllowed(p.allowedRepos, req.Method, req.URL.Path) { + } else if len(p.allowedRepos) > 0 && !RepoFilterAllowed(p.allowedRepos, req.Method, repoScopePath(req)) { blocked = true blockReason = "repo not in hive config" blockCode = DenyReasonRepoNotInConfig @@ -1524,7 +1621,7 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str } p.recordViolation(agentName, req.Method, detail) if writeSurfaceKind != "" { - p.auditWriteRefused(agentName, writeSurfaceKind, req.Method, req.URL.Path) + p.auditWriteRefused(agentName, writeSurfaceKind, req.Method, repoScopePath(req)) } if blockCode == "" { @@ -1649,11 +1746,7 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str p.logTimeout("proxy upstream response read timed out", err, "agent", agentName, "path", req.URL.Path) return } - if p.injectGHAuth && agentName != internalCallerName { - hgithub.RecordInjectedRESTRequest("agent:"+agentName, req.Method, req.URL.Path, resp.StatusCode, resp.Header) - } else { - hgithub.RecordRESTRequest("agent:"+agentName, req.Method, req.URL.Path, resp.StatusCode, resp.Header) - } + p.recordGitHubREST(agentName, req, resp) p.recordAgentGitHubBudget(agentName, req.URL.Path, resp.StatusCode, p.agentGitHubCap(agentName)) // Bound the BODY relay too, not just the header read above. The @@ -2131,7 +2224,37 @@ func sameLabels(a, b []string) bool { const internalCallerName = "hive-internal" func (p *GitHubProxy) recordViolation(agentName, method, path string) { - p.logger.Warn("proxy request blocked", "agent", agentName, "method", method, "path", path) + if p.firstBlock(agentName, method, path) { + p.logger.Warn("proxy request blocked", "agent", agentName, "method", method, "path", path) + } else { + p.logger.Debug("proxy request blocked (repeat)", "agent", agentName, "method", method, "path", path) + } + p.countViolation(agentName) +} + +// firstBlock reports whether this (agent, method, path) refusal has not been +// logged before. Once the set is full, new keys keep logging at WARN. +func (p *GitHubProxy) firstBlock(agentName, method, path string) bool { + key := agentName + "|" + method + "|" + path + p.mu.Lock() + defer p.mu.Unlock() + if _, seen := p.blockedSeen[key]; seen { + return false + } + if p.blockedSeen == nil { + p.blockedSeen = make(map[string]struct{}) + } + if len(p.blockedSeen) < maxViolationLog { + p.blockedSeen[key] = struct{}{} + } + return true +} + +// countViolation increments the per-agent refusal counter without logging, for +// callers that have already logged the refusal with their own detail (see +// refusePlainForward). The map is bounded by maxViolationLog so an +// unidentified or churning caller set cannot grow it without limit. +func (p *GitHubProxy) countViolation(agentName string) { p.mu.Lock() defer p.mu.Unlock() if _, exists := p.violations[agentName]; exists || len(p.violations) < maxViolationLog { @@ -2334,16 +2457,43 @@ func transfer(dst, src net.Conn) { _, _ = io.Copy(dst, src) } -// forwardPlainDirect handles non-CONNECT (plain HTTP) requests on a raw connection. -var plainHTTPClient = &http.Client{ - Transport: http.DefaultTransport, - Timeout: httpWriteTimeout, -} - +// forwardPlainDirect handles non-CONNECT (plain HTTP) requests on a raw +// connection. +// +// Two policies apply here that the CONNECT path has always applied and this +// one did not: the caller is identified from the connection, and the +// destination must be globally routable (see plain_forward_guard.go). Before +// that, any process that could reach the proxy port could have it open a +// connection, from the proxy's own uid, to any address it named — including +// the hive's loopback services, the cloud metadata endpoint and the +// cluster-internal estate — and nothing in the log said who asked. func (p *GitHubProxy) forwardPlainDirect(conn net.Conn, r *http.Request) { - resp, err := plainHTTPClient.Transport.RoundTrip(r) + // Identify from the connection, NOT from r.RemoteAddr: this request came + // from http.ReadRequest, which leaves RemoteAddr empty (see + // identifyAgentFromConn). "" means the caller could not be attributed, + // which is logged as such rather than guessed at. + agentName := p.identifyAgentFromConn(conn, r) + + // Proxy-* hop headers address THIS proxy and must not travel further: the + // identity they assert is the one identifyAgentFromConn just decided + // whether to believe, and forwarding it would hand an agent's claimed name + // (and, in the Basic form, whatever it put in the password field) to every + // plain-HTTP host it can name. + r.Header.Del("Proxy-Authorization") + r.Header.Del("Proxy-Connection") + + resp, err := p.plainForwardTransport().RoundTrip(r) if err != nil { - p.logger.Warn("proxy: plain HTTP forward failed", "agent", p.identifyAgentFromConn(conn, r), "url", r.URL.String(), "error", err) + var denied *plainForwardDenied + if errors.As(err, &denied) { + p.refusePlainForward(conn, r, agentName, denied) + return + } + if isLoopbackHost(r.URL.Host) { + p.logger.Debug("proxy: plain HTTP forward to loopback failed", "agent", agentName, "url", r.URL.String(), "error", err, "hint", "client should bypass the proxy for loopback; NO_PROXY") + } else { + p.logger.Warn("proxy: plain HTTP forward failed", "agent", agentName, "url", r.URL.String(), "error", err) + } _, _ = fmt.Fprintf(conn, "HTTP/1.1 502 Bad Gateway\r\n\r\nupstream request failed\n") return } @@ -2351,6 +2501,43 @@ func (p *GitHubProxy) forwardPlainDirect(conn net.Conn, r *http.Request) { _ = resp.Write(conn) } +// refusePlainForward answers a destination the guard barred. +// +// Loopback is treated as the misconfiguration it almost always is rather than +// as an attack: agents are handed NO_PROXY=127.0.0.1,localhost,::1 (#14076), +// so a loopback request arriving here means a client ignored it. It is logged +// at Debug with the same hint as the failure path and does NOT count as a +// violation. Every other class — link-local (cloud metadata), private, CGNAT, +// multicast — is an attempt to reach the estate behind the proxy and is +// counted against the caller. +func (p *GitHubProxy) refusePlainForward(conn net.Conn, r *http.Request, agentName string, denied *plainForwardDenied) { + if denied.reason == plainForwardReasonLoopback { + p.logger.Debug("proxy: plain HTTP forward to loopback refused (clients must bypass the proxy for loopback; NO_PROXY)", + "agent", agentName, "url", r.URL.String(), "target", denied.target) + } else { + p.logger.Warn("proxy: plain HTTP forward refused — destination is not globally routable", + "agent", agentName, "method", r.Method, "url", r.URL.String(), "target", denied.target, "class", denied.reason) + p.countViolation(agentName) + } + _, _ = fmt.Fprintf(conn, "HTTP/1.1 403 Forbidden\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+ + "the hive proxy does not forward plain HTTP to %s addresses (%s); set %s=true to allow it\n", + denied.reason, denied.target, plainForwardAllowPrivateEnv) +} + +// isLoopbackHost reports whether a host or host:port refers to the loopback interface. +func isLoopbackHost(hostport string) bool { + host := hostport + if h, _, err := net.SplitHostPort(hostport); err == nil { + host = h + } + host = strings.Trim(host, "[]") + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + // extractAgentName reads the agent name from the Proxy-Authorization header. // Supports "hive " (custom) and "Basic " (standard HTTP proxy auth // sent automatically when the proxy URL contains userinfo, e.g. http://quality@host:port). @@ -2886,8 +3073,13 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - apiKey := r.Header.Get("x-api-key") - agentName := strings.TrimPrefix(apiKey, "sk-hive-") + agentName, ok := p.translatorAgentName(r) + if !ok { + p.logger.Warn("inference request from unidentified caller refused", + "remote", r.RemoteAddr, "method", r.Method, "path", r.URL.Path) + http.Error(w, `{"type":"error","error":{"type":"authentication_error","message":"caller could not be identified"}}`, http.StatusUnauthorized) + return + } p.logger.Info("inference request", "agent", agentName, @@ -3029,7 +3221,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { p.logger.Error("inference SSE translation failed", "agent", agentName, "error", err) } if in > 0 || out > 0 { - p.tokenSink.Record(agentName, route.Model, in, out) + p.recordUsage(agentName, route.Model, in, out) } return } @@ -3047,7 +3239,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { ) if in, out := extractOpenAIUsage(respBody); in > 0 || out > 0 { - p.tokenSink.Record(agentName, route.Model, in, out) + p.recordUsage(agentName, route.Model, in, out) } translated, err := translateOpenAIResponseToAnthropic(respBody, route.Model) @@ -3076,7 +3268,11 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { func (p *GitHubProxy) StartInferenceTranslator() error { addr := fmt.Sprintf("127.0.0.1:%d", InferenceTranslatePort) p.logger.Info("inference translation server starting", "addr", addr) - server := &http.Server{Addr: addr, Handler: p.inferenceTranslatorHandler()} + server := &http.Server{ + Addr: addr, + Handler: p.inferenceTranslatorHandler(), + ReadHeaderTimeout: httpReadTimeout, + } return server.ListenAndServe() } @@ -3226,7 +3422,7 @@ func (p *GitHubProxy) handleInferenceRequest(conn net.Conn, req *http.Request, a p.logger.Error("inference SSE translation failed", "agent", agentName, "error", err) } if in > 0 || out > 0 { - p.tokenSink.Record(agentName, route.Model, in, out) + p.recordUsage(agentName, route.Model, in, out) } return } @@ -3238,7 +3434,7 @@ func (p *GitHubProxy) handleInferenceRequest(conn net.Conn, req *http.Request, a } if in, out := extractOpenAIUsage(respBody); in > 0 || out > 0 { - p.tokenSink.Record(agentName, route.Model, in, out) + p.recordUsage(agentName, route.Model, in, out) } translated, err := translateOpenAIResponseToAnthropic(respBody, route.Model) @@ -3438,7 +3634,7 @@ func (p *GitHubProxy) forwardCopilotAnthropicResponseWithUsage(client net.Conn, resolvedModel = u.Model } if resolvedModel != "" && resolvedModel != copilotAutoModelSentinel { - p.tokenSink.RecordCopilotWithCache(agentName, resolvedModel, u.Input, u.Output, u.CacheRead, u.CacheCreation) + p.recordCopilotUsageWithCache(agentName, resolvedModel, u.Input, u.Output, u.CacheRead, u.CacheCreation) p.logger.Info("copilot usage recorded", "agent", agentName, "host", host, @@ -3477,7 +3673,7 @@ func (p *GitHubProxy) forwardCopilotResponseWithUsage(client net.Conn, resp *htt } } if resolvedModel != "" && resolvedModel != copilotAutoModelSentinel { - p.tokenSink.RecordCopilot(agentName, resolvedModel, in, out) + p.recordCopilotUsage(agentName, resolvedModel, in, out) p.logger.Info("copilot usage recorded", "agent", agentName, "host", host, diff --git a/src/pkg/proxy/github_proxy_coverage_test.go b/src/crux/proxy/github_proxy_coverage_test.go similarity index 100% rename from src/pkg/proxy/github_proxy_coverage_test.go rename to src/crux/proxy/github_proxy_coverage_test.go diff --git a/src/pkg/proxy/github_proxy_util_test.go b/src/crux/proxy/github_proxy_util_test.go similarity index 77% rename from src/pkg/proxy/github_proxy_util_test.go rename to src/crux/proxy/github_proxy_util_test.go index 386b1996ab..ae84db4816 100644 --- a/src/pkg/proxy/github_proxy_util_test.go +++ b/src/crux/proxy/github_proxy_util_test.go @@ -7,11 +7,12 @@ import ( "log/slog" "net" "net/http" + "strings" "sync" "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // ---------- isGitPath ---------- @@ -118,6 +119,26 @@ func TestRecordViolation_Basic(t *testing.T) { } } +func TestRecordViolation_RepeatLogsWarnOnce(t *testing.T) { + var buf bytes.Buffer + p := &GitHubProxy{ + logger: slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})), + violations: make(map[string]int), + } + + for i := 0; i < 5; i++ { + p.recordViolation("supervisor", "POST", "/copilot_internal/subscribe_limited_user") + } + p.recordViolation("supervisor", "GET", "/copilot_internal/subscribe_limited_user") + + if got := strings.Count(buf.String(), "proxy request blocked"); got != 2 { + t.Errorf("WARN lines = %d, want 2 (one per distinct method/path)", got) + } + if got := p.AgentViolations("supervisor"); got != 6 { + t.Errorf("violations = %d, want 6 (every refusal still counted)", got) + } +} + func TestRecordViolation_MaxCap(t *testing.T) { p := &GitHubProxy{ logger: slog.New(slog.NewTextHandler(io.Discard, nil)), @@ -272,3 +293,35 @@ func (c *testBufConn) RemoteAddr() net.Addr { return &net.TCPAddr{ func (c *testBufConn) SetDeadline(_ time.Time) error { return nil } func (c *testBufConn) SetReadDeadline(_ time.Time) error { return nil } func (c *testBufConn) SetWriteDeadline(_ time.Time) error { return nil } + +// TestRepoScopePathCanonicalizesEscapes pins the proxy-level contract: the +// path handed to the per-repo gates keeps %2F (GitLab project separator) but +// decodes every other escape, so an escaped structural segment resolves to +// the same repo the forge will act on. +func TestRepoScopePathCanonicalizesEscapes(t *testing.T) { + tests := []struct { + rawURL string + wantPath string + wantRepo string + }{ + {"https://api.github.com/r%65pos/org/repo/issues", "/repos/org/repo/issues", "org/repo"}, + {"https://github.com/org/repo%2Egit/git-receive-pack", "/org/repo.git/git-receive-pack", "org/repo"}, + {"https://gitlab.com/api/v4/pro%6Aects/org%2Frepo/issues", "/api/v4/projects/org%2Frepo/issues", "org/repo"}, + {"https://gitlab.com/api/v4/projects/group%2Fsub%2Frepo/issues", "/api/v4/projects/group%2Fsub%2Frepo/issues", "group/sub/repo"}, + {"https://api.github.com/repos/org/repo/pulls/1/merg%65", "/repos/org/repo/pulls/1/merge", "org/repo"}, + } + for _, tt := range tests { + t.Run(tt.rawURL, func(t *testing.T) { + req, err := http.NewRequest(http.MethodPost, tt.rawURL, nil) + if err != nil { + t.Fatal(err) + } + if got := repoScopePath(req); got != tt.wantPath { + t.Errorf("repoScopePath = %q, want %q", got, tt.wantPath) + } + if got := ExtractRepo(repoScopePath(req)); got != tt.wantRepo { + t.Errorf("ExtractRepo(repoScopePath) = %q, want %q", got, tt.wantRepo) + } + }) + } +} diff --git a/src/crux/proxy/graphql_repo_policy_test.go b/src/crux/proxy/graphql_repo_policy_test.go new file mode 100644 index 0000000000..76bb11fdf9 --- /dev/null +++ b/src/crux/proxy/graphql_repo_policy_test.go @@ -0,0 +1,199 @@ +package proxy + +import ( + "strings" + "testing" + + "github.com/hivecommons/hive/crux/agent" +) + +// autoMergeOffFor is a repo-policy predicate that reports auto-merge disabled +// for exactly the given repos (case-insensitive) and enabled for every other. +func autoMergeOffFor(repos ...string) func(string) bool { + off := make(map[string]bool, len(repos)) + for _, r := range repos { + off[strings.ToLower(r)] = true + } + return func(repo string) bool { return !off[strings.ToLower(repo)] } +} + +const mergeMutationQuery = `mutation($input:MergePullRequestInput!){ mergePullRequest(input:$input){clientMutationId} }` + +// The refusal must never turn into a blanket merge denial: an attributable +// merge on a repo whose auto-merge is ON, an unattributable merge, a read +// query and a non-merge mutation all pass through to the ACMM merge gate. +func TestGraphQLAutoMergeRefusal_AllowsWhenPolicyPermits(t *testing.T) { + enabled := autoMergeOffFor("acme/frozen") + + cases := map[string]string{ + "merge on enabled repo via variables": `{"query":"` + mergeMutationQuery + `","variables":{"repo":"acme/open"}}`, + "merge on enabled repo via owner+name": `{"query":"` + mergeMutationQuery + `","variables":{"input":{"owner":"acme","name":"open"}}}`, + "merge with no repo attribution at all": `{"query":"` + mergeMutationQuery + `","variables":{"input":{"pullRequestId":"PR_kw"}}}`, + "read query naming the frozen repo": `{"query":"query{ repository(owner:\"acme\", name:\"frozen\"){ id } }","variables":{"repo":"acme/frozen"}}`, + "non-merge mutation naming the frozen repo": `{"query":"mutation{ addComment(input:{subjectId:\"I_kw\", body:\"hi\"}){clientMutationId} }","variables":{"repo":"acme/frozen"}}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + if msg, refused := GraphQLAutoMergeRefusal(enabled, []byte(body)); refused { + t.Fatalf("refused (%q); want pass-through to the ACMM merge gate", msg) + } + }) + } +} + +// With no repo-policy predicate configured (every pre-policy hive) or a body +// the proxy cannot parse, the refusal declines to judge rather than deny: +// the ACMM merge gate downstream still applies. +func TestGraphQLAutoMergeRefusal_NoPredicateOrUnparseableBodyPassesThrough(t *testing.T) { + body := []byte(`{"query":"` + mergeMutationQuery + `","variables":{"repo":"acme/frozen"}}`) + if msg, refused := GraphQLAutoMergeRefusal(nil, body); refused { + t.Fatalf("nil predicate refused (%q); want pass-through", msg) + } + enabled := autoMergeOffFor("acme/frozen") + if msg, refused := GraphQLAutoMergeRefusal(enabled, []byte(`{"query": not json`)); refused { + t.Fatalf("unparseable body refused (%q); want pass-through", msg) + } + if msg, refused := GraphQLAutoMergeRefusal(enabled, nil); refused { + t.Fatalf("empty body refused (%q); want pass-through", msg) + } +} + +// When the variables do not attribute the merge to a repo, the body-literal +// fallback still can: a repo spelled anywhere in the request (here under +// extensions) denies a merge on a frozen repo. +func TestGraphQLAutoMergeRefusal_BodyLiteralFallback(t *testing.T) { + enabled := autoMergeOffFor("acme/frozen") + + body := []byte(`{"query":"` + mergeMutationQuery + `","variables":{"input":{"pullRequestId":"PR_kw"}},"extensions":{"repository":"acme/frozen"}}`) + msg, refused := GraphQLAutoMergeRefusal(enabled, body) + if !refused || !strings.Contains(msg, "acme/frozen") { + t.Fatalf("literal fallback = (%q,%v), want refusal naming acme/frozen", msg, refused) + } + + body = []byte(`{"query":"` + mergeMutationQuery + `","variables":{"input":{"pullRequestId":"PR_kw"}},"extensions":{"owner":"acme","repo":"frozen"}}`) + msg, refused = GraphQLAutoMergeRefusal(enabled, body) + if !refused || !strings.Contains(msg, "acme/frozen") { + t.Fatalf("owner+repo literal fallback = (%q,%v), want refusal naming acme/frozen", msg, refused) + } + + // The fallback attributes to the OPEN repo: not refused. + body = []byte(`{"query":"` + mergeMutationQuery + `","variables":{"input":{"pullRequestId":"PR_kw"}},"extensions":{"repository":"acme/open"}}`) + if msg, refused := GraphQLAutoMergeRefusal(enabled, body); refused { + t.Fatalf("literal fallback naming an enabled repo refused (%q)", msg) + } +} + +func TestExtractRepoFromGraphQLBody(t *testing.T) { + cases := []struct { + name, body, want string + }{ + {"repo literal", `{"repo":"acme/widgets"}`, "acme/widgets"}, + {"repository literal", `{"repository": "acme/widgets"}`, "acme/widgets"}, + {"repositoryNameWithOwner literal", `{"repositoryNameWithOwner":"acme/widgets"}`, "acme/widgets"}, + {"owner then name", `{"owner":"acme","name":"widgets"}`, "acme/widgets"}, + {"owner then repo with spaces", `{"owner" : "acme" , "repo" : "widgets"}`, "acme/widgets"}, + {"owner then repository", `{"owner":"acme","repository":"widgets"}`, "acme/widgets"}, + {"case-insensitive key", `{"Repository":"acme/widgets"}`, "acme/widgets"}, + {"literal without a slash is not a repo", `{"repo":"widgets"}`, ""}, + {"owner without a name", `{"owner":"acme"}`, ""}, + {"name before owner is not matched by the literal regex", `{"name":"widgets","owner":"acme"}`, ""}, + {"empty body", ``, ""}, + {"unrelated body", `{"query":"query{ viewer { login } }"}`, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := extractRepoFromGraphQLBody([]byte(tc.body)); got != tc.want { + t.Fatalf("extractRepoFromGraphQLBody(%s) = %q, want %q", tc.body, got, tc.want) + } + }) + } +} + +// setAutonomyRepoLevelForTest registers a per-repo autonomy override and +// removes it when the test ends. autonomyRepoLevels is package-global, so a +// test that leaves its key behind changes what every later test sees. +func setAutonomyRepoLevelForTest(t *testing.T, repo string, level int) { + t.Helper() + SetAutonomyRepoLevel(repo, level) + t.Cleanup(func() { autonomyRepoLevels.Delete(strings.ToLower(repo)) }) +} + +// A GraphQL document that names a demoted repo inline +// (repository(owner:"…", name:"…")) resolves to that repo's demoted mode, +// not the hive-wide one. This is the branch the proxy relies on to apply +// project.repo_policies autonomy levels to GraphQL traffic. +func TestAutonomyGraphQLMode_InlineRepositoryRefResolvesDemotedRepo(t *testing.T) { + setAutonomyRepoLevelForTest(t, "acme/demoted-graphql", 4) + hiveMode := agent.DefaultAgentMode("scanner", 5) + demoted := agent.DefaultAgentMode("scanner", 4) + if demoted == hiveMode { + t.Fatal("fixture: scanner L4 and L5 must map to different modes") + } + + body := []byte(`{"query":"mutation { repository(owner: \"acme\", name: \"demoted-graphql\") { id } createPullRequest(input:{repositoryId:\"R\"}){ pullRequest { id } } }"}`) + if got := autonomyGraphQLMode("scanner", body, hiveMode); got != demoted { + t.Fatalf("mode = %v, want demoted %v", got, demoted) + } + + // Case and whitespace in the inline ref do not defeat attribution. + body = []byte(`{"query":"query {\n Repository (\n owner : \"ACME\" ,\n name : \"Demoted-GraphQL\" ) { id }\n}"}`) + if got := autonomyGraphQLMode("scanner", body, hiveMode); got != demoted { + t.Fatalf("mode with spaced/cased ref = %v, want demoted %v", got, demoted) + } +} + +// An inline ref to a repo WITHOUT an override keeps the hive-wide mode for +// read queries, while a PR write that cannot be attributed to an overridden +// repo fails closed to advisory while any override is active. +func TestAutonomyGraphQLMode_UnmatchedRepoKeepsFallbackForReadsOnly(t *testing.T) { + setAutonomyRepoLevelForTest(t, "acme/demoted-graphql", 4) + hiveMode := agent.DefaultAgentMode("scanner", 5) + + read := []byte(`{"query":"query { repository(owner: \"acme\", name: \"other\") { id } }"}`) + if got := autonomyGraphQLMode("scanner", read, hiveMode); got != hiveMode { + t.Fatalf("read on non-overridden repo = %v, want hive-wide %v", got, hiveMode) + } + write := []byte(`{"query":"mutation { repository(owner: \"acme\", name: \"other\") { id } createPullRequest(input:{repositoryId:\"R\"}){ pullRequest { id } } }"}`) + if got := autonomyGraphQLMode("scanner", write, hiveMode); got != agent.ModeAdvisory { + t.Fatalf("unattributed PR write = %v, want fail-closed %v", got, agent.ModeAdvisory) + } + merge := []byte(`{"query":"mutation { mergePullRequest(input:{pullRequestId:\"PR_kw\"}){ clientMutationId } }"}`) + if got := autonomyGraphQLMode("scanner", merge, hiveMode); got != agent.ModeAdvisory { + t.Fatalf("unattributed merge = %v, want fail-closed %v", got, agent.ModeAdvisory) + } +} + +// A body the proxy cannot parse carries no attribution and no mutation +// classification: the hive-wide mode stands (the generic GraphQL gate +// downstream still sees the raw body). +func TestAutonomyGraphQLMode_UnparseableBodyKeepsFallback(t *testing.T) { + setAutonomyRepoLevelForTest(t, "acme/demoted-graphql", 4) + hiveMode := agent.DefaultAgentMode("scanner", 5) + if got := autonomyGraphQLMode("scanner", []byte(`{"query": not json`), hiveMode); got != hiveMode { + t.Fatalf("unparseable body mode = %v, want fallback %v", got, hiveMode) + } +} + +// With no overrides registered, autonomyGraphQLMode is a no-op. The package +// map is global, so snapshot and clear it for the duration of this test. +func TestAutonomyGraphQLMode_NoOverridesIsNoop(t *testing.T) { + saved := map[any]any{} + autonomyRepoLevels.Range(func(k, v any) bool { + saved[k] = v + autonomyRepoLevels.Delete(k) + return true + }) + t.Cleanup(func() { + for k, v := range saved { + autonomyRepoLevels.Store(k, v) + } + }) + if hasAutonomyRepoLevels() { + t.Fatal("fixture: overrides should be cleared") + } + hiveMode := agent.DefaultAgentMode("scanner", 5) + body := []byte(`{"query":"mutation { mergePullRequest(input:{pullRequestId:\"PR_kw\"}){ clientMutationId } }"}`) + if got := autonomyGraphQLMode("scanner", body, hiveMode); got != hiveMode { + t.Fatalf("mode with no overrides = %v, want fallback %v", got, hiveMode) + } +} diff --git a/src/crux/proxy/hive_seams_sidecar_test.go b/src/crux/proxy/hive_seams_sidecar_test.go new file mode 100644 index 0000000000..fddfbf2014 --- /dev/null +++ b/src/crux/proxy/hive_seams_sidecar_test.go @@ -0,0 +1,170 @@ +package proxy + +import ( + "context" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api/callback" + "github.com/hivecommons/hive/crux/hivecap" +) + +// sidecarCaps is the hive end of the callback for the proxy seams: a canary +// registry with one planted token, a usage ledger and a REST ledger. +type sidecarCaps struct { + mu sync.Mutex + canary string + usage []callback.UsageRecord + rest []callback.RESTRecord +} + +func (c *sidecarCaps) MintToken(context.Context, callback.MintTokenRequest) (callback.MintTokenResponse, error) { + return callback.MintTokenResponse{}, callback.ErrNotConfigured +} + +func (c *sidecarCaps) RecordREST(_ context.Context, rec callback.RESTRecord) error { + c.mu.Lock() + defer c.mu.Unlock() + c.rest = append(c.rest, rec) + return nil +} + +func (c *sidecarCaps) RateLimit(context.Context) (callback.RateLimitSnapshot, error) { + return callback.RateLimitSnapshot{Remaining: 4321, Reset: time.Now().Add(time.Hour).Truncate(time.Second), OK: true}, nil +} + +func (c *sidecarCaps) RecordUsage(_ context.Context, rec callback.UsageRecord) error { + c.mu.Lock() + defer c.mu.Unlock() + c.usage = append(c.usage, rec) + return nil +} + +func (c *sidecarCaps) ScanCanary(_ context.Context, req callback.CanaryScanRequest) (callback.CanaryScanResponse, error) { + if c.canary != "" && strings.Contains(req.Text, c.canary) { + return callback.CanaryScanResponse{Detected: true, Agent: req.Agent, Source: req.Source}, nil + } + return callback.CanaryScanResponse{}, nil +} + +func (c *sidecarCaps) ScanText(context.Context, callback.TextScanRequest) (callback.TextScanResponse, error) { + return callback.TextScanResponse{}, nil +} + +func (c *sidecarCaps) ReportCondition(context.Context, callback.ConditionReport) error { return nil } + +func (c *sidecarCaps) LinearCredential(context.Context) (callback.LinearCredential, error) { + return callback.LinearCredential{}, callback.ErrNotConfigured +} + +func (c *sidecarCaps) CreatePR(context.Context, callback.CreatePRRequest) (callback.CreatePRResponse, error) { + return callback.CreatePRResponse{}, callback.ErrNotConfigured +} + +// serveCallback starts hive's end on sock and returns its stop function. +func serveCallback(t *testing.T, sock string, caps callback.Capabilities) func() { + t.Helper() + ls, err := callback.Listen(callback.ListenConfig{Socket: sock}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- callback.NewServer(caps, "").Serve(ctx, ls) }() + return func() { + cancel() + if err := <-done; err != nil { + t.Errorf("Serve: %v", err) + } + } +} + +// A proxy whose Accounting / Usage / Canary seams are the hive callback +// adapters (the sidecar arrangement, hivecommons/hive#14138 stage 4) +// refuses GitHub writes while hive is unreachable, reports no shared +// rate-limit snapshot and records no usage; when hive comes back the same +// proxy, with nothing re-installed, lets a clean write through, blocks a +// canary leak, and its usage and REST records reach hive. +func TestSidecarProxySeamsRefuseWhileHiveIsAwayAndResume(t *testing.T) { + dir, err := os.MkdirTemp("", "sidecar-proxy") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + if err := os.Chmod(dir, 0o700); err != nil { + t.Fatal(err) + } + sock := filepath.Join(dir, callback.SocketName) + client, err := callback.NewClient("unix://"+sock, "") + if err != nil { + t.Fatal(err) + } + set := hivecap.New(client, 300*time.Millisecond) + + var leaks []crux.CanaryLeak + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(os.Stderr, nil))} + p.SetGitHubRESTAccounting(set.Accounting) + p.SetTokenSink(set.Usage) + p.SetCanaryScanner(true, true, set.Canary, func(l crux.CanaryLeak) { leaks = append(leaks, l) }) + + write := func(body string) *http.Request { + req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/issues/1/comments", strings.NewReader(body)) + return req + } + + // Hive away: the write is refused, the refusal says why, nothing is + // recorded anywhere, and the budget sees no shared snapshot. + reason, deny, detected := p.inspectCanaryEgress("worker", write(`{"body":"hello"}`)) + if !deny || !detected || reason == "" { + t.Fatalf("hive away: deny=%v detected=%v reason=%q, want refused", deny, detected, reason) + } + if len(leaks) != 1 || !strings.HasSuffix(leaks[0].Source, hivecap.UnavailableSourceSuffix) || leaks[0].Token != "" { + t.Fatalf("hive away: leak record = %+v, want unavailable-suffixed source and no token", leaks) + } + if _, _, ok := p.restAccounting.CoreRateLimitSnapshot(); ok { + t.Fatal("hive away: snapshot must be ok=false") + } + p.tokenSink.Record("worker", "m", 1, 2) + p.restAccounting.RecordRESTRequest("worker", http.MethodGet, "/x", http.StatusOK, nil) + + // Hive back, same proxy: a clean write passes, the planted canary is + // blocked as a real detection, and the ledgers fill. + caps := &sidecarCaps{canary: "CANARY-worker-7"} + stop := serveCallback(t, sock, caps) + defer stop() + if _, err := client.Health(context.Background()); err != nil { + t.Fatalf("callback never ready: %v", err) + } + leaks = nil + if reason, deny, detected := p.inspectCanaryEgress("worker", write(`{"body":"hello"}`)); deny || detected || reason != "" { + t.Fatalf("hive back: clean write deny=%v detected=%v reason=%q", deny, detected, reason) + } + if reason, deny, detected := p.inspectCanaryEgress("worker", write(`{"body":"CANARY-worker-7"}`)); !deny || !detected || reason == "" { + t.Fatalf("hive back: leak deny=%v detected=%v reason=%q", deny, detected, reason) + } + if len(leaks) != 1 || leaks[0].Agent != "worker" || strings.HasSuffix(leaks[0].Source, hivecap.UnavailableSourceSuffix) { + t.Fatalf("hive back: leak record = %+v", leaks) + } + remaining, reset, ok := p.restAccounting.CoreRateLimitSnapshot() + if !ok || remaining != 4321 || reset.IsZero() { + t.Fatalf("hive back: snapshot = %d %v %v", remaining, reset, ok) + } + p.tokenSink.Record("worker", "m", 10, 20) + p.tokenSink.RecordCopilot("worker", "m", 1, 1) + p.restAccounting.RecordInjectedRESTRequest("worker", http.MethodGet, "/y", http.StatusOK, http.Header{"X-Ratelimit-Remaining": {"9"}}) + caps.mu.Lock() + defer caps.mu.Unlock() + if len(caps.usage) != 2 || caps.usage[0].InputTokens != 10 || caps.usage[1].Kind != callback.UsageKindCopilot { + t.Fatalf("usage at hive = %+v, want only the records made while reachable", caps.usage) + } + if len(caps.rest) != 1 || !caps.rest[0].Injected || caps.rest[0].Path != "/y" { + t.Fatalf("rest at hive = %+v", caps.rest) + } +} diff --git a/src/crux/proxy/host_seams.go b/src/crux/proxy/host_seams.go new file mode 100644 index 0000000000..d4383c9b37 --- /dev/null +++ b/src/crux/proxy/host_seams.go @@ -0,0 +1,91 @@ +package proxy + +import ( + "net/http" + "time" + + "github.com/hivecommons/hive/crux" +) + +// This file holds the three host services the GitHub proxy consumes without +// importing their Hive implementations (hivecommons/hive#13536, Stage 2 +// prerequisite 3). Each is a narrow interface the composition root (cmd/hive) +// satisfies with the real package: pkg/ioscan's canary registry, pkg/tokens' +// inference sink and pkg/github's REST accounting. A proxy with none of them +// installed still enforces every policy; it only stops observing. + +// CanaryScanner recognises planted canary tokens (and their encodings) in a +// GitHub write body. agentName is the identified caller, text the raw body and +// source a label for the surface. *ioscan.CanaryRegistry implements it; a nil +// *CanaryRegistry is a registry with no canaries and scans nothing. +type CanaryScanner interface { + Scan(agentName, text, source string) (crux.CanaryLeak, bool) +} + +// UsageSink receives per-agent inference token usage the proxy observes on +// upstream responses: Record for the OpenAI-compatible gateway path, +// RecordCopilot and RecordCopilotWithCache for the Copilot sniff path. +// *tokens.InferenceSink implements it; the proxy tolerates a nil sink. +type UsageSink interface { + Record(agent, model string, inputTokens, outputTokens int64) + RecordCopilot(agent, model string, inputTokens, outputTokens int64) + RecordCopilotWithCache(agent, model string, inputTokens, outputTokens, cacheRead, cacheCreation int64) +} + +// GitHubRESTAccounting is the host's view of GitHub REST traffic the proxy +// relays: one record per response, split by whether the daemon's injected +// credential (whose rate-limit headers describe the shared core bucket) or +// the agent's own was used, plus the current shared core-bucket snapshot the +// agent GitHub budget reads for its reserve floor. pkg/github's package-level +// RecordRESTRequest / RecordInjectedRESTRequest / CoreRateLimitSnapshot are +// the implementation cmd/hive installs. +type GitHubRESTAccounting interface { + RecordRESTRequest(caller, method, path string, status int, h http.Header) + RecordInjectedRESTRequest(caller, method, path string, status int, h http.Header) + CoreRateLimitSnapshot() (remaining int, reset time.Time, ok bool) +} + +// SetGitHubRESTAccounting installs the host's REST accounting. Until it is +// called the proxy records nothing and the agent GitHub budget sees no shared +// core-bucket snapshot (it then applies the per-agent cap alone). +func (p *GitHubProxy) SetGitHubRESTAccounting(a GitHubRESTAccounting) { + p.restAccounting = a + if a == nil { + p.githubReserveSnapshot = nil + return + } + p.githubReserveSnapshot = a.CoreRateLimitSnapshot +} + +func (p *GitHubProxy) recordGitHubREST(agentName string, req *http.Request, resp *http.Response) { + if p.restAccounting == nil { + return + } + if p.injectGHAuth && agentName != internalCallerName { + p.restAccounting.RecordInjectedRESTRequest("agent:"+agentName, req.Method, req.URL.Path, resp.StatusCode, resp.Header) + return + } + p.restAccounting.RecordRESTRequest("agent:"+agentName, req.Method, req.URL.Path, resp.StatusCode, resp.Header) +} + +// The three record helpers make a missing sink a no-op, the contract the +// previous concrete *tokens.InferenceSink field gave for free on a nil +// pointer. + +func (p *GitHubProxy) recordUsage(agent, model string, in, out int64) { + if p.tokenSink != nil { + p.tokenSink.Record(agent, model, in, out) + } +} + +func (p *GitHubProxy) recordCopilotUsage(agent, model string, in, out int64) { + if p.tokenSink != nil { + p.tokenSink.RecordCopilot(agent, model, in, out) + } +} + +func (p *GitHubProxy) recordCopilotUsageWithCache(agent, model string, in, out, cacheRead, cacheCreation int64) { + if p.tokenSink != nil { + p.tokenSink.RecordCopilotWithCache(agent, model, in, out, cacheRead, cacheCreation) + } +} diff --git a/src/crux/proxy/host_seams_test.go b/src/crux/proxy/host_seams_test.go new file mode 100644 index 0000000000..3d8208117e --- /dev/null +++ b/src/crux/proxy/host_seams_test.go @@ -0,0 +1,78 @@ +package proxy + +import ( + "log/slog" + "net/http" + "testing" + "time" +) + +type fakeRESTAccounting struct { + plain, injected []string + remaining int +} + +func (f *fakeRESTAccounting) RecordRESTRequest(caller, method, path string, status int, _ http.Header) { + f.plain = append(f.plain, caller+" "+method+" "+path) +} + +func (f *fakeRESTAccounting) RecordInjectedRESTRequest(caller, method, path string, status int, _ http.Header) { + f.injected = append(f.injected, caller+" "+method+" "+path) +} + +func (f *fakeRESTAccounting) CoreRateLimitSnapshot() (int, time.Time, bool) { + return f.remaining, time.Time{}, true +} + +// The proxy splits relayed REST responses by which credential made them and +// reads the shared core-bucket snapshot through the same seam; with nothing +// installed it records nothing and the budget sees no snapshot. +func TestSetGitHubRESTAccountingRoutesRecordsAndSnapshot(t *testing.T) { + p := &GitHubProxy{logger: slog.Default()} + req, _ := http.NewRequest(http.MethodGet, "https://api.github.com/repos/o/r/pulls", nil) + resp := &http.Response{StatusCode: http.StatusOK, Header: http.Header{}} + + p.recordGitHubREST("scanner", req, resp) + if _, _, ok := p.agentGitHubReserveSnapshot(); ok { + t.Fatal("snapshot reported ok with no accounting installed") + } + + acct := &fakeRESTAccounting{remaining: 4321} + p.SetGitHubRESTAccounting(acct) + p.recordGitHubREST("scanner", req, resp) + p.injectGHAuth = true + p.recordGitHubREST("scanner", req, resp) + p.recordGitHubREST(internalCallerName, req, resp) + + if len(acct.plain) != 2 || acct.plain[0] != "agent:scanner GET /repos/o/r/pulls" || acct.plain[1] != "agent:"+internalCallerName+" GET /repos/o/r/pulls" { + t.Fatalf("plain records = %q", acct.plain) + } + if len(acct.injected) != 1 || acct.injected[0] != "agent:scanner GET /repos/o/r/pulls" { + t.Fatalf("injected records = %q", acct.injected) + } + if remaining, _, ok := p.agentGitHubReserveSnapshot(); !ok || remaining != 4321 { + t.Fatalf("snapshot = %d, %v; want 4321 through the seam", remaining, ok) + } + + p.SetGitHubRESTAccounting(nil) + if _, _, ok := p.agentGitHubReserveSnapshot(); ok { + t.Fatal("snapshot still reported ok after the accounting was removed") + } +} + +// A proxy with no usage sink keeps serving: every record helper is a no-op. +func TestUsageRecordHelpersTolerateNilSink(t *testing.T) { + p := &GitHubProxy{logger: slog.Default()} + p.recordUsage("a", "m", 1, 2) + p.recordCopilotUsage("a", "m", 1, 2) + p.recordCopilotUsageWithCache("a", "m", 1, 2, 3, 4) + sink := &memUsageSink{} + p.SetTokenSink(sink) + p.recordUsage("a", "m", 1, 2) + p.recordCopilotUsage("a", "m", 3, 4) + p.recordCopilotUsageWithCache("a", "m", 5, 6, 7, 8) + recs := sink.forAgent("a") + if len(recs) != 3 || recs[0].copilot || !recs[1].copilot || recs[2].cacheCreation != 8 { + t.Fatalf("records = %+v", recs) + } +} diff --git a/src/pkg/proxy/identify_agent_by_uid_test.go b/src/crux/proxy/identify_agent_by_uid_test.go similarity index 98% rename from src/pkg/proxy/identify_agent_by_uid_test.go rename to src/crux/proxy/identify_agent_by_uid_test.go index 5a3938a6f6..ee674ad05f 100644 --- a/src/pkg/proxy/identify_agent_by_uid_test.go +++ b/src/crux/proxy/identify_agent_by_uid_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // TestIdentifyAgentByUID_FailsClosed pins the contract the Jev decision diff --git a/src/pkg/proxy/inference_auth.go b/src/crux/proxy/inference_auth.go similarity index 100% rename from src/pkg/proxy/inference_auth.go rename to src/crux/proxy/inference_auth.go diff --git a/src/pkg/proxy/inference_auth_signal_test.go b/src/crux/proxy/inference_auth_signal_test.go similarity index 100% rename from src/pkg/proxy/inference_auth_signal_test.go rename to src/crux/proxy/inference_auth_signal_test.go diff --git a/src/pkg/proxy/inference_auth_test.go b/src/crux/proxy/inference_auth_test.go similarity index 100% rename from src/pkg/proxy/inference_auth_test.go rename to src/crux/proxy/inference_auth_test.go diff --git a/src/pkg/proxy/inference_budget.go b/src/crux/proxy/inference_budget.go similarity index 100% rename from src/pkg/proxy/inference_budget.go rename to src/crux/proxy/inference_budget.go diff --git a/src/pkg/proxy/inference_budget_test.go b/src/crux/proxy/inference_budget_test.go similarity index 100% rename from src/pkg/proxy/inference_budget_test.go rename to src/crux/proxy/inference_budget_test.go diff --git a/src/pkg/proxy/inference_error_message.go b/src/crux/proxy/inference_error_message.go similarity index 100% rename from src/pkg/proxy/inference_error_message.go rename to src/crux/proxy/inference_error_message.go diff --git a/src/pkg/proxy/inference_error_message_test.go b/src/crux/proxy/inference_error_message_test.go similarity index 100% rename from src/pkg/proxy/inference_error_message_test.go rename to src/crux/proxy/inference_error_message_test.go diff --git a/src/pkg/proxy/inference_forward.go b/src/crux/proxy/inference_forward.go similarity index 100% rename from src/pkg/proxy/inference_forward.go rename to src/crux/proxy/inference_forward.go diff --git a/src/pkg/proxy/inference_forward_url_test.go b/src/crux/proxy/inference_forward_url_test.go similarity index 97% rename from src/pkg/proxy/inference_forward_url_test.go rename to src/crux/proxy/inference_forward_url_test.go index 157bd33417..b17fcb6d6c 100644 --- a/src/pkg/proxy/inference_forward_url_test.go +++ b/src/crux/proxy/inference_forward_url_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/inference" + "github.com/hivecommons/hive/crux/inference" ) // The dashboard's Test Connection prints inference.ChatCompletionsURL as "the diff --git a/src/pkg/proxy/inference_integration_test.go b/src/crux/proxy/inference_integration_test.go similarity index 100% rename from src/pkg/proxy/inference_integration_test.go rename to src/crux/proxy/inference_integration_test.go diff --git a/src/pkg/proxy/inference_last_error.go b/src/crux/proxy/inference_last_error.go similarity index 100% rename from src/pkg/proxy/inference_last_error.go rename to src/crux/proxy/inference_last_error.go diff --git a/src/pkg/proxy/inference_last_error_test.go b/src/crux/proxy/inference_last_error_test.go similarity index 100% rename from src/pkg/proxy/inference_last_error_test.go rename to src/crux/proxy/inference_last_error_test.go diff --git a/src/pkg/proxy/inference_path.go b/src/crux/proxy/inference_path.go similarity index 100% rename from src/pkg/proxy/inference_path.go rename to src/crux/proxy/inference_path.go diff --git a/src/pkg/proxy/inference_path_test.go b/src/crux/proxy/inference_path_test.go similarity index 100% rename from src/pkg/proxy/inference_path_test.go rename to src/crux/proxy/inference_path_test.go diff --git a/src/pkg/proxy/inference_route.go b/src/crux/proxy/inference_route.go similarity index 99% rename from src/pkg/proxy/inference_route.go rename to src/crux/proxy/inference_route.go index 3a53b6e22b..9a9effe597 100644 --- a/src/pkg/proxy/inference_route.go +++ b/src/crux/proxy/inference_route.go @@ -12,7 +12,7 @@ import ( "sync" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // InferenceRoute defines where to send an agent's API traffic when using diff --git a/src/pkg/proxy/inference_route_test.go b/src/crux/proxy/inference_route_test.go similarity index 100% rename from src/pkg/proxy/inference_route_test.go rename to src/crux/proxy/inference_route_test.go diff --git a/src/crux/proxy/inference_translator_identity_test.go b/src/crux/proxy/inference_translator_identity_test.go new file mode 100644 index 0000000000..ca16e897a7 --- /dev/null +++ b/src/crux/proxy/inference_translator_identity_test.go @@ -0,0 +1,169 @@ +package proxy + +import ( + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/agent" +) + +// fakeProcNetTCP points the UID lookup at a socket table in which local port +// 43210 (0xA8CA) belongs to uid 2003 and nothing else is listed. +func fakeProcNetTCP(t *testing.T) { + t.Helper() + fakeProcNetTCPUID(t, 2003) +} + +func fakeProcNetTCPUID(t *testing.T, uid int) { + t.Helper() + dir := t.TempDir() + tcp := filepath.Join(dir, "tcp") + tcp6 := filepath.Join(dir, "tcp6") + if err := os.WriteFile(tcp, []byte(fmt.Sprintf(" sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n"+ + " 0: 0100007F:A8CA 0100007F:480E 01 00000000:00000000 00:00000000 00000000 %d 0 1 0000000000000000 100 0 0 10 0\n", uid)), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(tcp6, []byte("header\n"), 0o644); err != nil { + t.Fatal(err) + } + oldTCP, oldTCP6 := procNetTCPPath, procNetTCP6Path + procNetTCPPath, procNetTCP6Path = tcp, tcp6 + t.Cleanup(func() { procNetTCPPath, procNetTCP6Path = oldTCP, oldTCP6 }) +} + +func translatorReq(remoteAddr, claimedAgent string) *http.Request { + req := httptest.NewRequest(http.MethodPost, "http://127.0.0.1:18444/v1/messages", strings.NewReader(`{}`)) + req.RemoteAddr = remoteAddr + if claimedAgent != "" { + req.Header.Set("x-api-key", inferenceAPIKeyPrefix+claimedAgent) + } + return req +} + +// TestTranslatorAgentName_SocketUIDWins pins that the inference translator +// names its caller from the socket owner's UID, not from the x-api-key the +// caller chose: "sk-hive-" is the agent's public name, so a header +// claiming another agent must not hand over that agent's inference route. +func TestTranslatorAgentName_SocketUIDWins(t *testing.T) { + fakeProcNetTCP(t) + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + p.uidMap = &agent.UIDMap{BaseUID: 2001, Agents: map[string]int{"scanner": 2003, "quality": 2002}} + + // Mapped UID, header claims a different agent → the UID's agent. + name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "quality")) + if !ok || name != "scanner" { + t.Fatalf("mapped uid with foreign claim: got (%q,%v), want (scanner,true)", name, ok) + } + // Mapped UID, no header at all → still identified. + name, ok = p.translatorAgentName(translatorReq("127.0.0.1:43210", "")) + if !ok || name != "scanner" { + t.Fatalf("mapped uid without header: got (%q,%v), want (scanner,true)", name, ok) + } +} + +// TestTranslatorAgentName_UnmappedRefusedUnlessAdvisoryOK pins the fail-closed +// half: with a UID map loaded, a socket whose UID is not an agent is refused +// whatever the header says, unless the operator set HIVE_PROXY_ADVISORY_OK. +func TestTranslatorAgentName_UnmappedRefusedUnlessAdvisoryOK(t *testing.T) { + fakeProcNetTCP(t) + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + p.uidMap = &agent.UIDMap{BaseUID: 2001, Agents: map[string]int{"quality": 2002}} + + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "quality")); ok || name != "" { + t.Fatalf("unmapped uid: got (%q,%v), want refused", name, ok) + } + // Port absent from the table → unidentified too. + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:1", "quality")); ok || name != "" { + t.Fatalf("unknown socket: got (%q,%v), want refused", name, ok) + } + + p.proxyAdvisoryOK = true + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "quality")); !ok || name != "quality" { + t.Fatalf("advisory-ok unmapped uid: got (%q,%v), want (quality,true)", name, ok) + } +} + +// TestTranslatorAgentName_InternalUIDKeepsHeader pins that the hive's own +// control plane (an internal UID) may still name the agent it is acting for: +// it is the party that issues the sk-hive- keys in the first place. +func TestTranslatorAgentName_InternalUIDKeepsHeader(t *testing.T) { + fakeProcNetTCPUID(t, 0) + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + // Root is the control plane; the separate proxy UID is not trusted. + p.uidMap = &agent.UIDMap{BaseUID: 2001, ProxyUID: 2003, Agents: map[string]int{"quality": 2002}} + + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "supervisor")); !ok || name != "supervisor" { + t.Fatalf("internal uid: got (%q,%v), want (supervisor,true)", name, ok) + } +} + +// TestTranslatorAgentName_NoUIDMap pins the no-map deployments: the header is +// trusted only under HIVE_PROXY_ADVISORY_OK, matching fallbackAgentName. +func TestTranslatorAgentName_NoUIDMap(t *testing.T) { + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "quality")); ok || name != "" { + t.Fatalf("no map, no advisory-ok: got (%q,%v), want refused", name, ok) + } + p.proxyAdvisoryOK = true + if name, ok := p.translatorAgentName(translatorReq("127.0.0.1:43210", "quality")); !ok || name != "quality" { + t.Fatalf("no map, advisory-ok: got (%q,%v), want (quality,true)", name, ok) + } +} + +// TestInferenceTranslatorHandler_RefusesUnidentified drives the handler itself: +// an unidentifiable caller gets 401 with an Anthropic-shaped error body and no +// route lookup, so a borrowed name never reaches the backend. +func TestInferenceTranslatorHandler_RefusesUnidentified(t *testing.T) { + fakeProcNetTCP(t) + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + p.uidMap = &agent.UIDMap{BaseUID: 2001, Agents: map[string]int{"quality": 2002}} + + rec := httptest.NewRecorder() + p.inferenceTranslatorHandler().ServeHTTP(rec, translatorReq("127.0.0.1:43210", "quality")) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401; body %s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), `"authentication_error"`) { + t.Fatalf("body %q lacks Anthropic-shaped authentication_error", rec.Body.String()) + } +} + +// Private forwarding may reach the translator, but its proxy-owned socket must +// not turn a forged agent name into an authenticated inference request. +func TestPlainForwardPrivateTranslatorRejectsProxyUID(t *testing.T) { + fakeProcNetTCP(t) + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil)), inference: newInferenceRouter()} + p.uidMap = &agent.UIDMap{BaseUID: 2001, ProxyUID: 2003, Agents: map[string]int{"quality": 2002}} + t.Setenv(plainForwardAllowPrivateEnv, "true") + p.allowPrivateForward = plainForwardAllowPrivate() + handler := p.inferenceTranslatorHandler() + translator := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Model the proxy-owned upstream socket using the synthetic proc table. + r.RemoteAddr = "127.0.0.1:43210" + handler.ServeHTTP(w, r) + })) + defer translator.Close() + req, err := http.NewRequest(http.MethodPost, translator.URL+"/v1/messages", strings.NewReader(`{}`)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("x-api-key", inferenceAPIKeyPrefix+"quality") + resp := plainForwardRoundTrip(t, p, req) + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("forwarded proxy UID: status = %d, want 401", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(body), `"authentication_error"`) { + t.Fatalf("body %q lacks authentication_error", body) + } +} diff --git a/src/pkg/proxy/inference_translator_start_test.go b/src/crux/proxy/inference_translator_start_test.go similarity index 100% rename from src/pkg/proxy/inference_translator_start_test.go rename to src/crux/proxy/inference_translator_start_test.go diff --git a/src/pkg/proxy/issue_shape_limits_test.go b/src/crux/proxy/issue_shape_limits_test.go similarity index 100% rename from src/pkg/proxy/issue_shape_limits_test.go rename to src/crux/proxy/issue_shape_limits_test.go diff --git a/src/pkg/proxy/issue_shape_test.go b/src/crux/proxy/issue_shape_test.go similarity index 100% rename from src/pkg/proxy/issue_shape_test.go rename to src/crux/proxy/issue_shape_test.go diff --git a/src/pkg/proxy/linear_inject_test.go b/src/crux/proxy/linear_inject_test.go similarity index 99% rename from src/pkg/proxy/linear_inject_test.go rename to src/crux/proxy/linear_inject_test.go index d636920d53..f1f3e9c1e2 100644 --- a/src/pkg/proxy/linear_inject_test.go +++ b/src/crux/proxy/linear_inject_test.go @@ -7,7 +7,7 @@ import ( "net/http" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // relayLinearRequest pushes one agent request for api.linear.app through the diff --git a/src/pkg/proxy/linear_rules.go b/src/crux/proxy/linear_rules.go similarity index 99% rename from src/pkg/proxy/linear_rules.go rename to src/crux/proxy/linear_rules.go index 1e5e3297ba..5546be8e33 100644 --- a/src/pkg/proxy/linear_rules.go +++ b/src/crux/proxy/linear_rules.go @@ -4,7 +4,7 @@ import ( "encoding/json" "strings" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // Linear ACMM enforcement (#4492, component F). diff --git a/src/pkg/proxy/linear_rules_parser_edges_test.go b/src/crux/proxy/linear_rules_parser_edges_test.go similarity index 99% rename from src/pkg/proxy/linear_rules_parser_edges_test.go rename to src/crux/proxy/linear_rules_parser_edges_test.go index b89d50e87e..7d0209b531 100644 --- a/src/pkg/proxy/linear_rules_parser_edges_test.go +++ b/src/crux/proxy/linear_rules_parser_edges_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // ───────────────────────────────────────────────────────────────────────────── diff --git a/src/pkg/proxy/linear_rules_test.go b/src/crux/proxy/linear_rules_test.go similarity index 99% rename from src/pkg/proxy/linear_rules_test.go rename to src/crux/proxy/linear_rules_test.go index b0d6c7de89..6f9374e007 100644 --- a/src/pkg/proxy/linear_rules_test.go +++ b/src/crux/proxy/linear_rules_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // allModes is every ACMM tier, lowest first. Tests that assert "at every tier" diff --git a/src/crux/proxy/loopback_host_test.go b/src/crux/proxy/loopback_host_test.go new file mode 100644 index 0000000000..093af30643 --- /dev/null +++ b/src/crux/proxy/loopback_host_test.go @@ -0,0 +1,24 @@ +package proxy + +import "testing" + +func TestIsLoopbackHost(t *testing.T) { + cases := map[string]bool{ + "localhost": true, + "LOCALHOST:3000": true, + "127.0.0.1": true, + "127.0.0.1:8080": true, + "[::1]:3000": true, + "::1": true, + "[::1]": true, + "example.com:80": false, + "10.0.0.5:3000": false, + "": false, + "localhost.evil.x": false, + } + for in, want := range cases { + if got := isLoopbackHost(in); got != want { + t.Errorf("isLoopbackHost(%q) = %v, want %v", in, got, want) + } + } +} diff --git a/src/pkg/proxy/origin_label_inject_test.go b/src/crux/proxy/origin_label_inject_test.go similarity index 100% rename from src/pkg/proxy/origin_label_inject_test.go rename to src/crux/proxy/origin_label_inject_test.go diff --git a/src/crux/proxy/plain_forward_guard.go b/src/crux/proxy/plain_forward_guard.go new file mode 100644 index 0000000000..f2628b4af9 --- /dev/null +++ b/src/crux/proxy/plain_forward_guard.go @@ -0,0 +1,171 @@ +package proxy + +import ( + "context" + "fmt" + "net" + "net/http" + "net/netip" + "os" + "strings" + "syscall" +) + +// The plain-HTTP (non-CONNECT) forward path is the one egress path through +// this proxy that carried NO destination policy at all: handleConn hands every +// non-CONNECT request to forwardPlainDirect, which RoundTripped it unchanged. +// An agent that sets a proxy URL by hand — or simply asks its CLI for +// `http://` — therefore had the proxy open a connection on its +// behalf, from the PROXY's uid, to any address the agent named. The forced +// egress gate does not cover it: the iptables rule only redirects :443, so +// plain HTTP is reachable only through this path, and the proxy's own uid is +// exempted from the redirect precisely so its upstream dials work. +// +// The interesting targets are not on the public internet, where the agent's +// own egress would have reached anyway. They are the addresses only the pod +// can see: the hive's own loopback services beside the proxy (the inference +// translator on :18444, the decision endpoint of hivecommons/hive#8939, the +// dashboard/API), the cloud instance-metadata service on 169.254.169.254 +// (node credentials on most clouds), and the cluster-internal RFC1918 estate +// (the Kubernetes API, other spokes, internal registries). +// +// So the plain forward path dials only globally routable addresses. The check +// runs in the dialer's Control hook rather than on the request's host string: +// Control is called AFTER name resolution with the concrete remote address, so +// a hostname that resolves to 127.0.0.1 or 169.254.169.254 — a public DNS +// record pointed inward, the classic SSRF/DNS-rebinding shape — is refused on +// exactly the same evidence as a literal IP, and the connection that is +// actually opened is the one that was checked. + +// plainForwardAllowPrivateEnv lifts the destination guard for deployments that +// genuinely egress to a private network over plain HTTP (a LAN mirror or an +// internal registry on a native/laptop install, where no cluster estate sits +// behind the proxy to protect). Boot-time operator choice, read once at +// construction, same shape as HIVE_PROXY_ADVISORY_OK: only the exact string +// "true" turns it on. +const plainForwardAllowPrivateEnv = "HIVE_PROXY_ALLOW_PRIVATE_FORWARD" + +func plainForwardAllowPrivate() bool { + return strings.TrimSpace(os.Getenv(plainForwardAllowPrivateEnv)) == "true" +} + +// Destination classes refused on the plain forward path. The reason strings are +// user-facing: they appear in the 403 body an agent's curl prints, so they name +// what was refused rather than a policy code. +const ( + plainForwardReasonLoopback = "loopback" + plainForwardReasonLinkLocal = "link-local" + plainForwardReasonMulticast = "multicast" + plainForwardReasonPrivate = "private" + plainForwardReasonCGNAT = "carrier-grade NAT" + plainForwardReasonUnspecified = "unspecified" +) + +// cgnatPrefix is RFC 6598 100.64.0.0/10. It is neither RFC 1918 (so +// netip.Addr.IsPrivate does not report it) nor routable on the public +// internet; several managed Kubernetes platforms number pod and service +// networks out of it, which is exactly the estate this guard protects. +var cgnatPrefix = netip.MustParsePrefix("100.64.0.0/10") + +// plainForwardDenied is the dial-time refusal, returned from the Control hook +// so http.Transport aborts the dial. forwardPlainDirect recovers it with +// errors.As to answer 403 (a policy refusal the agent can act on) instead of +// 502 (an upstream that failed). +type plainForwardDenied struct { + target string + reason string +} + +func (e *plainForwardDenied) Error() string { + return fmt.Sprintf("plain HTTP forward to %s refused: %s address", e.target, e.reason) +} + +// plainForwardDenyReasonAddr names the class that bars an address, or "" when +// the address is globally routable and may be dialled. +func plainForwardDenyReasonAddr(addr netip.Addr) string { + // An IPv4-mapped IPv6 address (::ffff:127.0.0.1) is the same host as its + // IPv4 form; unmap so one rule covers both spellings. + addr = addr.Unmap() + switch { + case !addr.IsValid(), addr.IsUnspecified(): + return plainForwardReasonUnspecified + case addr.IsLoopback(): + return plainForwardReasonLoopback + case addr.IsLinkLocalUnicast(), addr.IsLinkLocalMulticast(): + // 169.254.0.0/16 and fe80::/10 — cloud instance metadata lives here. + return plainForwardReasonLinkLocal + case addr.IsMulticast(), addr.IsInterfaceLocalMulticast(): + return plainForwardReasonMulticast + case addr.IsPrivate(): + // RFC 1918 for IPv4, fc00::/7 (unique local) for IPv6. + return plainForwardReasonPrivate + case cgnatPrefix.Contains(addr): + return plainForwardReasonCGNAT + } + return "" +} + +// plainForwardDenyReason classifies the "ip:port" the dialer is about to +// connect to. An address that does not parse as a literal IP cannot be +// classified and is refused: Control is always handed a resolved address, so +// an unparseable one means the policy could not be applied, and the deny +// direction is the safe one. +func plainForwardDenyReason(address string) string { + host := address + if h, _, err := net.SplitHostPort(address); err == nil { + host = h + } + addr, err := netip.ParseAddr(strings.Trim(host, "[]")) + if err != nil { + return plainForwardReasonUnspecified + } + return plainForwardDenyReasonAddr(addr) +} + +// guardedPlainForwardTransport and unguardedPlainForwardTransport are built +// once and shared: a per-request Transport would drop keep-alives and leak +// idle connections. Which one a forward uses is the proxy's +// allowPrivateForward choice, so the policy stays a property of the +// deployment, not of the request. +var ( + guardedPlainForwardTransport = newPlainForwardTransport(true) + unguardedPlainForwardTransport = newPlainForwardTransport(false) +) + +// newPlainForwardTransport clones the standard transport (keeping its +// connection pooling and timeouts), drops its ProxyFromEnvironment so an +// HTTP_PROXY/HTTPS_PROXY on the proxy process is deliberately ignored — with +// it the dial-time check would see the upstream proxy's address instead of the +// agent-named destination and evaluate the wrong peer — and replaces the dialer so every plain forward is SO_MARKed like the proxy's +// other upstream dials — without the mark, forwarding `http://host:443` dials +// :443 and the forced-egress redirect sends it straight back into this proxy — +// and, when guarded, destination-checked at resolution time. +func newPlainForwardTransport(guard bool) *http.Transport { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + dialer := markDialer(upstreamDialTimeout) + if guard { + mark := dialer.Control + dialer.Control = func(network, address string, c syscall.RawConn) error { + if reason := plainForwardDenyReason(address); reason != "" { + return &plainForwardDenied{target: address, reason: reason} + } + return mark(network, address, c) + } + } + return dialer.DialContext(ctx, network, address) + } + return transport +} + +// plainForwardTransport returns the transport this proxy forwards plain HTTP +// through. The zero value of allowPrivateForward is the guarded one, so a +// GitHubProxy built as a struct literal (tests, embedders) is protected +// without opting in. +func (p *GitHubProxy) plainForwardTransport() http.RoundTripper { + if p.allowPrivateForward { + return unguardedPlainForwardTransport + } + return guardedPlainForwardTransport +} diff --git a/src/crux/proxy/plain_forward_guard_test.go b/src/crux/proxy/plain_forward_guard_test.go new file mode 100644 index 0000000000..a8569d57df --- /dev/null +++ b/src/crux/proxy/plain_forward_guard_test.go @@ -0,0 +1,283 @@ +package proxy + +import ( + "bufio" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "testing" + "time" +) + +// plainForwardRoundTrip drives forwardPlainDirect over a pipe and returns the +// response the agent-side client would see. The handler side of the pipe is +// closed after the handler returns so the body read sees EOF (forwardPlainDirect +// leaves the conn open; handleConn owns its lifecycle in production). +func plainForwardRoundTrip(t *testing.T, p *GitHubProxy, req *http.Request) *http.Response { + t.Helper() + clientConn, proxyClient := net.Pipe() + t.Cleanup(func() { _ = clientConn.Close() }) + + go func() { + p.forwardPlainDirect(proxyClient, req) + _ = proxyClient.Close() + }() + + _ = clientConn.SetReadDeadline(time.Now().Add(10 * time.Second)) + resp, err := http.ReadResponse(bufio.NewReader(clientConn), req) + if err != nil { + t.Fatalf("read response: %v", err) + } + t.Cleanup(func() { _ = resp.Body.Close() }) + return resp +} + +func TestPlainForwardDenyReasonClassifiesDestinations(t *testing.T) { + cases := map[string]string{ + // Refused: everything only the pod/node can reach. + "127.0.0.1:80": plainForwardReasonLoopback, + "127.0.0.53:8080": plainForwardReasonLoopback, + "[::1]:80": plainForwardReasonLoopback, + "[::ffff:127.0.0.1]:80": plainForwardReasonLoopback, + "169.254.169.254:80": plainForwardReasonLinkLocal, + "[fe80::1]:80": plainForwardReasonLinkLocal, + "10.0.0.5:80": plainForwardReasonPrivate, + "172.16.4.1:8080": plainForwardReasonPrivate, + "192.168.1.1:80": plainForwardReasonPrivate, + "[fd00::1]:80": plainForwardReasonPrivate, + "100.64.0.1:80": plainForwardReasonCGNAT, + // 224.0.0.0/24 (mDNS) is link-local multicast; 239.0.0.0/8 is not. + "224.0.0.251:5353": plainForwardReasonLinkLocal, + "239.255.255.250:1900": plainForwardReasonMulticast, + "[ff02::1]:80": plainForwardReasonLinkLocal, + "0.0.0.0:80": plainForwardReasonUnspecified, + "[::]:80": plainForwardReasonUnspecified, + "not-an-ip:80": plainForwardReasonUnspecified, + "192.168.1.1": plainForwardReasonPrivate, + // Allowed: globally routable. + "93.184.216.34:80": "", + "[2606:2800:220:1::1]:80": "", + "140.82.121.4:443": "", + } + for address, want := range cases { + if got := plainForwardDenyReason(address); got != want { + t.Errorf("plainForwardDenyReason(%q) = %q, want %q", address, got, want) + } + } +} + +// TestPlainForwardDenyReasonAddrUnmapsV4 pins the IPv4-mapped IPv6 case +// separately: ::ffff:10.0.0.1 is the private host 10.0.0.1 written the other +// way round, and netip's IsPrivate reports false until the address is unmapped. +func TestPlainForwardDenyReasonAddrUnmapsV4(t *testing.T) { + mapped := netip.MustParseAddr("::ffff:10.0.0.1") + if got := plainForwardDenyReasonAddr(mapped); got != plainForwardReasonPrivate { + t.Errorf("plainForwardDenyReasonAddr(%v) = %q, want %q", mapped, got, plainForwardReasonPrivate) + } +} + +// TestForwardPlainDirectRefusesLoopbackDestination is the SSRF regression: a +// plain-HTTP request naming a service beside the proxy must be refused with 403 +// and must never reach that service. +func TestForwardPlainDirectRefusesLoopbackDestination(t *testing.T) { + reached := make(chan struct{}, 1) + internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + reached <- struct{}{} + w.WriteHeader(http.StatusOK) + })) + defer internal.Close() + + p := newTestProxy() + req, err := http.NewRequest(http.MethodGet, internal.URL+"/secret", nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + + resp := plainForwardRoundTrip(t, p, req) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want 403 for a loopback destination", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + if !strings.Contains(string(body), plainForwardReasonLoopback) { + t.Errorf("body = %q, want the refused address class named", string(body)) + } + select { + case <-reached: + t.Fatal("the loopback service was reached — the proxy forwarded a request it must refuse") + default: + } +} + +// TestForwardPlainDirectRefusesHostnameResolvingInward covers the SSRF shape a +// host-string check misses: the request names a HOSTNAME, and only the resolved +// address is non-routable. "localhost" stands in for any public DNS record +// pointed at the loopback or metadata address. +func TestForwardPlainDirectRefusesHostnameResolvingInward(t *testing.T) { + internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer internal.Close() + + _, port, err := net.SplitHostPort(strings.TrimPrefix(internal.URL, "http://")) + if err != nil { + t.Fatalf("split upstream addr: %v", err) + } + + p := newTestProxy() + req, err := http.NewRequest(http.MethodGet, "http://localhost:"+port+"/secret", nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + + resp := plainForwardRoundTrip(t, p, req) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want 403 for a hostname that resolves to loopback", resp.StatusCode) + } +} + +// TestForwardPlainDirectCountsNonLoopbackRefusalAsViolation separates the two +// refusal flavours: loopback is a client that ignored NO_PROXY (#14076) and is +// not held against the caller, while reaching for the private estate is. +func TestForwardPlainDirectCountsNonLoopbackRefusalAsViolation(t *testing.T) { + p := newTestProxy() + p.proxyAdvisoryOK = true // no UID map in tests; accept the header identity + + metadata, err := http.NewRequest(http.MethodGet, "http://169.254.169.254/latest/meta-data/", nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + metadata.Header.Set("Proxy-Authorization", "hive scanner") + + if resp := plainForwardRoundTrip(t, p, metadata); resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want 403 for the cloud metadata address", resp.StatusCode) + } + if got := p.violations["scanner"]; got != 1 { + t.Errorf("violations[scanner] = %d, want 1 — a metadata-service attempt must be attributed and counted", got) + } + + loopback, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:9/", nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + loopback.Header.Set("Proxy-Authorization", "hive scanner") + + if resp := plainForwardRoundTrip(t, p, loopback); resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want 403 for a loopback destination", resp.StatusCode) + } + if got := p.violations["scanner"]; got != 1 { + t.Errorf("violations[scanner] = %d, want it unchanged at 1 — a loopback request is a client misconfiguration, not a violation", got) + } +} + +// TestForwardPlainDirectStripsProxyHopHeaders pins that the hop-by-hop headers +// addressed to THIS proxy do not travel upstream: Proxy-Authorization carries +// the caller's asserted identity (and, in the Basic form, a password field). +func TestForwardPlainDirectStripsProxyHopHeaders(t *testing.T) { + seen := make(chan http.Header, 1) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen <- r.Header.Clone() + w.WriteHeader(http.StatusOK) + })) + defer upstream.Close() + + p := newTestProxy() + p.allowPrivateForward = true // the upstream under test is a loopback httptest server + + req, err := http.NewRequest(http.MethodGet, upstream.URL+"/ok", nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Header.Set("Proxy-Authorization", "hive scanner") + req.Header.Set("Proxy-Connection", "keep-alive") + req.Header.Set("X-Keep", "me") + + if resp := plainForwardRoundTrip(t, p, req); resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + + headers := <-seen + if got := headers.Get("Proxy-Authorization"); got != "" { + t.Errorf("upstream saw Proxy-Authorization = %q, want it stripped", got) + } + if got := headers.Get("Proxy-Connection"); got != "" { + t.Errorf("upstream saw Proxy-Connection = %q, want it stripped", got) + } + if got := headers.Get("X-Keep"); got != "me" { + t.Errorf("upstream saw X-Keep = %q, want the request's own headers forwarded untouched", got) + } +} + +// TestNewGitHubProxyWiresPlainForwardEscapeHatch pins the operator opt-in: a +// deployment that genuinely egresses to a private network over plain HTTP can +// turn the guard off, and nothing else can. +func TestNewGitHubProxyWiresPlainForwardEscapeHatch(t *testing.T) { + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + t.Setenv(plainForwardAllowPrivateEnv, "") + guarded, err := NewGitHubProxyEphemeral(logger, "hivecommons", []string{"hive"}) + if err != nil { + t.Fatalf("NewGitHubProxyEphemeral: %v", err) + } + if guarded.allowPrivateForward { + t.Error("the plain-HTTP destination guard must be on when the opt-in is unset") + } + if guarded.plainForwardTransport() != http.RoundTripper(guardedPlainForwardTransport) { + t.Error("a guarded proxy must forward through the guarded transport") + } + + t.Setenv(plainForwardAllowPrivateEnv, "true") + lifted, err := NewGitHubProxyEphemeral(logger, "hivecommons", []string{"hive"}) + if err != nil { + t.Fatalf("NewGitHubProxyEphemeral: %v", err) + } + if !lifted.allowPrivateForward { + t.Errorf("%s=true must lift the guard", plainForwardAllowPrivateEnv) + } + if lifted.plainForwardTransport() != http.RoundTripper(unguardedPlainForwardTransport) { + t.Error("a proxy with the guard lifted must forward through the unguarded transport") + } + + t.Setenv(plainForwardAllowPrivateEnv, "yes") + fuzzy, err := NewGitHubProxyEphemeral(logger, "hivecommons", []string{"hive"}) + if err != nil { + t.Fatalf("NewGitHubProxyEphemeral: %v", err) + } + if fuzzy.allowPrivateForward { + t.Errorf("only the exact string \"true\" may lift the guard, not %q", "yes") + } +} + +// TestPlainForwardTransportIgnoresProxyEnvironment pins that the plain-forward +// transports dial the named destination directly, so the dial-time check never +// evaluates an upstream proxy's address. +func TestPlainForwardTransportIgnoresProxyEnvironment(t *testing.T) { + for _, guard := range []bool{true, false} { + if newPlainForwardTransport(guard).Proxy != nil { + t.Errorf("newPlainForwardTransport(%v).Proxy is set, want nil", guard) + } + } + + t.Setenv("HTTP_PROXY", "http://127.0.0.1:1") + t.Setenv("http_proxy", "http://127.0.0.1:1") + internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer internal.Close() + + req, err := http.NewRequest(http.MethodGet, internal.URL+"/secret", nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + resp := plainForwardRoundTrip(t, newTestProxy(), req) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want 403 for a loopback destination", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + if !strings.Contains(string(body), plainForwardReasonLoopback) { + t.Errorf("body = %q, want the refused address class named", string(body)) + } +} diff --git a/src/pkg/proxy/procnet.go b/src/crux/proxy/procnet.go similarity index 100% rename from src/pkg/proxy/procnet.go rename to src/crux/proxy/procnet.go diff --git a/src/pkg/proxy/procnet_test.go b/src/crux/proxy/procnet_test.go similarity index 100% rename from src/pkg/proxy/procnet_test.go rename to src/crux/proxy/procnet_test.go diff --git a/src/pkg/proxy/proxy_coverage2_test.go b/src/crux/proxy/proxy_coverage2_test.go similarity index 99% rename from src/pkg/proxy/proxy_coverage2_test.go rename to src/crux/proxy/proxy_coverage2_test.go index 113a9de51f..12f38376f6 100644 --- a/src/pkg/proxy/proxy_coverage2_test.go +++ b/src/crux/proxy/proxy_coverage2_test.go @@ -17,8 +17,6 @@ import ( "strings" "testing" "time" - - "github.com/hivecommons/hive/pkg/tokens" ) // newInferenceTestProxy builds a GitHubProxy with the maps/routers the @@ -73,9 +71,9 @@ func TestSetTokenSink(t *testing.T) { if p.tokenSink != nil { t.Fatal("expected nil sink before SetTokenSink") } - sink := tokens.NewInferenceSink("", slog.Default()) + sink := &memUsageSink{} p.SetTokenSink(sink) - if p.tokenSink != sink { + if p.tokenSink != UsageSink(sink) { t.Fatal("SetTokenSink did not wire the sink") } } @@ -596,7 +594,7 @@ func TestJSONEscape(t *testing.T) { // port and issuing requests against it. func TestStartInferenceTranslator_HandlerBranches(t *testing.T) { p := newInferenceTestProxy() - p.SetTokenSink(tokens.NewInferenceSink("", slog.Default())) + p.SetTokenSink(&memUsageSink{}) // Drive the real handler without binding the fixed production port. // Mock upstream vLLM. @@ -675,7 +673,7 @@ func TestStartInferenceTranslator_HandlerBranches(t *testing.T) { // are rerouted through handleAnthropicReroute → handleInferenceRequest. func TestHandleAnthropicReroute_NonStreaming(t *testing.T) { p := newInferenceTestProxy() - p.SetTokenSink(tokens.NewInferenceSink("", slog.Default())) + p.SetTokenSink(&memUsageSink{}) upstream := startMockVLLM(t) defer upstream.Close() @@ -874,7 +872,7 @@ func TestLoadOrGenerateCA_UnusablePersistedRegenerates(t *testing.T) { // connection (streaming branch), recording usage into the sink. func TestHandleInferenceRequest_Streaming(t *testing.T) { p := newInferenceTestProxy() - p.SetTokenSink(tokens.NewInferenceSink("", slog.Default())) + p.SetTokenSink(&memUsageSink{}) upstream := startMockVLLM(t) defer upstream.Close() diff --git a/src/pkg/proxy/proxy_coverage_test.go b/src/crux/proxy/proxy_coverage_test.go similarity index 99% rename from src/pkg/proxy/proxy_coverage_test.go rename to src/crux/proxy/proxy_coverage_test.go index 90b75c4f1e..3bc839b369 100644 --- a/src/pkg/proxy/proxy_coverage_test.go +++ b/src/crux/proxy/proxy_coverage_test.go @@ -9,7 +9,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) func TestAllowedByModeOAuthDeviceFlow(t *testing.T) { diff --git a/src/pkg/proxy/proxy_deep2_test.go b/src/crux/proxy/proxy_deep2_test.go similarity index 99% rename from src/pkg/proxy/proxy_deep2_test.go rename to src/crux/proxy/proxy_deep2_test.go index 05ab22a296..ebdcf8e5f0 100644 --- a/src/pkg/proxy/proxy_deep2_test.go +++ b/src/crux/proxy/proxy_deep2_test.go @@ -18,7 +18,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // ---------- NewGitHubProxy: test construction ---------- @@ -67,6 +67,7 @@ func TestNewGitHubProxy(t *testing.T) { func TestStartInferenceTranslatorIntegration(t *testing.T) { p := newTestProxy() p.inference = newInferenceRouter() + p.proxyAdvisoryOK = true // no UID map in tests; exercise routing, not identity // Mock vLLM backend mock := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/src/pkg/proxy/proxy_deep3_test.go b/src/crux/proxy/proxy_deep3_test.go similarity index 97% rename from src/pkg/proxy/proxy_deep3_test.go rename to src/crux/proxy/proxy_deep3_test.go index 655bc3255f..a90aeadde2 100644 --- a/src/pkg/proxy/proxy_deep3_test.go +++ b/src/crux/proxy/proxy_deep3_test.go @@ -15,7 +15,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // ---------- StartInferenceTranslator: real integration test ---------- @@ -61,12 +61,13 @@ func TestStartInferenceTranslatorReal(t *testing.T) { } p := &GitHubProxy{ - caCert: caCert, - caX509: caX509, - logger: slog.Default(), - violations: make(map[string]int), - certCache: make(map[string]cachedCert), - inference: newInferenceRouter(), + caCert: caCert, + caX509: caX509, + logger: slog.Default(), + violations: make(map[string]int), + certCache: make(map[string]cachedCert), + inference: newInferenceRouter(), + proxyAdvisoryOK: true, // no UID map in tests; exercise routing, not identity } p.inference.Set("agent-non-stream", &InferenceRoute{Backend: "vllm", Endpoint: mockNonStream.URL, Model: "test-model"}) p.inference.Set("agent-stream", &InferenceRoute{Backend: "vllm", Endpoint: mockNonStream.URL, Model: "test-model"}) diff --git a/src/pkg/proxy/proxy_deep4_test.go b/src/crux/proxy/proxy_deep4_test.go similarity index 93% rename from src/pkg/proxy/proxy_deep4_test.go rename to src/crux/proxy/proxy_deep4_test.go index 6674730a18..dcd9ba0d43 100644 --- a/src/pkg/proxy/proxy_deep4_test.go +++ b/src/crux/proxy/proxy_deep4_test.go @@ -53,12 +53,13 @@ func TestStartInferenceTranslatorActual(t *testing.T) { caCert, caX509, _ := generateCA() p := &GitHubProxy{ - caCert: caCert, - caX509: caX509, - logger: slog.Default(), - violations: make(map[string]int), - certCache: make(map[string]cachedCert), - inference: newInferenceRouter(), + caCert: caCert, + caX509: caX509, + logger: slog.Default(), + violations: make(map[string]int), + certCache: make(map[string]cachedCert), + inference: newInferenceRouter(), + proxyAdvisoryOK: true, // no UID map in tests; exercise routing, not identity } p.inference.Set("agent1", &InferenceRoute{Backend: "vllm", Endpoint: mock.URL, Model: "test-model"}) diff --git a/src/pkg/proxy/proxy_deep5_test.go b/src/crux/proxy/proxy_deep5_test.go similarity index 98% rename from src/pkg/proxy/proxy_deep5_test.go rename to src/crux/proxy/proxy_deep5_test.go index f86c75d46c..cb78916d90 100644 --- a/src/pkg/proxy/proxy_deep5_test.go +++ b/src/crux/proxy/proxy_deep5_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // ---------- extractSNI: various truncation boundary tests ---------- @@ -510,12 +510,13 @@ func TestStartInferenceTranslatorTranslationError(t *testing.T) { caCert, caX509, _ := generateCA() p := &GitHubProxy{ - caCert: caCert, - caX509: caX509, - logger: slog.Default(), - violations: make(map[string]int), - certCache: make(map[string]cachedCert), - inference: newInferenceRouter(), + caCert: caCert, + caX509: caX509, + logger: slog.Default(), + violations: make(map[string]int), + certCache: make(map[string]cachedCert), + inference: newInferenceRouter(), + proxyAdvisoryOK: true, // no UID map in tests; exercise routing, not identity } p.inference.Set("agent-bad-resp", &InferenceRoute{Backend: "vllm", Endpoint: mock.URL, Model: "test"}) diff --git a/src/pkg/proxy/proxy_deep6_test.go b/src/crux/proxy/proxy_deep6_test.go similarity index 99% rename from src/pkg/proxy/proxy_deep6_test.go rename to src/crux/proxy/proxy_deep6_test.go index f94f3e7587..5e47a80176 100644 --- a/src/pkg/proxy/proxy_deep6_test.go +++ b/src/crux/proxy/proxy_deep6_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // ---------- proxyHTTP: git path bidirectional streaming ---------- diff --git a/src/pkg/proxy/proxy_deep7_test.go b/src/crux/proxy/proxy_deep7_test.go similarity index 99% rename from src/pkg/proxy/proxy_deep7_test.go rename to src/crux/proxy/proxy_deep7_test.go index 5bd5fde7c8..4d2b7b6bf3 100644 --- a/src/pkg/proxy/proxy_deep7_test.go +++ b/src/crux/proxy/proxy_deep7_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // ---------- handleInferenceRequest: response translation fails ---------- diff --git a/src/pkg/proxy/proxy_deep8_test.go b/src/crux/proxy/proxy_deep8_test.go similarity index 99% rename from src/pkg/proxy/proxy_deep8_test.go rename to src/crux/proxy/proxy_deep8_test.go index 74320bf612..e2434e768c 100644 --- a/src/pkg/proxy/proxy_deep8_test.go +++ b/src/crux/proxy/proxy_deep8_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // TestHandleTransparentTLSFullMITM tests the full transparent TLS path diff --git a/src/pkg/proxy/proxy_deep_test.go b/src/crux/proxy/proxy_deep_test.go similarity index 98% rename from src/pkg/proxy/proxy_deep_test.go rename to src/crux/proxy/proxy_deep_test.go index 3c3e7705cd..d678aa0d79 100644 --- a/src/pkg/proxy/proxy_deep_test.go +++ b/src/crux/proxy/proxy_deep_test.go @@ -24,7 +24,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // ---------- extractSNI: build a real TLS ClientHello ---------- @@ -355,6 +355,10 @@ func TestForwardPlainDirectSuccess(t *testing.T) { defer upstream.Close() p := newTestProxy() + // The httptest upstream listens on loopback, which the plain-forward + // destination guard refuses by default (#14082). This case is about the + // successful relay, so it runs with the operator opt-in lifted. + p.allowPrivateForward = true clientConn, proxyClient := net.Pipe() defer clientConn.Close() @@ -471,6 +475,10 @@ func TestProxyHTTPGitPath(t *testing.T) { func TestHandleConnHTTPRequest(t *testing.T) { p := newTestProxy() + // Loopback upstream: this case covers handleConn's routing of a + // non-CONNECT request, not the destination guard (#14082), so the guard is + // lifted and the forward must actually succeed. + p.allowPrivateForward = true upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -491,10 +499,8 @@ func TestHandleConnHTTPRequest(t *testing.T) { t.Fatalf("read response: %v", err) } defer resp.Body.Close() - // The forwardPlainDirect will attempt to RoundTrip - // Status could be 200 or 502 depending on host resolution - if resp.StatusCode != 200 && resp.StatusCode != 502 { - t.Errorf("unexpected status %d", resp.StatusCode) + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want 200 — handleConn must route a non-CONNECT request to forwardPlainDirect", resp.StatusCode) } } @@ -1109,6 +1115,7 @@ func TestHandleConnectDirectAnthropicReroute(t *testing.T) { func TestStartInferenceTranslator(t *testing.T) { p := newTestProxy() p.inference = newInferenceRouter() + p.proxyAdvisoryOK = true // no UID map in tests; exercise routing, not identity // Start a mock vLLM backend mock := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/src/pkg/proxy/proxy_ephemeral_plain_error_test.go b/src/crux/proxy/proxy_ephemeral_plain_error_test.go similarity index 81% rename from src/pkg/proxy/proxy_ephemeral_plain_error_test.go rename to src/crux/proxy/proxy_ephemeral_plain_error_test.go index 4c21092077..e8daa63db9 100644 --- a/src/pkg/proxy/proxy_ephemeral_plain_error_test.go +++ b/src/crux/proxy/proxy_ephemeral_plain_error_test.go @@ -2,6 +2,7 @@ package proxy import ( "bufio" + "bytes" "io" "log/slog" "net" @@ -70,6 +71,12 @@ func TestForwardPlainDirectUpstreamError(t *testing.T) { _ = l.Close() p := newTestProxy() + // A reserved-then-closed port is only reachable on loopback, which the + // destination guard refuses before dialling (#14082). This case is about + // the 502 the DIAL failure must produce, so the guard is lifted. + p.allowPrivateForward = true + var logBuf bytes.Buffer + p.logger = slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelDebug})) clientConn, proxyClient := net.Pipe() defer clientConn.Close() @@ -99,4 +106,11 @@ func TestForwardPlainDirectUpstreamError(t *testing.T) { if !strings.Contains(string(body), "upstream request failed") { t.Errorf("body = %q, want upstream failure message", string(body)) } + logs := logBuf.String() + if !strings.Contains(logs, "proxy: plain HTTP forward to loopback failed") { + t.Errorf("logs = %q, want loopback failure message", logs) + } + if !strings.Contains(logs, `hint="client should bypass the proxy for loopback; NO_PROXY"`) { + t.Errorf("logs = %q, want loopback hint attribute", logs) + } } diff --git a/src/pkg/proxy/proxy_extra_test.go b/src/crux/proxy/proxy_extra_test.go similarity index 99% rename from src/pkg/proxy/proxy_extra_test.go rename to src/crux/proxy/proxy_extra_test.go index 6211e02576..44427e69c2 100644 --- a/src/pkg/proxy/proxy_extra_test.go +++ b/src/crux/proxy/proxy_extra_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) func TestTransfer(t *testing.T) { diff --git a/src/pkg/proxy/proxy_http_test.go b/src/crux/proxy/proxy_http_test.go similarity index 99% rename from src/pkg/proxy/proxy_http_test.go rename to src/crux/proxy/proxy_http_test.go index b9fe91ae98..e071b595f4 100644 --- a/src/pkg/proxy/proxy_http_test.go +++ b/src/crux/proxy/proxy_http_test.go @@ -9,7 +9,7 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) func newTestProxy() *GitHubProxy { diff --git a/src/pkg/proxy/proxy_test.go b/src/crux/proxy/proxy_test.go similarity index 99% rename from src/pkg/proxy/proxy_test.go rename to src/crux/proxy/proxy_test.go index 559fc91718..bc73eba09b 100644 --- a/src/pkg/proxy/proxy_test.go +++ b/src/crux/proxy/proxy_test.go @@ -5,7 +5,7 @@ import ( "net/http" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) func TestIsGitHubHostExtended(t *testing.T) { diff --git a/src/pkg/proxy/proxy_threshold_test.go b/src/crux/proxy/proxy_threshold_test.go similarity index 100% rename from src/pkg/proxy/proxy_threshold_test.go rename to src/crux/proxy/proxy_threshold_test.go diff --git a/src/pkg/proxy/proxy_tls_test.go b/src/crux/proxy/proxy_tls_test.go similarity index 99% rename from src/pkg/proxy/proxy_tls_test.go rename to src/crux/proxy/proxy_tls_test.go index 54b190f9c7..3f5fd10cbd 100644 --- a/src/pkg/proxy/proxy_tls_test.go +++ b/src/crux/proxy/proxy_tls_test.go @@ -8,7 +8,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) func TestGenerateCA(t *testing.T) { diff --git a/src/pkg/proxy/proxy_transparent_attribution_test.go b/src/crux/proxy/proxy_transparent_attribution_test.go similarity index 99% rename from src/pkg/proxy/proxy_transparent_attribution_test.go rename to src/crux/proxy/proxy_transparent_attribution_test.go index a07fec3b58..3eafef0e5f 100644 --- a/src/pkg/proxy/proxy_transparent_attribution_test.go +++ b/src/crux/proxy/proxy_transparent_attribution_test.go @@ -14,8 +14,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" - "github.com/hivecommons/hive/pkg/tokens" + "github.com/hivecommons/hive/crux/agent" ) // These tests exercise handleTransparentTLS's attribution and routing branches @@ -251,7 +250,7 @@ func TestTransparentTLSCopilotSniffGate(t *testing.T) { writeProcNetTable(t, procNetRow{peerPort(t, server), agentUIDFixture}) p := newAttributionProxy(t, uidMap) - p.tokenSink = &tokens.InferenceSink{} + p.tokenSink = &memUsageSink{} copilotDialed := make(chan string, 1) p.copilotDial = func(host string) (net.Conn, error) { copilotDialed <- host diff --git a/src/pkg/proxy/proxy_transparent_tls_repro_test.go b/src/crux/proxy/proxy_transparent_tls_repro_test.go similarity index 100% rename from src/pkg/proxy/proxy_transparent_tls_repro_test.go rename to src/crux/proxy/proxy_transparent_tls_repro_test.go diff --git a/src/pkg/proxy/repo_pause_test.go b/src/crux/proxy/repo_pause_test.go similarity index 99% rename from src/pkg/proxy/repo_pause_test.go rename to src/crux/proxy/repo_pause_test.go index 580d1ed02a..a2252e7243 100644 --- a/src/pkg/proxy/repo_pause_test.go +++ b/src/crux/proxy/repo_pause_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // pausedSet turns a list of org-qualified repo names into the predicate the diff --git a/src/pkg/proxy/rules.go b/src/crux/proxy/rules.go similarity index 69% rename from src/pkg/proxy/rules.go rename to src/crux/proxy/rules.go index 3fc4a2b6ee..2888fc9c8b 100644 --- a/src/pkg/proxy/rules.go +++ b/src/crux/proxy/rules.go @@ -3,11 +3,12 @@ package proxy import ( "encoding/json" "net/http" + "net/url" "regexp" "strings" "sync" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // ProxyRule maps a GitHub API (method, path-pattern) to the minimum @@ -139,38 +140,58 @@ var rules = []ProxyRule{ // so it was refused even at ISSUES_PRS_MERGE. It is gated at the same level // as the merge itself — it writes to the PR's head branch, nothing more. {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls/\d+/update-branch$`), Method: "PUT", MinMode: agent.ModeIssuesPRsMerge}, + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/pulls/\d+/update(?:-branch)?$`), Method: "POST", MinMode: agent.ModeIssuesPRsMerge}, + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/pulls/\d+/update-branch$`), Method: "PUT", MinMode: agent.ModeIssuesPRsMerge}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/rebase$`), Method: "PUT", MinMode: agent.ModeIssuesPRsMerge}, // ── PR operations — ISSUES_AND_PRS and above ── // NOTE: direct PR creation (POST /pulls) is NOT here — it is a HARD DENY for // every mode, handled by denyRules below, so agents route through hive-open-pr // (App-bot authorship) instead of `gh pr create` / the GitHub MCP create_pull_request. {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls/\d+$`), Method: "PATCH", MinMode: agent.ModeIssuesAndPRs}, + {PathPattern: regexp.MustCompile(`^/api/v1/repos/[^/]+/[^/]+/pulls/\d+$`), Method: "PATCH", MinMode: agent.ModeIssuesAndPRs}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+$`), Method: "PUT", MinMode: agent.ModeIssuesAndPRs}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+$`), Method: "PATCH", MinMode: agent.ModeIssuesAndPRs}, // Leaving review commentary is conversation, not code. It sat behind // ModeIssuesAndPRs only because it lives under /pulls/, which meant an agent // had to be able to push branches and open PRs before it could say "this // looks wrong" (#4492). The tier is unchanged for everyone; `converse` is an // additional way in. {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls/\d+/reviews`), Method: "POST", MinMode: agent.ModeIssuesAndPRs, Capability: agent.AgentCapabilities.CanConverse}, + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/pulls/\d+/reviews`), Method: "POST", MinMode: agent.ModeIssuesAndPRs, Capability: agent.AgentCapabilities.CanConverse}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/notes`), Method: "POST", MinMode: agent.ModeIssuesAndPRs, Capability: agent.AgentCapabilities.CanConverse}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/discussions`), Method: "POST", MinMode: agent.ModeIssuesAndPRs, Capability: agent.AgentCapabilities.CanConverse}, // ── Git fetch — all modes (read-only, despite using POST) ── {PathPattern: regexp.MustCompile(`\.git/git-upload-pack$`), Method: "POST", MinMode: agent.ModeAdvisory}, // ── Git push operations — ISSUES_AND_PRS and above ── {PathPattern: regexp.MustCompile(`\.git/git-receive-pack$`), Method: "POST", MinMode: agent.ModeIssuesAndPRs}, - {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/git/refs$`), Method: "POST", MinMode: agent.ModeIssuesAndPRs}, - {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/git/commits$`), Method: "POST", MinMode: agent.ModeIssuesAndPRs}, - {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/git/refs/`), Method: "DELETE", MinMode: agent.ModeIssuesAndPRs}, + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/git/refs$`), Method: "POST", MinMode: agent.ModeIssuesAndPRs}, + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/git/commits$`), Method: "POST", MinMode: agent.ModeIssuesAndPRs}, + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/git/refs/`), Method: "DELETE", MinMode: agent.ModeIssuesAndPRs}, + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/branches`), Method: "POST", MinMode: agent.ModeIssuesAndPRs}, + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/branches/`), Method: "DELETE", MinMode: agent.ModeIssuesAndPRs}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/repository/branches`), Method: "POST", MinMode: agent.ModeIssuesAndPRs}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/repository/branches/`), Method: "DELETE", MinMode: agent.ModeIssuesAndPRs}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/repository/commits`), Method: "POST", MinMode: agent.ModeIssuesAndPRs}, // ── Issue operations — ISSUES_ONLY and above ── - {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues$`), Method: "POST", MinMode: agent.ModeIssuesOnly}, - {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+$`), Method: "PATCH", MinMode: agent.ModeIssuesOnly}, + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/issues$`), Method: "POST", MinMode: agent.ModeIssuesOnly}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/issues$`), Method: "POST", MinMode: agent.ModeIssuesOnly}, + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/issues/\d+$`), Method: "PATCH", MinMode: agent.ModeIssuesOnly}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/issues/\d+$`), Method: "PUT", MinMode: agent.ModeIssuesOnly}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/issues/\d+$`), Method: "PATCH", MinMode: agent.ModeIssuesOnly}, // Commenting is conversation; creating, editing and relabelling are artifact // production. Bundling them at one tier meant an ADVISORY agent that noticed // something on a thread could not reply — only emit a bead the reporter never // sees (#4492). The tier is unchanged; `converse` is an additional way in. - {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+/comments`), Method: "POST", MinMode: agent.ModeIssuesOnly, Capability: agent.AgentCapabilities.CanConverse}, - {PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/\d+/labels`), Method: "POST", MinMode: agent.ModeIssuesOnly}, - + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/issues/\d+/comments`), Method: "POST", MinMode: agent.ModeIssuesOnly, Capability: agent.AgentCapabilities.CanConverse}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/issues/\d+/notes`), Method: "POST", MinMode: agent.ModeIssuesOnly, Capability: agent.AgentCapabilities.CanConverse}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/issues/\d+/discussions`), Method: "POST", MinMode: agent.ModeIssuesOnly, Capability: agent.AgentCapabilities.CanConverse}, + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/issues/\d+/labels`), Method: "POST", MinMode: agent.ModeIssuesOnly}, + {PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/labels`), Method: "POST", MinMode: agent.ModeIssuesOnly}, + {PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/issues/\d+/labels`), Method: "DELETE", MinMode: agent.ModeIssuesOnly}, // ── Read operations — ADVISORY and above ── // Catch-all: any GET/HEAD/OPTIONS on any path. {PathPattern: regexp.MustCompile(`.*`), Method: "GET", MinMode: agent.ModeAdvisory}, @@ -214,6 +235,7 @@ type denyRule struct { // agent proxy — it originates from the hive process (owner UID), which the // forced-egress iptables redirect exempts — so the relay is unaffected. var denyRules = []denyRule{ + // ── GitHub ── { PathPattern: regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls$`), Method: "POST", @@ -224,6 +246,40 @@ var denyRules = []denyRule{ Method: "PUT", Msg: "direct PR merge is disabled for agents — use `hive-merge --repo --number --expect-sha ` so the hive merges as the App bot with the SHA-pin + merge-eligible binding enforced. Do NOT use `gh api -X PUT .../merge`, `gh pr merge`, or the GitHub MCP merge_pull_request.", }, + + // ── GitLab ── + { + PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests$`), + Method: "POST", + Msg: "direct MR creation is disabled for agents — use the request-file path so the hive opens the MR as the App/bot identity. Do NOT use direct GitLab API calls.", + }, + { + PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/merge$`), + Method: "PUT", + Msg: "direct MR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct GitLab API calls.", + }, + { + PathPattern: regexp.MustCompile(`^(?:/api/v4)?/projects/.+?/merge_requests/\d+/merge$`), + Method: "POST", + Msg: "direct MR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct GitLab API calls.", + }, + + // ── Gitea / Forgejo ── + { + PathPattern: regexp.MustCompile(`^/api/v1/repos/[^/]+/[^/]+/pulls$`), + Method: "POST", + Msg: "direct PR creation is disabled for agents — use the request-file path so the hive opens the PR as the App/bot identity. Do NOT use direct Gitea API calls.", + }, + { + PathPattern: regexp.MustCompile(`^(?:/api/v1)?/repos/[^/]+/[^/]+/pulls/\d+/merge$`), + Method: "POST", + Msg: "direct PR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct Gitea API calls.", + }, + { + PathPattern: regexp.MustCompile(`^/api/v1/repos/[^/]+/[^/]+/pulls/\d+/merge$`), + Method: "PUT", + Msg: "direct PR merge is disabled for agents — use the request-file path so the hive merges as the App/bot identity with bindings enforced. Do NOT use direct Gitea API calls.", + }, } func AllowedByMode(mode agent.AgentMode, method, path string) bool { @@ -277,9 +333,94 @@ func DeniedMessage(method, path string) (string, bool) { return "", false } -var repoPathPrefix = regexp.MustCompile(`^/repos/([^/]+/[^/]+)`) +var repoPathPrefix = regexp.MustCompile(`^(?:/api/v1)?/repos/([^/]+/[^/]+)`) -var gitPathPrefix = regexp.MustCompile(`^/([^/]+/[^/]+)\.git/`) +// gitSmartHTTPEndpoints are the only things that may follow ".git/" for +// gitPathRepo to treat the path as a git smart-HTTP request. Anchoring on the +// remainder keeps a ".git/" that merely appears inside some other API path +// (GitLab /repository/files/foo.git/raw) from yielding a bogus slug. +var gitSmartHTTPEndpoints = map[string]bool{ + "info/refs": true, + "git-upload-pack": true, + "git-receive-pack": true, +} + +// gitPathRepo returns the repo slug of a git smart-HTTP path ("/org/repo.git/ +// git-receive-pack"), or "" if the path is not one. The slug ends at the FIRST +// ".git/" and must be two or more segments, so GitLab nested groups +// ("/group/sub/repo.git/...") are gated on the full project path instead of +// extracting no repo at all. The remainder after ".git/" must be one of the +// smart-HTTP endpoints in gitSmartHTTPEndpoints. +func gitPathRepo(path string) string { + if !strings.HasPrefix(path, "/") { + return "" + } + i := strings.Index(path, ".git/") + if i < 0 { + return "" + } + if !gitSmartHTTPEndpoints[path[i+len(".git/"):]] { + return "" + } + slug := path[1:i] + if slug == "" || !strings.Contains(slug, "/") || strings.HasPrefix(slug, "/") || + strings.HasSuffix(slug, "/") || strings.Contains(slug, "//") { + return "" + } + return slug +} + +// gitLabProjectPathPrefix captures everything after /projects/ in a GitLab API +// path. GitLab addresses a project either by its URL-encoded "namespace%2Fproject" +// path (one segment) or by its numeric id. The proxy's gates hand ExtractRepo +// the request's escaped path, so a well-formed client keeps the project in a +// single %2F segment and is resolved exactly. A decoded (unencoded) spelling +// may span any number of segments (nested groups); ExtractRepo then takes +// segments up to the first project sub-resource in gitLabProjectResources. +var gitLabProjectPathPrefix = regexp.MustCompile(`^(?:/api/v4)?/projects/(.+)$`) + +// gitLabProjectResources are the segments that follow a project path in the +// GitLab REST API (and "-" for its /-/ routes); they end the project path when +// ExtractRepo has to scan a decoded "group/sub/repo/..." spelling. +// +// This is a heuristic with two known failure modes, both bounded: +// +// - A sub-resource missing from this map is absorbed into the repo slug +// ("org/repo/foo"), which matches no allowlist, scope or pause entry, so +// writes fail CLOSED with a misleading slug. Keep the map complete — every +// project route that accepts a write must be here. +// - A namespace or project literally named like a resource word +// ("acme/releases") is cut short. ExtractRepo never returns "" for such a +// path: it falls back to the first two segments so the write is still +// gated against SOME repo rather than skipping every gate. +// +// Neither applies to the %2F-encoded form every GitLab client emits. +var gitLabProjectResources = map[string]bool{ + "-": true, "access_requests": true, "access_tokens": true, "alert_management_alerts": true, + "approval_rules": true, "approval_settings": true, "approvals": true, "archive": true, + "audit_events": true, "avatar": true, "badges": true, "boards": true, "branches": true, + "ci": true, "cluster_agents": true, "clusters": true, "commits": true, "custom_attributes": true, + "debian_distributions": true, "deploy_keys": true, "deploy_tokens": true, "deployments": true, + "discussions": true, "environments": true, "error_tracking": true, "events": true, + "export": true, "feature_flags": true, "feature_flags_user_lists": true, "fork": true, + "forks": true, "freeze_periods": true, "groups": true, "hooks": true, "housekeeping": true, + "import": true, "integrations": true, "invitations": true, "issues": true, + "issues_statistics": true, "iterations": true, "job_token_scope": true, "jobs": true, + "labels": true, "languages": true, "members": true, "merge_requests": true, + "merge_trains": true, "milestones": true, "mirror": true, "notes": true, + "notification_settings": true, "packages": true, "pages": true, "pages_access": true, + "pipeline": true, "pipeline_schedules": true, "pipelines": true, "protected_branches": true, + "protected_environments": true, "protected_tags": true, "push_rule": true, "ref": true, + "registry": true, "releases": true, "remote_mirrors": true, "repository": true, + "repository_size": true, "resource_groups": true, "restore": true, "runners": true, + "search": true, "secure_files": true, "services": true, "share": true, "snapshot": true, + "snippets": true, "star": true, "starrers": true, "statistics": true, "statuses": true, + "storage": true, "tags": true, "templates": true, "transfer": true, "trigger": true, + "triggers": true, "unarchive": true, "unstar": true, "uploads": true, "users": true, + "variables": true, "vulnerabilities": true, "vulnerability_findings": true, "wikis": true, +} + +var gitLabNumericProjectID = regexp.MustCompile(`^\d+$`) // gitUploadPackPath matches the git FETCH endpoint. It is a POST that reads — // the same exception the mode rule table makes for it a few lines up — so the @@ -296,12 +437,88 @@ var writeMethods = map[string]bool{ "DELETE": true, } +// unescapeSlug percent-decodes each "/"-separated segment of a repo slug +// taken from an escaped request path. The gates compare slugs against +// operator config (pause list, agent scope, allowlist), so a client spelling +// "o%72g/repo" must resolve to "org/repo" — the forge decodes it to exactly +// that — rather than to a garbled slug that no deny-list entry matches. A +// segment that fails to decode is kept verbatim. +func unescapeSlug(slug string) string { + segs := strings.Split(slug, "/") + for i, seg := range segs { + if u, err := url.PathUnescape(seg); err == nil { + segs[i] = u + } + } + return strings.Join(segs, "/") +} + +// CanonicalGatePath normalizes an escaped request path (req.URL.EscapedPath()) +// into the one spelling the per-repo gates and ExtractRepo understand. Every +// segment is percent-decoded exactly as the forge will decode it, then only +// the two characters that carry structure are re-escaped: "/" as %2F so a +// GitLab "namespace%2Fproject" stays one segment, and "%" as %25 so a later +// unescapeSlug decodes each segment once, never twice. +// +// Without this, a client could spell a literal prefix with an escape — +// /r%65pos/org/repo, /org/repo%2Egit/git-receive-pack, +// /api/v4/pro%6Aects/org%2Frepo — and Go's URL parser would hand the gates the +// escaped bytes verbatim. ExtractRepo only decodes slug segments, so it would +// return "" and pause, agent scope, repo filter and per-repo ACMM would all +// fail OPEN while the forge decoded the relayed request to the real repo. +// A segment that fails to decode is kept verbatim. +func CanonicalGatePath(escaped string) string { + segs := strings.Split(escaped, "/") + for i, seg := range segs { + u, err := url.PathUnescape(seg) + if err != nil { + continue + } + u = strings.ReplaceAll(u, "%", "%25") + segs[i] = strings.ReplaceAll(u, "/", "%2F") + } + return strings.Join(segs, "/") +} + func ExtractRepo(path string) string { if m := repoPathPrefix.FindStringSubmatch(path); len(m) > 1 { - return m[1] - } - if m := gitPathPrefix.FindStringSubmatch(path); len(m) > 1 { - return m[1] + return unescapeSlug(m[1]) + } + if slug := gitPathRepo(path); slug != "" { + return unescapeSlug(slug) + } + if m := gitLabProjectPathPrefix.FindStringSubmatch(path); len(m) > 1 { + segs := strings.Split(m[1], "/") + // Decode every segment before scanning so "%2F" in the project + // segment, and escaped letters in slug or resource words alike, + // are read the way the forge reads them. + for i, seg := range segs { + if u, err := url.PathUnescape(seg); err == nil { + segs[i] = u + } + } + first := segs[0] + if strings.Contains(first, "/") { + return first + } + if gitLabNumericProjectID.MatchString(first) { + // Numeric project id: the repo slug is not recoverable from the path. + return "" + } + n := 0 + for n < len(segs) && segs[n] != "" && !gitLabProjectResources[segs[n]] { + n++ + } + if n >= 2 { + return strings.Join(segs[:n], "/") + } + // The scan stopped on a resource word before a namespace/project pair + // was complete: the group or project is itself named like a resource. + // Fail closed on the plain two-segment reading rather than return "" + // (which every gate treats as "no repo in path" and lets through). + if len(segs) >= 2 && segs[0] != "" && segs[1] != "" { + return segs[0] + "/" + segs[1] + } } return "" } diff --git a/src/pkg/proxy/rules_test.go b/src/crux/proxy/rules_test.go similarity index 59% rename from src/pkg/proxy/rules_test.go rename to src/crux/proxy/rules_test.go index 1bda72024c..17dd1d7faa 100644 --- a/src/pkg/proxy/rules_test.go +++ b/src/crux/proxy/rules_test.go @@ -1,9 +1,10 @@ package proxy import ( + "net/http" "testing" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) func TestAllowedByMode(t *testing.T) { @@ -118,6 +119,47 @@ func TestExtractRepo(t *testing.T) { {"/repos/org/repo/contents/README.md", "org/repo"}, {"/org/repo.git/git-receive-pack", "org/repo"}, {"/org/repo.git/git-upload-pack", "org/repo"}, + {"/group/sub/repo.git/git-receive-pack", "group/sub/repo"}, + {"/group/sub/sub2/repo.git/git-upload-pack", "group/sub/sub2/repo"}, + {"/group/sub/repo.git/info/refs", "group/sub/repo"}, + {"/repo.git/git-receive-pack", ""}, + // ".git/" that is not followed by a smart-HTTP endpoint is some other + // API path, not a git request: no slug. + {"/api/v4/projects/org%2Frepo/repository/files/foo.git/raw", "org/repo"}, + {"/org/repo.git/", ""}, + {"/org/repo.git/foo", ""}, + {"/org/repo.git/git-receive-pack/extra", ""}, + // Percent-escaped slug letters decode to the slug the forge sees, so + // deny-list gates (pause, agent scope) cannot be evaded by escaping. + {"/repos/o%72g/repo/issues", "org/repo"}, + {"/repos/org/r%65po/pulls/1/merge", "org/repo"}, + {"/o%72g/repo.git/git-receive-pack", "org/repo"}, + {"/api/v4/projects/o%72g/repo/issues", "org/repo"}, + {"/api/v4/projects/o%72g%2Frepo/issues", "org/repo"}, + {"/api/v4/projects/org/repo/iss%75es", "org/repo"}, + {"/api/v1/repos/org/repo/issues", "org/repo"}, + {"/api/v4/projects/org%2Frepo/issues", "org/repo"}, + {"/api/v4/projects/org/repo/issues", "org/repo"}, + {"/api/v4/projects/group/sub/repo/issues", "group/sub/repo"}, + {"/api/v4/projects/group/sub/sub2/repo/merge_requests/3/notes", "group/sub/sub2/repo"}, + {"/api/v4/projects/group%2Fsub%2Frepo/issues", "group/sub/repo"}, + {"/projects/group/sub/repo/repository/branches", "group/sub/repo"}, + {"/api/v4/projects/group/sub/repo", "group/sub/repo"}, + {"/api/v4/projects/group/repo/pipeline", "group/repo"}, + {"/api/v4/projects/group/repo/pipeline_schedules/7/play", "group/repo"}, + {"/api/v4/projects/group/repo/uploads", "group/repo"}, + {"/api/v4/projects/group/repo/-/packages", "group/repo"}, + // Group or project named like a resource word: gated on the plain + // two-segment reading, never "" (which would skip every repo gate). + {"/api/v4/projects/acme/releases/merge_requests", "acme/releases"}, + {"/api/v4/projects/org/issues", "org/issues"}, + // Documented limitation: an unencoded subgroup named like a resource + // word cuts the slug short (the real project is "group/packages/repo"); + // the %2F-encoded spelling resolves exactly. + {"/api/v4/projects/group/packages/repo/issues", "group/packages"}, + {"/api/v4/projects/group%2Fpackages%2Frepo/issues", "group/packages/repo"}, + {"/api/v4/projects/org/", ""}, + {"/api/v4/projects/123/issues", ""}, {"/user", ""}, {"/rate_limit", ""}, {"/search/issues", ""}, @@ -134,10 +176,86 @@ func TestExtractRepo(t *testing.T) { } } +// TestCanonicalGatePath covers the escaped-structural-segment seam: the gates +// receive the client's escaped path so %2F keeps a GitLab project in one +// segment, which means every OTHER escape must be decoded first or the literal +// prefixes ExtractRepo anchors on (/repos/, .git/, /projects/) can be hidden +// and the write slips every per-repo gate. +func TestCanonicalGatePath(t *testing.T) { + tests := []struct { + escaped string + want string + wantRepo string + }{ + {"/repos/org/repo/issues", "/repos/org/repo/issues", "org/repo"}, + {"/r%65pos/org/repo/issues", "/repos/org/repo/issues", "org/repo"}, + {"/org/repo%2Egit/git-receive-pack", "/org/repo.git/git-receive-pack", "org/repo"}, + {"/org/repo.git/git-receive-p%61ck", "/org/repo.git/git-receive-pack", "org/repo"}, + {"/api/v4/pro%6Aects/org%2Frepo/issues", "/api/v4/projects/org%2Frepo/issues", "org/repo"}, + {"/api/v4/projects/group%2Fsub%2Frepo/issues", "/api/v4/projects/group%2Fsub%2Frepo/issues", "group/sub/repo"}, + {"/api/v4/projects/o%72g%2Frepo/iss%75es", "/api/v4/projects/org%2Frepo/issues", "org/repo"}, + {"/repos/org/repo/pulls/1/merg%65", "/repos/org/repo/pulls/1/merge", "org/repo"}, + // A doubly-escaped slash decodes once, as the forge decodes it: the + // segment is the literal "org%2Frepo", not a namespace/project pair, + // and ExtractRepo's resource-word fallback yields a slug that matches + // no real repo — the write fails CLOSED rather than skipping the gates. + {"/api/v4/projects/org%252Frepo/issues", "/api/v4/projects/org%252Frepo/issues", "org%2Frepo/issues"}, + // Undecodable segments are kept verbatim. + {"/repos/org/re%zzpo/issues", "/repos/org/re%zzpo/issues", "org/re%zzpo"}, + {"/graphql", "/graphql", ""}, + } + for _, tt := range tests { + t.Run(tt.escaped, func(t *testing.T) { + got := CanonicalGatePath(tt.escaped) + if got != tt.want { + t.Errorf("CanonicalGatePath(%q) = %q, want %q", tt.escaped, got, tt.want) + } + if repo := ExtractRepo(got); repo != tt.wantRepo { + t.Errorf("ExtractRepo(CanonicalGatePath(%q)) = %q, want %q", tt.escaped, repo, tt.wantRepo) + } + }) + } +} + +// TestRepoGatesEscapedStructuralSegments is the end-to-end assertion that an +// escaped prefix cannot turn a gated write into an ungated one. +func TestRepoGatesEscapedStructuralSegments(t *testing.T) { + paused := func(repo string) bool { return repo == "org/repo" } + serves := func(agent, repo string) bool { return repo == "other/repo" } + allowed := map[string]bool{"other/repo": true} + autoMerge := func(repo string) bool { return repo != "org/repo" } + + for _, escaped := range []string{ + "/r%65pos/org/repo/issues", + "/org/repo%2Egit/git-receive-pack", + "/api/v4/pro%6Aects/org%2Frepo/issues", + "/api/v4/projects/o%72g%2Frepo/merge_requests", + } { + t.Run(escaped, func(t *testing.T) { + path := CanonicalGatePath(escaped) + if _, refused := RepoPauseRefusal(paused, "POST", path); !refused { + t.Errorf("RepoPauseRefusal let %q through", escaped) + } + if _, refused := AgentRepoScopeRefusal(serves, "reviewer", "POST", path); !refused { + t.Errorf("AgentRepoScopeRefusal let %q through", escaped) + } + if RepoFilterAllowed(allowed, "POST", path) { + t.Errorf("RepoFilterAllowed let %q through", escaped) + } + }) + } + + path := CanonicalGatePath("/repos/org/repo/pulls/1/merg%65") + if _, refused := RepoAutoMergeRefusal(autoMerge, http.MethodPut, path); !refused { + t.Errorf("RepoAutoMergeRefusal let escaped merge path through") + } +} + func TestRepoFilterAllowed(t *testing.T) { allowed := map[string]bool{ - "myorg/console": true, - "myorg/docs": true, + "myorg/console": true, + "myorg/docs": true, + "myorg/sub/console": true, } tests := []struct { @@ -158,6 +276,8 @@ func TestRepoFilterAllowed(t *testing.T) { {"POST user endpoint passes", "POST", "/user", true}, {"git push allowed repo passes", "POST", "/myorg/console.git/git-receive-pack", true}, {"git push disallowed repo blocked", "POST", "/myorg/other.git/git-receive-pack", false}, + {"nested-group git push disallowed blocked", "POST", "/myorg/sub/other.git/git-receive-pack", false}, + {"nested-group git push allowed passes", "POST", "/myorg/sub/console.git/git-receive-pack", true}, } for _, tt := range tests { diff --git a/src/pkg/proxy/somark_linux.go b/src/crux/proxy/somark_linux.go similarity index 100% rename from src/pkg/proxy/somark_linux.go rename to src/crux/proxy/somark_linux.go diff --git a/src/pkg/proxy/somark_other.go b/src/crux/proxy/somark_other.go similarity index 100% rename from src/pkg/proxy/somark_other.go rename to src/crux/proxy/somark_other.go diff --git a/src/pkg/proxy/somark_test.go b/src/crux/proxy/somark_test.go similarity index 100% rename from src/pkg/proxy/somark_test.go rename to src/crux/proxy/somark_test.go diff --git a/src/pkg/proxy/stream_write_stall_test.go b/src/crux/proxy/stream_write_stall_test.go similarity index 100% rename from src/pkg/proxy/stream_write_stall_test.go rename to src/crux/proxy/stream_write_stall_test.go diff --git a/src/pkg/proxy/tunnel_drain_test.go b/src/crux/proxy/tunnel_drain_test.go similarity index 100% rename from src/pkg/proxy/tunnel_drain_test.go rename to src/crux/proxy/tunnel_drain_test.go diff --git a/src/pkg/proxy/upstream_timeout_test.go b/src/crux/proxy/upstream_timeout_test.go similarity index 100% rename from src/pkg/proxy/upstream_timeout_test.go rename to src/crux/proxy/upstream_timeout_test.go diff --git a/src/pkg/proxy/write_counters.go b/src/crux/proxy/write_counters.go similarity index 100% rename from src/pkg/proxy/write_counters.go rename to src/crux/proxy/write_counters.go diff --git a/src/pkg/proxy/write_counters_store.go b/src/crux/proxy/write_counters_store.go similarity index 96% rename from src/pkg/proxy/write_counters_store.go rename to src/crux/proxy/write_counters_store.go index c0d4fad9d6..ba9ee43337 100644 --- a/src/pkg/proxy/write_counters_store.go +++ b/src/crux/proxy/write_counters_store.go @@ -181,6 +181,8 @@ func (d DurableWriteCounts) empty() bool { return len(d.Totals) == 0 && len(d.Weeks) == 0 } +var errInvalidWriteCounts = errors.New("invalid write counters JSON") + // LoadDurableWriteCounts reads the file at path. A missing file is an empty // result, not an error. func LoadDurableWriteCounts(path string) (DurableWriteCounts, error) { @@ -193,7 +195,7 @@ func LoadDurableWriteCounts(path string) (DurableWriteCounts, error) { return d, err } if err := json.Unmarshal(data, &d); err != nil { - return newDurableWriteCounts(), fmt.Errorf("parse %s: %w", path, err) + return newDurableWriteCounts(), fmt.Errorf("parse %s: %w: %w", path, errInvalidWriteCounts, err) } return d, nil } @@ -220,7 +222,10 @@ func (c *writeCounters) flush(path string, now time.Time) error { func mergeIntoFile(path string, pending DurableWriteCounts, now time.Time) error { cur, err := LoadDurableWriteCounts(path) if err != nil { - // A corrupt file must not stall persistence forever; start over. + // Only corrupt JSON can safely be replaced; preserve unreadable files. + if !errors.Is(err, errInvalidWriteCounts) { + return err + } cur = newDurableWriteCounts() } cur.merge(pending) diff --git a/src/pkg/proxy/write_counters_store_test.go b/src/crux/proxy/write_counters_store_test.go similarity index 74% rename from src/pkg/proxy/write_counters_store_test.go rename to src/crux/proxy/write_counters_store_test.go index 5374325d1c..bec4831883 100644 --- a/src/pkg/proxy/write_counters_store_test.go +++ b/src/crux/proxy/write_counters_store_test.go @@ -4,6 +4,7 @@ import ( "errors" "os" "path/filepath" + "syscall" "testing" "time" ) @@ -88,6 +89,59 @@ func TestWriteCountersFlushFailureKeepsPending(t *testing.T) { } } +func TestWriteCountersReadFailureKeepsFileAndPending(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "counters.json") + now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) + var c writeCounters + c.now = func() time.Time { return now } + c.record("ADVISORY", "") + if err := c.flush(path, now); err != nil { + t.Fatal(err) + } + original, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := os.Rename(path, path+".saved"); err != nil { + t.Fatal(err) + } + // A symlink loop produces a deterministic read error even when CI runs as root. + if err := os.Symlink(filepath.Base(path), path); err != nil { + t.Fatal(err) + } + c.record("ADVISORY", DenyReasonMode) + if err := c.flush(path, now); !errors.Is(err, syscall.ELOOP) { + t.Fatalf("flush error = %v, want ELOOP", err) + } + if info, err := os.Lstat(path); err != nil || info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("unreadable path was replaced: %v, %v", info, err) + } + saved, err := os.ReadFile(path + ".saved") + if err != nil || string(saved) != string(original) { + t.Fatalf("durable data changed: %v", err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := os.Rename(path+".saved", path); err != nil { + t.Fatal(err) + } + if err := c.flush(path, now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + got, err := LoadDurableWriteCounts(path) + if err != nil { + t.Fatal(err) + } + if wc := got.Totals["ADVISORY"]; wc.Attempted != 2 || wc.Blocked != 1 { + t.Errorf("retry totals = %+v", wc) + } + if !got.RecordedSince.Equal(time.Date(2026, 10, 12, 0, 0, 0, 0, time.UTC)) { + t.Errorf("recorded_since changed: %s", got.RecordedSince) + } +} + func TestLoadDurableWriteCountsMissingAndCorrupt(t *testing.T) { dir := t.TempDir() if d, err := LoadDurableWriteCounts(filepath.Join(dir, "none.json")); err != nil || !d.empty() { @@ -95,8 +149,8 @@ func TestLoadDurableWriteCountsMissingAndCorrupt(t *testing.T) { } bad := filepath.Join(dir, "bad.json") _ = os.WriteFile(bad, []byte("{"), 0o644) - if _, err := LoadDurableWriteCounts(bad); err == nil { - t.Error("expected parse error") + if _, err := LoadDurableWriteCounts(bad); !errors.Is(err, errInvalidWriteCounts) { + t.Errorf("expected parse error, got %v", err) } var c writeCounters c.record("ADVISORY", "") diff --git a/src/pkg/proxy/write_counters_test.go b/src/crux/proxy/write_counters_test.go similarity index 100% rename from src/pkg/proxy/write_counters_test.go rename to src/crux/proxy/write_counters_test.go diff --git a/src/pkg/proxy/write_leak_gate_test.go b/src/crux/proxy/write_leak_gate_test.go similarity index 100% rename from src/pkg/proxy/write_leak_gate_test.go rename to src/crux/proxy/write_leak_gate_test.go diff --git a/src/pkg/proxy/write_surface_enforce.go b/src/crux/proxy/write_surface_enforce.go similarity index 100% rename from src/pkg/proxy/write_surface_enforce.go rename to src/crux/proxy/write_surface_enforce.go diff --git a/src/pkg/proxy/write_surface_enforce_test.go b/src/crux/proxy/write_surface_enforce_test.go similarity index 99% rename from src/pkg/proxy/write_surface_enforce_test.go rename to src/crux/proxy/write_surface_enforce_test.go index 22ccc0230e..a977478f68 100644 --- a/src/pkg/proxy/write_surface_enforce_test.go +++ b/src/crux/proxy/write_surface_enforce_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/agent" + "github.com/hivecommons/hive/crux/agent" ) // enforceOnly is the shape config.WriteSurfaceEnforced has for a hive that diff --git a/src/pkg/pushbroker/pushbroker.go b/src/crux/pushbroker/pushbroker.go similarity index 96% rename from src/pkg/pushbroker/pushbroker.go rename to src/crux/pushbroker/pushbroker.go index fa2cc74bd2..b9ae5cedd0 100644 --- a/src/pkg/pushbroker/pushbroker.go +++ b/src/crux/pushbroker/pushbroker.go @@ -16,10 +16,9 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/effects" - ghpkg "github.com/hivecommons/hive/pkg/github" - "github.com/hivecommons/hive/pkg/gitidentity" - "github.com/hivecommons/hive/pkg/logscrub" + "github.com/hivecommons/hive/crux/effects" + "github.com/hivecommons/hive/crux/gitidentity" + "github.com/hivecommons/hive/crux/logscrub" ) const ( @@ -243,8 +242,20 @@ func (ExecRunner) Run(ctx context.Context, dir string, env []string, name string return cmd.CombinedOutput() } +// TokenScoper mints an installation token narrowed to a permission tier and, +// when non-empty, to the named repositories. It is the one method the broker +// needs from the host's GitHub App credential (*github.AppAuth in hive), kept +// as an interface so this package stays inside the Crux ring. +type TokenScoper interface { + ScopedTokenForRepos(ctx context.Context, tier string, repos []string) (string, error) +} + +// GitHubAppMinter mints a per-push token through the host's TokenScoper. +// Auth must be a non-nil interface holding a non-nil implementation; hosts +// building it from a possibly-nil pointer should leave Auth nil instead of +// storing a typed nil (see cmd/hive's appMinter). type GitHubAppMinter struct { - Auth *ghpkg.AppAuth + Auth TokenScoper Tier string } @@ -293,8 +304,8 @@ type Result struct { FinishedAt time.Time `json:"finished_at"` } -func (b *Broker) Run(ctx context.Context) (Result, error) { - res := Result{Workspace: b.Workspace, Repo: b.Repo, Branch: b.Branch, Remote: b.remote(), StartedAt: b.now()} +func (b *Broker) Run(ctx context.Context) (res Result, err error) { + res = Result{Workspace: b.Workspace, Repo: b.Repo, Branch: b.Branch, Remote: b.remote(), StartedAt: b.now()} defer func() { res.FinishedAt = b.now() }() if err := b.validate(); err != nil { res.Error = err.Error() diff --git a/src/pkg/pushbroker/pushbroker_coverage_test.go b/src/crux/pushbroker/pushbroker_coverage_test.go similarity index 82% rename from src/pkg/pushbroker/pushbroker_coverage_test.go rename to src/crux/pushbroker/pushbroker_coverage_test.go index 93357b11bf..5c80a85d5d 100644 --- a/src/pkg/pushbroker/pushbroker_coverage_test.go +++ b/src/crux/pushbroker/pushbroker_coverage_test.go @@ -2,23 +2,14 @@ package pushbroker import ( "context" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "encoding/json" - "encoding/pem" "errors" "log/slog" - "net/http" - "net/http/httptest" "os" "path/filepath" "slices" "strings" "testing" "time" - - ghpkg "github.com/hivecommons/hive/pkg/github" ) // scriptedGit answers git invocations from a table keyed on the joined argv, @@ -130,6 +121,7 @@ func TestValidateRejectsUnusableConfigurations(t *testing.T) { // remote-ref default or the broker diffs the wrong range and can miss a // secret or a protected path introduced earlier in the branch. func TestRunPrefersAnExplicitBaseRef(t *testing.T) { + fixed := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) git := &scriptedGit{replies: map[string]string{ "rev-parse HEAD": "cafebabe\n", "rev-parse --verify origin/main": "deadbeef\n", @@ -139,6 +131,7 @@ func TestRunPrefersAnExplicitBaseRef(t *testing.T) { res, err := (&Broker{ Workspace: fakeGitWorkspace(t), Branch: "work", BaseRef: "origin/main", Repo: "hivecommons/hive", Minter: fakeMinter{"ghs_tok"}, Runner: git, + Now: func() time.Time { return fixed }, }).Run(context.Background()) if err != nil { t.Fatalf("Run: %v (res=%+v)", err, res) @@ -149,6 +142,9 @@ func TestRunPrefersAnExplicitBaseRef(t *testing.T) { if slices.Contains(git.calls, "diff --name-only -z refs/remotes/origin/work...HEAD") { t.Fatalf("broker diffed against the remote ref despite an explicit BaseRef: %v", git.calls) } + if !res.StartedAt.Equal(fixed) || !res.FinishedAt.Equal(fixed) { + t.Fatalf("timestamps = %v/%v, want the injected clock %v", res.StartedAt, res.FinishedAt, fixed) + } } // With no BaseRef and no tracked remote branch (the first push of a new @@ -377,76 +373,57 @@ func TestGitHubAppMinterRequiresAuth(t *testing.T) { } } +// scopeRecorder is the TokenScoper the minter tests drive: it records what +// the broker asked for so the test can assert the scope, not GitHub's reply. +type scopeRecorder struct { + tier string + repos []string + token string +} + +func (r *scopeRecorder) ScopedTokenForRepos(_ context.Context, tier string, repos []string) (string, error) { + r.tier, r.repos = tier, repos + return r.token, nil +} + // The point of minting per push is that the credential cannot reach any other // repository the installation covers, and that an unspecified tier lands on the // least-privileged default rather than on whatever the installation grants. func TestGitHubAppMinterScopesTokenToTheSingleRepo(t *testing.T) { - var body map[string]any - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !strings.HasSuffix(r.URL.Path, "/access_tokens") { - w.WriteHeader(http.StatusNotFound) - return - } - _ = json.NewDecoder(r.Body).Decode(&body) - w.WriteHeader(http.StatusCreated) - _, _ = w.Write([]byte(`{"token":"ghs_scoped","expires_at":"2099-01-01T00:00:00Z"}`)) - })) - defer srv.Close() - - auth, err := ghpkg.NewAppAuthFromPEM(1234, 5678, testRSAKeyPEM(t), slog.New(slog.DiscardHandler), srv.URL) - if err != nil { - t.Fatalf("NewAppAuthFromPEM: %v", err) - } - token, err := (GitHubAppMinter{Auth: auth}).MintPushToken(context.Background(), " hivecommons/hive ") + rec := &scopeRecorder{token: "ghs_scoped"} + token, err := (GitHubAppMinter{Auth: rec}).MintPushToken(context.Background(), " hivecommons/hive ") if err != nil { t.Fatalf("MintPushToken: %v", err) } if token != "ghs_scoped" { t.Fatalf("token = %q, want the minted token", token) } - repos, _ := body["repositories"].([]any) - if len(repos) != 1 || repos[0] != "hive" { - t.Fatalf("repositories = %v, want exactly [hive]", body["repositories"]) - } - // DefaultTier is "contributor": issues/contents/pull_requests write, no merge rights. - perms, _ := body["permissions"].(map[string]any) - if perms["contents"] != "write" || perms["pull_requests"] != "write" { - t.Fatalf("permissions = %v, want the contributor tier", perms) + if len(rec.repos) != 1 || rec.repos[0] != "hive" { + t.Fatalf("repositories = %v, want exactly [hive]", rec.repos) } - if _, hasChecks := perms["checks"]; hasChecks { - t.Fatalf("default tier requested merger-shaped permissions: %v", perms) + if rec.tier != DefaultTier { + t.Fatalf("tier = %q, want the default %q", rec.tier, DefaultTier) } } +// The read-only tier must reach the scoper by name so the host maps it to +// read permissions; the repo narrowing is the same as for the default tier. +// (The permission table itself lives in the root module's github package and +// is covered there.) func TestGitHubAppMinterReadOnlyTierScopesRepoAndPermissions(t *testing.T) { - var body map[string]any - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _ = json.NewDecoder(r.Body).Decode(&body) - w.WriteHeader(http.StatusCreated) - _, _ = w.Write([]byte(`{"token":"ghs_readonly","expires_at":"2099-01-01T00:00:00Z"}`)) - })) - defer srv.Close() - - auth, err := ghpkg.NewAppAuthFromPEM(1234, 5678, testRSAKeyPEM(t), slog.New(slog.DiscardHandler), srv.URL) - if err != nil { - t.Fatalf("NewAppAuthFromPEM: %v", err) - } - token, err := (GitHubAppMinter{Auth: auth, Tier: ReadOnlyTier}).MintPushToken(context.Background(), "hivecommons/hive") + rec := &scopeRecorder{token: "ghs_readonly"} + token, err := (GitHubAppMinter{Auth: rec, Tier: ReadOnlyTier}).MintPushToken(context.Background(), "hivecommons/hive") if err != nil { t.Fatalf("MintPushToken: %v", err) } if token != "ghs_readonly" { t.Fatalf("token = %q, want the minted token", token) } - repos, _ := body["repositories"].([]any) - if len(repos) != 1 || repos[0] != "hive" { - t.Fatalf("repositories = %v, want exactly [hive]", body["repositories"]) + if len(rec.repos) != 1 || rec.repos[0] != "hive" { + t.Fatalf("repositories = %v, want exactly [hive]", rec.repos) } - perms, _ := body["permissions"].(map[string]any) - for _, key := range []string{"contents", "issues", "metadata", "pull_requests"} { - if perms[key] != "read" { - t.Fatalf("permissions = %v, want read for %s", perms, key) - } + if rec.tier != ReadOnlyTier { + t.Fatalf("tier = %q, want %q", rec.tier, ReadOnlyTier) } } @@ -454,26 +431,15 @@ func TestGitHubAppMinterReadOnlyTierScopesRepoAndPermissions(t *testing.T) { // minter must leave the scope unrestricted rather than send a bogus name that // GitHub would reject. func TestGitHubAppMinterLeavesScopeOpenForAnUnqualifiedRepo(t *testing.T) { - var body map[string]any - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _ = json.NewDecoder(r.Body).Decode(&body) - w.WriteHeader(http.StatusCreated) - _, _ = w.Write([]byte(`{"token":"ghs_open","expires_at":"2099-01-01T00:00:00Z"}`)) - })) - defer srv.Close() - - auth, err := ghpkg.NewAppAuthFromPEM(1234, 5678, testRSAKeyPEM(t), slog.New(slog.DiscardHandler), srv.URL) - if err != nil { - t.Fatal(err) - } - if _, err := (GitHubAppMinter{Auth: auth, Tier: "newcomer"}).MintPushToken(context.Background(), "hive"); err != nil { + rec := &scopeRecorder{token: "ghs_open"} + if _, err := (GitHubAppMinter{Auth: rec, Tier: "newcomer"}).MintPushToken(context.Background(), "hive"); err != nil { t.Fatalf("MintPushToken: %v", err) } - if _, ok := body["repositories"]; ok { - t.Fatalf("repositories = %v, want the field omitted", body["repositories"]) + if rec.repos != nil { + t.Fatalf("repositories = %v, want the field omitted", rec.repos) } - if perms, _ := body["permissions"].(map[string]any); perms["contents"] != "read" { - t.Fatalf("permissions = %v, want the newcomer tier", body["permissions"]) + if rec.tier != "newcomer" { + t.Fatalf("tier = %q, want the explicit newcomer tier", rec.tier) } } @@ -502,12 +468,3 @@ func mergeErrs(maps ...map[string]error) map[string]error { type errMinter struct{ err error } func (m errMinter) MintPushToken(context.Context, string) (string, error) { return "", m.err } - -func testRSAKeyPEM(t *testing.T) []byte { - t.Helper() - key, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - t.Fatalf("GenerateKey: %v", err) - } - return pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) -} diff --git a/src/pkg/pushbroker/pushbroker_test.go b/src/crux/pushbroker/pushbroker_test.go similarity index 100% rename from src/pkg/pushbroker/pushbroker_test.go rename to src/crux/pushbroker/pushbroker_test.go diff --git a/src/pkg/agent/ringbuffer.go b/src/crux/ringbuffer.go similarity index 98% rename from src/pkg/agent/ringbuffer.go rename to src/crux/ringbuffer.go index 83a9e14c1c..dde6d3ffb3 100644 --- a/src/pkg/agent/ringbuffer.go +++ b/src/crux/ringbuffer.go @@ -1,4 +1,4 @@ -package agent +package crux import ( "sync" diff --git a/src/pkg/agent/ringbuffer_test.go b/src/crux/ringbuffer_test.go similarity index 99% rename from src/pkg/agent/ringbuffer_test.go rename to src/crux/ringbuffer_test.go index 69ede45118..30a8521743 100644 --- a/src/pkg/agent/ringbuffer_test.go +++ b/src/crux/ringbuffer_test.go @@ -1,4 +1,4 @@ -package agent +package crux import ( "fmt" diff --git a/src/pkg/crux/runtime.go b/src/crux/runtime.go similarity index 50% rename from src/pkg/crux/runtime.go rename to src/crux/runtime.go index 4285f31673..8d326051e0 100644 --- a/src/pkg/crux/runtime.go +++ b/src/crux/runtime.go @@ -1,123 +1,54 @@ // Package crux is the exported surface of Hive's agent runtime // (hivecommons/hive#13536, src/docs/design/crux.md). // -// pkg/agent is the runtime's implementation: the manager, kick files, engine +// crux/agent is the runtime's implementation: the manager, kick files, engine // adapters and the tmux funnel. Consumers outside that tree (the dashboard, // the hub, cmd/hive) are meant to program against the types in this package -// rather than against *agent.Manager, so that the runtime can move to its own -// module (Stage 2 of the design record) without every consumer following its -// internals. The architecture ratchet in +// rather than against *agent.Manager, so that they do not follow the +// runtime's internals. The architecture ratchet in // internal/testutil/crux_boundary_ratchet_test.go counts the agent. // references each consumer still makes; this package is how that number goes // down. // -// Stage 1 keeps this a façade: Process is an alias of agent.AgentProcess, so -// the adapter is zero-cost and nothing about the runtime's behaviour changes. -// Every other value type on this surface (process states, kick dispatch and -// outcome, kick logs, restart events, sub-agent models, start-failure classes, -// turn loss, the provider-status vocabulary) is DEFINED in the leaf package -// pkg/crux/types and aliased both here and in pkg/agent: Crux owns them and -// the runtime implementation re-exports them. The leaf exists because this -// package imports pkg/agent for Process and so cannot be imported by it. Each -// interface method is annotated with the consumer that needs it today; a -// method with no consumer does not belong here. +// This package does not import crux/agent: crux/agent imports it. Process is +// its own type, a snapshot the manager copies out of its internal agent +// record, and ProjectContext and RingBuffer are defined here and aliased by +// crux/agent, so *agent.Manager implements Runtime directly (agent.AsRuntime +// adapts a possibly-nil manager). Every other value type on this surface +// (process states, kick dispatch and outcome, kick logs, restart events, +// sub-agent models, start-failure classes, turn loss, the provider-status +// vocabulary, watchdog conditions) is defined here too and aliased by +// crux/agent and crux/watchdog. Each interface method is annotated with the +// consumer that needs it today; a method with no consumer does not belong +// here. package crux import ( "context" + "time" - "github.com/hivecommons/hive/pkg/agent" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/crux/types" -) - -// Process is the observable snapshot of one agent: state, pins, overrides, -// kick history, pane-derived conditions. It is what Observer.GetStatus and -// Observer.AllStatuses return. -type Process = agent.AgentProcess - -// ProjectContext is the host-derived project facts (org, repos, policy dir, -// pause state) a runtime is built with. It is on the Crux surface so a host -// such as cmd/crux can build one without naming pkg/agent. -type ProjectContext = agent.ProjectContext - -// ProcessState is the runtime's intent for an agent (running, stopped, -// paused, …), as distinct from what its pane is observed to be doing. -type ProcessState = types.ProcessState - -// Process states. -const ( - StateIdle = types.StateIdle - StateRunning = types.StateRunning - StateStopped = types.StateStopped - StateFailed = types.StateFailed - StatePaused = types.StatePaused -) - -// KickRecord is one entry of Process.KickHistory. -type KickRecord = types.KickRecord - -// KickDispatch is the observable outcome of one asynchronous kick; see -// Kicker.SendKickAsync and Kicker.KickDispatchState. -type KickDispatch = types.KickDispatch - -// Kick dispatch phases. A dispatch is indeterminate while pending; only -// KickPhaseFailed is a definitive failure. -const ( - KickPhasePending = types.KickPhasePending - KickPhaseDelivered = types.KickPhaseDelivered - KickPhaseFailed = types.KickPhaseFailed -) - -// KickOutcome is how an agent's most recent kicked turn ended. -type KickOutcome = types.KickOutcome + "github.com/hivecommons/hive/crux/trajectory" -// Kick outcome kinds. -const ( - KickOutcomeQuestion = types.KickOutcomeQuestion - KickOutcomeStandDown = types.KickOutcomeStandDown - KickOutcomeNoOp = types.KickOutcomeNoOp - KickOutcomeEnded = types.KickOutcomeEnded + "github.com/hivecommons/hive/crux/config" ) -// KickLogInfo describes one archived kick log; see Kicker.ListKickLogs. -type KickLogInfo = types.KickLogInfo - -// RestartEvent is one entry of Process.RestartEvents. -type RestartEvent = types.RestartEvent - -// SubAgentModel is one observed Copilot sub-agent row; see -// Process.SubAgentModels. -type SubAgentModel = types.SubAgentModel - -// StartFailureClass classifies why an agent failed to start; see -// Process.StartFailureClass. -type StartFailureClass = types.StartFailureClass - -// Start failure classes. -const ( - StartFailureLoginRequired = types.StartFailureLoginRequired - StartFailureCredentialRejected = types.StartFailureCredentialRejected - StartFailureCredentialMissing = types.StartFailureCredentialMissing - StartFailureBackendMismatch = types.StartFailureBackendMismatch - StartFailureBinaryMissing = types.StartFailureBinaryMissing - StartFailureNoOutput = types.StartFailureNoOutput -) - -// TurnInterruption is one teardown that killed a turn in flight; see -// Process.TurnLoss.Recent. -type TurnInterruption = types.TurnInterruption - -// TurnLoss is what teardowns have discarded from an agent's in-flight turns; -// see Process.TurnLoss. cmd/hive: state persistence. -type TurnLoss = types.TurnLoss - // FatalNetworkErrorPatterns returns the pane substrings the runtime treats // as a fatal startup network failure and restarts the agent on. Hosts that // compose prompts echoed into the pane assert their text never contains one // (pkg/review's prompt builders; #7553). The slice is a copy. func FatalNetworkErrorPatterns() []string { - return append([]string(nil), agent.FatalNetworkErrorPatterns...) + return append([]string(nil), fatalNetworkErrorPatterns...) +} + +// fatalNetworkErrorPatterns are substrings that indicate a transient TLS or +// network failure killed the agent at startup. These errors leave the Copilot +// chrome visible (❯, / commands) so the pane looks ready, but the agent is +// dead and will never recover without a restart. crux/agent's +// FatalNetworkErrorPatterns is initialised from this list. +var fatalNetworkErrorPatterns = []string{ + "invalid peer certificate", + "BadSignature", + "fetch failed", } // Lifecycle starts, stops, pauses and restarts agents. @@ -288,9 +219,140 @@ type Credentials interface { OmpAgentHomes() []string } +// Resumer is backend-native session resume: naming the conversation that +// authored a PR and typing a follow-up into it later (hivecommons/hive#9606). +// Errors are the crux.ErrResume* sentinels. +type Resumer interface { + // ResumeHandle captures the agent's current resumable conversation. + // cmd/hive: prfollowupwire.go (PR-opened hook). + ResumeHandle(name string) (ResumeHandle, bool) + // SendResumeKick types message into the session sessionID names, or + // returns ErrResumeBusy / ErrResumeSessionGone / ErrResumeUnsupported. + // cmd/hive: prfollowupwire.go. + SendResumeKick(name, message, sessionID string) error + // SessionID is the agent's live session id, if it has one. cmd/hive: + // prfollowupwire.go. + SessionID(name string) (string, bool) + // SessionMissing reports an agent whose session is gone (dead pane). + // cmd/hive: eval cycle resume-kick hold. + SessionMissing(name string) bool +} + +// Supervisor is what the governor asks the runtime while scheduling work: +// may it start an agent, what is each agent launched as, which agents have +// crashed, which credentials can be refreshed. These reads and actions sit +// between Lifecycle and the host's scheduler. +type Supervisor interface { + // GovernorMayStart reports whether the runtime will accept a Start for + // name right now (not blocked by a start-failure hold). cmd/hive. + GovernorMayStart(name string) bool + // MarkStartupLaunchQueued records that the host has queued the named + // agents for its staggered boot launch. cmd/hive: boot. + MarkStartupLaunchQueued(names []string) + // CheckAndRestartCrashedAgents relaunches crashed agents and returns + // their names. cmd/hive: run loop. + CheckAndRestartCrashedAgents(ctx context.Context) []string + // ConsentWedgedAgents lists agents stuck on a consent prompt. cmd/hive. + ConsentWedgedAgents() []string + // RefreshAgentTokens rewrites every agent's scoped GitHub token. + // cmd/hive: watchers, GitHub client refresh. + RefreshAgentTokens(ctx context.Context) + // AgentCredentialRefreshable reports whether the host can refresh the + // agent's credential. cmd/hive: heartbeat. + AgentCredentialRefreshable(agentName string) (refreshable, known bool) + // AgentCapabilities is the agent's GitHub capability bits. cmd/hive. + AgentCapabilities(agentName string) (canOpenIssue, canOpenPR, canMerge, ok bool) + // EffectiveBackend is the backend the agent runs on after overrides. + // cmd/hive: rotation. + EffectiveBackend(agentName string) (backend string, ok bool) + // InvocationMetadata is what the agent was launched as. cmd/hive: + // attribution trailers. + InvocationMetadata(agentName string) (backend, model, effort string, ok bool) + // RestartTelemetry is the agent's restart counters. cmd/hive: heartbeat. + RestartTelemetry(name string) (total, last24h int, lastRestartAt time.Time, lastReason string, ok bool) + // StartFailureState is the agent's start-failure record. cmd/hive. + StartFailureState(name string) (StartFailureSnapshot, bool) + // ProviderErrorBackoffRemaining reports an active provider-error backoff: + // remaining time, provider, reason. cmd/hive: eval cycle. + ProviderErrorBackoffRemaining(name string) (time.Duration, string, string, bool) + // PauseByCause is PauseBy with the machine-readable causation recorded. + // cmd/hive: fleet breaker, escalations. + PauseByCause(name, trigger, reason, by string, cause PauseCausation) error + // ArchiveAllKickLogs rotates every agent's kick log. cmd/hive: mode + // change. + ArchiveAllKickLogs(reason string) + // CountAgentsWithModel reports how many agents launched with an explicit + // model, for the hourly spoke metrics. cmd/hive. + CountAgentsWithModel() int + // CrashOOMSuspected reports whether the agent's last crash looks like an + // OOM kill, so a held resume kick is explained correctly. cmd/hive. + CrashOOMSuspected(name string) bool + // BackendAuthAvailable and AgentAuthAvailable report whether credentials + // for a backend / an agent are present; known is false when the runtime + // cannot tell. The dashboard's auth badges read them. cmd/hive. + BackendAuthAvailable(backend string) (available, known bool) + AgentAuthAvailable(agentName string) (available, known bool) + // SendGitHubAPIBudgetNudge tells an agent its GitHub API budget is + // nearly spent; a nudge, not a kick, so it never counts against the kick + // limits. cmd/hive: egress proxy. + SendGitHubAPIBudgetNudge(name, message string) + // RecordAudit appends a host-side action against an agent to the + // runtime's audit trail. cmd/hive. + RecordAudit(action, agentName string, fields map[string]any) + // AgentHasValidCredential and RefreshAgentTokenFor back the login-prompt + // scanner: an agent sitting on a login prompt with a valid credential is + // re-minted rather than paused. cmd/hive. + AgentHasValidCredential(agentName string) bool + RefreshAgentTokenFor(ctx context.Context, name string) error + // TrajectoryAgents snapshots the running agents with intent and + // transcript for the trajectory lane. cmd/hive. + TrajectoryAgents() []trajectory.AgentView +} + +// Restorer seeds the runtime with state the host persisted across a restart +// (pkg/snapshot). Each call replaces the record for one agent and is only +// meaningful before that agent runs. +type Restorer interface { + SeedPauseState(name string, pausedAt time.Time, trigger, reason, by string) + SeedRestartTelemetry(name string, count int, events []RestartEvent, lastReason string) + SeedTurnLoss(name string, loss TurnLoss) + SeedLastKick(name string, t time.Time) + SeedKickHistory(name string, records []KickRecord) + // RestoreBreaker re-engages the fleet breaker as it was persisted. + RestoreBreaker(engaged bool, paused []string) +} + +// Background are the runtime's long-running maintenance loops. Each blocks +// until ctx ends; the host runs them as goroutines after boot. In a sidecar +// they run inside the serving process and this interface is a no-op. +type Background interface { + StartAgentTokenRefresh(ctx context.Context) + StartCredentialWatchdog(ctx context.Context) + StartCopilotSessionRefresh(ctx context.Context) + StartTranscriptActivityRefresh(ctx context.Context) +} + +// Authorizer is the per-request gate the host's relay watchers consult +// before acting on an agent's request (open a PR, open an issue, request a +// review, merge, push a branch): is the request from the agent it claims +// (fileUID), and does that agent's ACMM level permit the action. The five +// are method values handed to the relays at boot, so they are the one +// family a sidecar must answer synchronously on the request path. +type Authorizer interface { + AuthorizePROpen(agentName string, fileUID int) error + AuthorizePushBranch(agentName string, fileUID int) error + AuthorizeIssueOpen(agentName string, fileUID int, kind string) error + AuthorizeMerge(agentName string, fileUID int, repo string) error + AuthorizeReviewRequest(agentName string, fileUID int) error +} + // Runtime is the full surface a consumer needs to run, observe and control -// agents. *agent.Manager implements it directly. Consumers that need less -// should depend on the narrower interfaces above. +// agents. *agent.Manager implements it directly (asserted in crux/agent). +// Consumers that need less should depend on the narrower interfaces above. +// +// Resumer, Supervisor, Restorer, Background and Authorizer are the families cmd/hive +// uses beyond the dashboard's six (hivecommons/hive#14138, stage 1); a +// sidecar runtime serves each over the API before the default flips. type Runtime interface { Lifecycle Observer @@ -298,15 +360,9 @@ type Runtime interface { Definitions Fleet Credentials -} - -var _ Runtime = (*agent.Manager)(nil) - -// FromManager adapts a manager to Runtime, returning a nil interface for a -// nil manager so consumers' `runtime != nil` checks keep their meaning. -func FromManager(m *agent.Manager) Runtime { - if m == nil { - return nil - } - return m + Resumer + Supervisor + Restorer + Background + Authorizer } diff --git a/src/crux/runtime/quiet_internal_test.go b/src/crux/runtime/quiet_internal_test.go new file mode 100644 index 0000000000..ead23cc60f --- /dev/null +++ b/src/crux/runtime/quiet_internal_test.go @@ -0,0 +1,8 @@ +package runtime + +import ( + "io" + "log/slog" +) + +func quietLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } diff --git a/src/crux/runtime/runtime.go b/src/crux/runtime/runtime.go new file mode 100644 index 0000000000..688df8c80c --- /dev/null +++ b/src/crux/runtime/runtime.go @@ -0,0 +1,259 @@ +// Package runtime constructs a crux.Runtime for a host (hivecommons/hive#14138, +// stage 1 of the sidecar plan). It is the one place outside crux/agent that +// names *agent.Manager on the construction path: New picks the implementation +// from config.RuntimeConfig.Mode and returns it as the interface, so cmd/hive +// and the dashboard program against crux.Runtime whichever mode is selected. +// +// Two modes exist. ModeInProcess (the default) links crux/agent and returns +// the manager, exactly as cmd/hive constructed it before the seam existed; no +// behaviour changes. ModeSidecar (stage 3) returns a crux/api/client.Runtime +// that drives a `crux serve` process at RuntimeConfig.Server over the Crux +// API; the host does not link crux/agent's machinery for it. +// +// The package also re-exports the two process-level entry points cmd/hive +// uses that are not methods of a runtime instance: RunAgyTurn (the +// `hive agy-turn` subprocess) and StartPermissionsWatcher. Hive-provided +// capabilities (crux.AppTokenMinter, crux.AgentMintIssuer, +// crux.LinearCredential, crux.PRCreator and the rest of crux.Host) are +// installed after construction through HostOf, which only the in-process +// implementation satisfies; a sidecar host serves them back over the +// callback socket (crux/api/callback, stage 2). +package runtime + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "strings" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" + "github.com/hivecommons/hive/crux/api/client" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/watchdog" +) + +// ErrModeUnavailable is returned by New for a mode that is recognised but +// has no implementation in this build. Every mode config.Mode accepts has +// one today; the sentinel stays so a host can keep distinguishing "unknown +// mode" from "not built here". +var ErrModeUnavailable = errors.New("crux runtime mode not yet available") + +// ErrNotInProcess is returned by helpers that need the in-process manager +// (HostOf) or a runtime with a watchdog seam (NewWatchdogFleet) when given +// another implementation. +var ErrNotInProcess = errors.New("crux runtime is not the in-process implementation") + +// Option configures New. +type Option func(*options) + +type options struct { + defs map[string]config.AgentConfig + logger *slog.Logger + project crux.ProjectContext + uidMapPath *string +} + +// WithDefinitions supplies the agent definitions the runtime manages. The +// default is none; definitions can also be added later through +// crux.Definitions. +func WithDefinitions(defs map[string]config.AgentConfig) Option { + return func(o *options) { o.defs = defs } +} + +// WithLogger sets the runtime's logger. The default is slog.Default(). +func WithLogger(logger *slog.Logger) Option { + return func(o *options) { o.logger = logger } +} + +// WithProject supplies the project facts (ACMM level, org, repos) the runtime +// gates launches on. +func WithProject(project crux.ProjectContext) Option { + return func(o *options) { o.project = project } +} + +// WithUIDMapPath points the in-process runtime at a specific persisted UID +// map. Production keeps the default; tests and the crux CLI inject a path. +func WithUIDMapPath(path string) Option { + return func(o *options) { o.uidMapPath = &path } +} + +// New validates cfg and returns the runtime its Mode selects. Options are +// applied in order; a nil option is ignored. +func New(cfg config.RuntimeConfig, opts ...Option) (crux.Runtime, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + o := options{logger: slog.Default()} + for _, opt := range opts { + if opt != nil { + opt(&o) + } + } + switch cfg.Mode { + case config.ModeInProcess: + return newInProcess(o), nil + case config.ModeSidecar: + return newSidecar(cfg, o) + } + return nil, fmt.Errorf("%w: %q", ErrModeUnavailable, cfg.Mode) +} + +// newSidecar is the sidecar implementation: a crux/api/client.Runtime over +// the `crux serve` process at cfg.Server. Nothing is dialed here; the first +// call does, so a host can construct the runtime before the sidecar is up +// and wait on it with client.Runtime.Ready. +func newSidecar(cfg config.RuntimeConfig, o options) (crux.Runtime, error) { + rt, err := client.New(cfg.Server, cfg.Token) + if err != nil { + return nil, fmt.Errorf("%s=%s: %w", config.ModeEnvVar, cfg.Mode, err) + } + rt.Logger = o.logger + return rt, nil +} + +// newInProcess is the in-process implementation: *agent.Manager itself, +// which satisfies crux.Runtime and crux.Host (asserted in crux/agent). +func newInProcess(o options) crux.Runtime { + var mopts []agent.ManagerOption + if o.uidMapPath != nil { + mopts = append(mopts, agent.WithUIDMapPath(*o.uidMapPath)) + } + return agent.AsRuntime(agent.NewManagerWithOptions(o.defs, o.logger, o.project, mopts...)) +} + +// HostOf returns rt's crux.Host when rt is the in-process implementation. +// A sidecar runtime has no Host: its capabilities arrive over the API +// (stage 2 and 3), so callers treat ok == false as "nothing to wire". +func HostOf(rt crux.Runtime) (crux.Host, bool) { + h, ok := rt.(crux.Host) + return h, ok && h != nil +} + +// WatchdogHooks are the governor-side facts the fleet watchdog needs and the +// runtime cannot know: queue depth, deliberate kick suppression and the +// agent's cadence period. A nil hook means "unknown" to the reconciler. +type WatchdogHooks struct { + Queued func() (int, bool) + // QueuedRefs names the items behind Queued's count (at most + // watchdog.MaxQueuedRefs are used), so a not-producing alert can link + // them (#11224). Nil names nothing. + QueuedRefs func() []watchdog.QueuedRef + KickSuppressed func(name string) string + CadencePeriod func(name string) (time.Duration, bool) +} + +// WatchdogSourcer is the seam a runtime exposes for the fleet watchdog: +// the in-process manager answers from its agent table, the sidecar client +// from the server's watchdog routes. +type WatchdogSourcer interface { + WatchdogSource() watchdog.Source +} + +// NewWatchdogFleet adapts rt to watchdog.Fleet for the self-healing +// reconciler (RFC #4665). The in-process manager gets agent.WatchdogFleet, +// which reads the agent table under the manager's lock; any other runtime +// with a WatchdogSource (the sidecar client) gets a fleet composed of that +// source and the runtime's Lifecycle. A runtime with neither returns +// ErrNotInProcess. +func NewWatchdogFleet(rt crux.Runtime, hooks WatchdogHooks) (watchdog.Fleet, error) { + if m, ok := rt.(*agent.Manager); ok && m != nil { + return agent.WatchdogFleet{ + M: m, + Queued: hooks.Queued, + QueuedRefs: hooks.QueuedRefs, + KickSuppressed: hooks.KickSuppressed, + CadencePeriod: hooks.CadencePeriod, + }, nil + } + ws, ok := rt.(WatchdogSourcer) + if !ok || rt == nil { + return nil, ErrNotInProcess + } + src := ws.WatchdogSource() + if src == nil { + return nil, ErrNotInProcess + } + return sourceFleet{Source: src, rt: rt, hooks: hooks}, nil +} + +// sourceFleet is watchdog.Fleet over a watchdog.Source plus the runtime's +// Lifecycle, for runtimes that are not the in-process manager. +type sourceFleet struct { + watchdog.Source + rt crux.Runtime + hooks WatchdogHooks +} + +func (f sourceFleet) IsPaused(name string) bool { return f.rt.IsPaused(name) } + +func (f sourceFleet) Restart(ctx context.Context, name string) error { + return f.rt.Restart(ctx, name) +} + +func (f sourceFleet) Pause(name, trigger, reason string) error { + return f.rt.Pause(name, trigger, reason) +} + +// QueuedWork mirrors agent.WatchdogFleet: advisory-tier agents report an +// empty queue because they cannot drain it, read off the agent's config. +func (f sourceFleet) QueuedWork(name string) (int, bool) { + if f.hooks.Queued == nil { + return 0, false + } + p, err := f.rt.GetStatusFast(name) + if err != nil || p == nil { + return 0, false + } + if advisoryOnly(p.Config) { + return 0, true + } + return f.hooks.Queued() +} + +// QueuedWorkRefs implements watchdog.QueuedWorkLister; the queue is the +// governor's hive-wide one, so hiveWide is always true. +func (f sourceFleet) QueuedWorkRefs(string) ([]watchdog.QueuedRef, bool) { + if f.hooks.QueuedRefs == nil { + return nil, true + } + return f.hooks.QueuedRefs(), true +} + +func (f sourceFleet) KickSuppressedReason(name string) string { + if f.hooks.KickSuppressed == nil { + return "" + } + return f.hooks.KickSuppressed(name) +} + +func (f sourceFleet) KickPeriod(name string) (time.Duration, bool) { + if f.hooks.CadencePeriod == nil { + return 0, false + } + return f.hooks.CadencePeriod(name) +} + +func advisoryOnly(cfg config.AgentConfig) bool { + const advisoryMode = "ADVISORY" + if strings.EqualFold(strings.TrimSpace(cfg.Mode), advisoryMode) { + return true + } + return cfg.Tools.EffectiveMode() == advisoryMode +} + +// RunAgyTurn implements the `hive agy-turn` subprocess: one headless agy +// turn, exit code returned. It is agent.RunAgyTurn. +func RunAgyTurn(args []string, stdout, stderr io.Writer) int { + return agent.RunAgyTurn(args, stdout, stderr) +} + +// StartPermissionsWatcher runs the agent home-directory permissions watcher +// for the life of the process. It is agent.StartPermissionsWatcher; the +// host starts it as a goroutine. +func StartPermissionsWatcher(logger *slog.Logger) { + agent.StartPermissionsWatcher(logger) +} diff --git a/src/crux/runtime/runtime_test.go b/src/crux/runtime/runtime_test.go new file mode 100644 index 0000000000..6703ce48ce --- /dev/null +++ b/src/crux/runtime/runtime_test.go @@ -0,0 +1,241 @@ +package runtime_test + +import ( + "bytes" + "context" + "errors" + "io" + "log/slog" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api/client" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/cruxtest" + cruxruntime "github.com/hivecommons/hive/crux/runtime" + "github.com/hivecommons/hive/crux/serve" + "github.com/hivecommons/hive/crux/watchdog" +) + +func quietLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } + +func newInProcess(t *testing.T, defs map[string]config.AgentConfig) crux.Runtime { + t.Helper() + t.Setenv("HIVE_WORK_DIR", t.TempDir()) + rt, err := cruxruntime.New(config.RuntimeConfig{}, + cruxruntime.WithDefinitions(defs), + cruxruntime.WithLogger(quietLogger()), + cruxruntime.WithProject(crux.ProjectContext{ACMMLevel: 2}), + cruxruntime.WithUIDMapPath(filepath.Join(t.TempDir(), "uid-map.json")), + nil, + ) + if err != nil { + t.Fatal(err) + } + return rt +} + +func TestNewDefaultsToInProcess(t *testing.T) { + rt := newInProcess(t, map[string]config.AgentConfig{ + "fixture": {ID: "fixture-id", Enabled: true, Backend: "claude", Model: "sonnet"}, + }) + proc, err := rt.GetStatusFast("fixture") + if err != nil { + t.Fatal(err) + } + if proc.State != crux.StateStopped || proc.Config.Model != "sonnet" { + t.Fatalf("state/model = %s/%s, want stopped/sonnet", proc.State, proc.Config.Model) + } + host, ok := cruxruntime.HostOf(rt) + if !ok || host == nil { + t.Fatal("in-process runtime must expose crux.Host") + } + // Host wiring is accepted and takes effect through the Runtime surface. + host.SetKickLogSourceResolver(func() string { return config.KickLogSourceTranscript }) + if _, isTest := rt.(cruxtest.Runtime); !isTest { + t.Fatal("in-process runtime should also satisfy the cruxtest fixture surface") + } + if got := rt.(cruxtest.Runtime).KickLogSource(); got != config.KickLogSourceTranscript { + t.Fatalf("KickLogSource = %q after Host wiring, want transcript", got) + } +} + +func TestNewWithoutOptionsIsUsable(t *testing.T) { + t.Setenv("HIVE_WORK_DIR", t.TempDir()) + rt, err := cruxruntime.New(config.RuntimeConfig{Mode: config.ModeInProcess}) + if err != nil { + t.Fatal(err) + } + if n := len(rt.AllStatuses()); n != 0 { + t.Fatalf("empty runtime has %d agents", n) + } +} + +// TestNewSidecarDrivesAServer builds the sidecar runtime against a serve +// handler over a Recorder and checks calls land there, the Host seam is +// absent, and the watchdog fleet composes from the wire source. +func TestNewSidecarDrivesAServer(t *testing.T) { + rec := cruxtest.NewRecorder() + rec.Add("fixture", "fixture-id", crux.StateRunning) + rec.WatchdogNames = []string{"fixture"} + srv := serve.New(rec) + srv.Token = "tok" + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(ts.Close) + + rt, err := cruxruntime.New(config.RuntimeConfig{Mode: config.ModeSidecar, Server: ts.URL, Token: "tok"}, cruxruntime.WithLogger(quietLogger())) + if err != nil { + t.Fatal(err) + } + if _, ok := rt.(*client.Runtime); !ok { + t.Fatalf("New(sidecar) = %T, want *client.Runtime", rt) + } + if h, ok := cruxruntime.HostOf(rt); ok || h != nil { + t.Fatalf("HostOf(sidecar) = %v, %v; want nil, false", h, ok) + } + if err := rt.Stop("fixture"); err != nil { + t.Fatal(err) + } + if _, ok := rec.Last("Stop"); !ok { + t.Fatal("Stop did not reach the server") + } + if n := len(rt.AllStatuses()); n != 1 { + t.Fatalf("AllStatuses over the wire = %d agents, want 1", n) + } + + queued := 0 + fleet, err := cruxruntime.NewWatchdogFleet(rt, cruxruntime.WatchdogHooks{ + Queued: func() (int, bool) { queued++; return 3, true }, + KickSuppressed: func(string) string { return "quiet" }, + CadencePeriod: func(string) (time.Duration, bool) { return time.Hour, true }, + }) + if err != nil { + t.Fatal(err) + } + if got := fleet.AgentNames(); len(got) != 1 || got[0] != "fixture" { + t.Fatalf("AgentNames = %v", got) + } + if fleet.IsPaused("fixture") { + t.Fatal("fresh agent reported paused") + } + if err := fleet.Pause("fixture", "watchdog", "test"); err != nil { + t.Fatal(err) + } + if !fleet.IsPaused("fixture") { + t.Fatal("pause over the wire not reflected") + } + if err := fleet.Restart(context.Background(), "fixture"); err != nil { + t.Fatal(err) + } + if _, ok := rec.Last("Restart"); !ok { + t.Fatal("Restart did not reach the server") + } + if n, ok := fleet.QueuedWork("fixture"); !ok || n != 3 || queued != 1 { + t.Fatalf("QueuedWork = %d, %v (hook calls %d)", n, ok, queued) + } + if n, ok := fleet.QueuedWork("ghost"); ok || n != 0 { + t.Fatalf("QueuedWork(unknown) = %d, %v; want 0, false", n, ok) + } + rec.Procs["fixture"].Config.Mode = "advisory" + if n, ok := fleet.QueuedWork("fixture"); !ok || n != 0 || queued != 1 { + t.Fatalf("QueuedWork(advisory) = %d, %v (hook calls %d); want 0, true, 1", n, ok, queued) + } + if got := fleet.(watchdog.KickSuppressionReporter).KickSuppressedReason("fixture"); got != "quiet" { + t.Fatalf("KickSuppressedReason = %q", got) + } + if p, ok := fleet.(watchdog.KickPeriodReporter).KickPeriod("fixture"); !ok || p != time.Hour { + t.Fatalf("KickPeriod = %v, %v", p, ok) + } + bare, err := cruxruntime.NewWatchdogFleet(rt, cruxruntime.WatchdogHooks{}) + if err != nil { + t.Fatal(err) + } + if _, ok := bare.QueuedWork("fixture"); ok { + t.Fatal("QueuedWork without a hook must be unknown") + } + if got := bare.(watchdog.KickSuppressionReporter).KickSuppressedReason("fixture"); got != "" { + t.Fatalf("KickSuppressedReason without a hook = %q", got) + } + if _, ok := bare.(watchdog.KickPeriodReporter).KickPeriod("fixture"); ok { + t.Fatal("KickPeriod without a hook must be unknown") + } +} + +func TestNewSidecarRejectsBadServer(t *testing.T) { + rt, err := cruxruntime.New(config.RuntimeConfig{Mode: config.ModeSidecar, Server: "://nope"}) + if rt != nil || err == nil || errors.Is(err, cruxruntime.ErrModeUnavailable) { + t.Fatalf("New(sidecar, bad server) = %v, %v; want an address error", rt, err) + } + if !strings.Contains(err.Error(), config.ModeEnvVar) { + t.Errorf("error %q should mention %s", err, config.ModeEnvVar) + } +} + +// nilSource has the watchdog seam but nothing behind it. +type nilSource struct{ crux.Runtime } + +func (nilSource) WatchdogSource() watchdog.Source { return nil } + +func TestNewWatchdogFleetNilSource(t *testing.T) { + if _, err := cruxruntime.NewWatchdogFleet(nilSource{}, cruxruntime.WatchdogHooks{}); !errors.Is(err, cruxruntime.ErrNotInProcess) { + t.Fatalf("NewWatchdogFleet(nil source) err = %v, want ErrNotInProcess", err) + } +} + +func TestNewRejectsUnknownMode(t *testing.T) { + rt, err := cruxruntime.New(config.RuntimeConfig{Mode: "remote"}) + if rt != nil || err == nil || errors.Is(err, cruxruntime.ErrModeUnavailable) { + t.Fatalf("New(remote) = %v, %v; want a validation error, not ErrModeUnavailable", rt, err) + } +} + +type notInProcess struct{ crux.Runtime } + +func TestHostOfOtherImplementations(t *testing.T) { + if h, ok := cruxruntime.HostOf(notInProcess{}); ok || h != nil { + t.Fatalf("HostOf(non-manager) = %v, %v; want nil, false", h, ok) + } + if h, ok := cruxruntime.HostOf(nil); ok || h != nil { + t.Fatalf("HostOf(nil) = %v, %v; want nil, false", h, ok) + } +} + +func TestNewWatchdogFleet(t *testing.T) { + rt := newInProcess(t, map[string]config.AgentConfig{ + "fixture": {ID: "fixture-id", Enabled: true, Backend: "claude"}, + }) + var queuedCalls int + fleet, err := cruxruntime.NewWatchdogFleet(rt, cruxruntime.WatchdogHooks{ + Queued: func() (int, bool) { queuedCalls++; return 3, true }, + KickSuppressed: func(string) string { return "" }, + CadencePeriod: func(string) (time.Duration, bool) { return time.Hour, true }, + }) + if err != nil { + t.Fatal(err) + } + // A stopped agent is not reconciled (the fleet scopes to running ones). + if names := fleet.AgentNames(); len(names) != 0 { + t.Fatalf("AgentNames = %v for a stopped fleet, want none", names) + } + if fleet.IsPaused("fixture") { + t.Fatal("fresh agent reported paused") + } + if _, err := cruxruntime.NewWatchdogFleet(notInProcess{}, cruxruntime.WatchdogHooks{}); !errors.Is(err, cruxruntime.ErrNotInProcess) { + t.Fatalf("NewWatchdogFleet(non-manager) err = %v, want ErrNotInProcess", err) + } + if _, err := cruxruntime.NewWatchdogFleet(nil, cruxruntime.WatchdogHooks{}); !errors.Is(err, cruxruntime.ErrNotInProcess) { + t.Fatalf("NewWatchdogFleet(nil) err = %v, want ErrNotInProcess", err) + } + _ = queuedCalls +} + +func TestRunAgyTurnHelp(t *testing.T) { + var stdout, stderr bytes.Buffer + if code := cruxruntime.RunAgyTurn([]string{"-h"}, &stdout, &stderr); code != 0 { + t.Fatalf("RunAgyTurn(-h) = %d, stderr %q", code, stderr.String()) + } +} diff --git a/src/crux/runtime/supervise.go b/src/crux/runtime/supervise.go new file mode 100644 index 0000000000..117a5c3064 --- /dev/null +++ b/src/crux/runtime/supervise.go @@ -0,0 +1,256 @@ +package runtime + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "os/exec" + "sync" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" +) + +// Supervisor keeps a sidecar's `crux serve` process alive +// (hivecommons/hive#14138 stage 4, config.SuperviseChild): it spawns the +// binary, waits for the health route, and when the process exits restarts +// it after a backoff that doubles up to MaxBackoff and resets once a child +// has stayed healthy for ResetAfter. The host's client.Runtime reconnects +// on its own; the supervisor only guarantees there is something to connect +// to. Under config.SuperviseExternal nothing here runs. +type Supervisor struct { + // Config is the validated sidecar config: Server says what to listen + // on, Binary what to run, Token what to require. + Config config.RuntimeConfig + // Args replaces the default argument list (`serve` plus the listen + // flags derived from Config.Server) when non-nil. + Args []string + // Env is appended to the child's environment (the parent's is + // inherited; CRUX_TOKEN is added from Config.Token when set). + Env []string + // Stdout and Stderr receive the child's output; nil means the + // parent's. + Stdout, Stderr io.Writer + // Logger receives lifecycle events; nil means slog.Default(). + Logger *slog.Logger + // MinBackoff, MaxBackoff and ResetAfter tune the restart loop; zero + // means the defaults. + MinBackoff, MaxBackoff, ResetAfter time.Duration + // ReadyTimeout bounds the wait for a freshly started child's health + // route; zero means DefaultReadyTimeout. A child that never answers + // is killed and counted as a failed start. + ReadyTimeout time.Duration + + // lookPath and start are the exec seams tests replace. + lookPath func(string) (string, error) + start func(*exec.Cmd) error + + mu sync.Mutex + state SupervisorState +} + +// Supervisor defaults. +const ( + DefaultMinBackoff = time.Second + DefaultMaxBackoff = time.Minute + DefaultResetAfter = 5 * time.Minute + DefaultReadyTimeout = 30 * time.Second + readyPoll = 100 * time.Millisecond +) + +// SupervisorState is what the supervisor knows about its child. +type SupervisorState struct { + // Running reports whether a child process exists right now. + Running bool + // PID is the current child's pid (0 when none). + PID int + // Ready reports whether the current child has answered its health + // route. + Ready bool + // Starts counts child processes started so far; Restarts is Starts-1 + // floored at zero. + Starts int + // LastExit is the most recent exit error ("" when none or clean). + LastExit string + // LastExitAt is when the last child exited. + LastExitAt time.Time + // NextStart is when the next start is due during a backoff. + NextStart time.Time + // Binary is the resolved executable path. + Binary string +} + +// ErrBinaryNotFound is returned by Run when the `crux` executable cannot +// be found; the host then falls back to reconnect-only and says so. +var ErrBinaryNotFound = errors.New("crux binary not found") + +// State is the current SupervisorState. +func (s *Supervisor) State() SupervisorState { + s.mu.Lock() + defer s.mu.Unlock() + return s.state +} + +func (s *Supervisor) logger() *slog.Logger { + if s.Logger == nil { + return slog.Default() + } + return s.Logger +} + +func (s *Supervisor) defaults() { + if s.MinBackoff <= 0 { + s.MinBackoff = DefaultMinBackoff + } + if s.MaxBackoff < s.MinBackoff { + s.MaxBackoff = DefaultMaxBackoff + if s.MaxBackoff < s.MinBackoff { + s.MaxBackoff = s.MinBackoff + } + } + if s.ResetAfter <= 0 { + s.ResetAfter = DefaultResetAfter + } + if s.ReadyTimeout <= 0 { + s.ReadyTimeout = DefaultReadyTimeout + } + if s.lookPath == nil { + s.lookPath = exec.LookPath + } + if s.start == nil { + s.start = (*exec.Cmd).Start + } +} + +// ServeArgs derives the `crux serve` argument list for a server address: +// a unix address becomes --socket, a TCP one --listen. +func ServeArgs(server string) ([]string, error) { + addr, err := api.ParseAddress(server) + if err != nil { + return nil, err + } + if addr.Network == "unix" { + return []string{"serve", "--socket", addr.Path}, nil + } + return []string{"serve", "--listen", addr.Host}, nil +} + +// Run supervises until ctx ends. It returns ErrBinaryNotFound (wrapped) +// at once when the binary is missing, otherwise nil when ctx ends; the +// child is signalled to stop with the context. +func (s *Supervisor) Run(ctx context.Context) error { + s.defaults() + bin, err := s.lookPath(s.Config.Binary) + if err != nil { + return fmt.Errorf("%w: %s: %v", ErrBinaryNotFound, s.Config.Binary, err) + } + args := s.Args + if args == nil { + if args, err = ServeArgs(s.Config.Server); err != nil { + return err + } + } + probe, err := api.NewClient(s.Config.Server, s.Config.Token) + if err != nil { + return err + } + s.mu.Lock() + s.state.Binary = bin + s.mu.Unlock() + backoff := s.MinBackoff + for { + startedAt := time.Now() + exitErr := s.runOnce(ctx, bin, args, probe) + if ctx.Err() != nil { + return nil + } + if time.Since(startedAt) >= s.ResetAfter { + backoff = s.MinBackoff + } + s.mu.Lock() + s.state.NextStart = time.Now().Add(backoff) + s.mu.Unlock() + s.logger().Warn("crux serve exited; restarting", "err", exitErr, "backoff", backoff, "starts", s.State().Starts) + select { + case <-ctx.Done(): + return nil + case <-time.After(backoff): + } + backoff *= 2 + if backoff > s.MaxBackoff { + backoff = s.MaxBackoff + } + } +} + +// runOnce starts one child, waits for it to be ready, then waits for it to +// exit; it returns the exit error (nil for a clean exit). +func (s *Supervisor) runOnce(ctx context.Context, bin string, args []string, probe *api.Client) error { + cctx, cancel := context.WithCancel(ctx) + defer cancel() + cmd := exec.CommandContext(cctx, bin, args...) + cmd.Env = append(os.Environ(), s.Env...) + if s.Config.Token != "" { + cmd.Env = append(cmd.Env, config.ServeTokenEnvVar+"="+s.Config.Token) + } + cmd.Stdout, cmd.Stderr = s.Stdout, s.Stderr + if cmd.Stdout == nil { + cmd.Stdout = os.Stdout + } + if cmd.Stderr == nil { + cmd.Stderr = os.Stderr + } + cmd.WaitDelay = 5 * time.Second + if err := s.start(cmd); err != nil { + s.noteExit(err) + return err + } + s.mu.Lock() + s.state.Running, s.state.Ready, s.state.PID, s.state.NextStart = true, false, cmd.Process.Pid, time.Time{} + s.state.Starts++ + s.mu.Unlock() + s.logger().Info("crux serve started", "pid", cmd.Process.Pid, "binary", bin) + + exited := make(chan error, 1) + go func() { exited <- cmd.Wait() }() + + rctx, rcancel := context.WithTimeout(cctx, s.ReadyTimeout) + ready := make(chan error, 1) + go func() { ready <- probe.WaitReady(rctx, readyPoll) }() + select { + case err := <-exited: + rcancel() + <-ready + s.noteExit(err) + return err + case err := <-ready: + rcancel() + if err != nil && cctx.Err() == nil { + s.logger().Error("crux serve never became ready; killing", "pid", cmd.Process.Pid, "err", err) + cancel() + s.noteExit(<-exited) + return fmt.Errorf("crux serve not ready within %s: %w", s.ReadyTimeout, err) + } + s.mu.Lock() + s.state.Ready = true + s.mu.Unlock() + } + err := <-exited + s.noteExit(err) + return err +} + +func (s *Supervisor) noteExit(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.state.Running, s.state.Ready, s.state.PID = false, false, 0 + s.state.LastExitAt = time.Now() + s.state.LastExit = "" + if err != nil { + s.state.LastExit = err.Error() + } +} diff --git a/src/crux/runtime/supervise_test.go b/src/crux/runtime/supervise_test.go new file mode 100644 index 0000000000..82c68fae3d --- /dev/null +++ b/src/crux/runtime/supervise_test.go @@ -0,0 +1,174 @@ +package runtime + +import ( + "context" + "errors" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/cruxtest" + "github.com/hivecommons/hive/crux/serve" +) + +// waitFor polls cond until it holds or the test deadline nears. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.After(10 * time.Second) + tick := time.NewTicker(10 * time.Millisecond) + defer tick.Stop() + for !cond() { + select { + case <-deadline: + t.Fatalf("timed out waiting for %s", what) + case <-tick.C: + } + } +} + +// fakeCrux writes a script that ignores its arguments and sleeps so the +// supervisor sees a long-lived child. +func fakeCrux(t *testing.T) string { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("shell fixture") + } + p := filepath.Join(t.TempDir(), "crux") + if err := os.WriteFile(p, []byte("#!/bin/sh\nexec sleep 60\n"), 0o755); err != nil { + t.Fatal(err) + } + return p +} + +func TestServeArgs(t *testing.T) { + got, err := ServeArgs("unix:///run/crux.sock") + if err != nil || strings.Join(got, " ") != "serve --socket /run/crux.sock" { + t.Fatalf("unix: %v %v", got, err) + } + got, err = ServeArgs("127.0.0.1:9000") + if err != nil || strings.Join(got, " ") != "serve --listen 127.0.0.1:9000" { + t.Fatalf("tcp: %v %v", got, err) + } + if _, err := ServeArgs(""); err == nil { + t.Fatal("empty address must fail") + } +} + +func TestSupervisorBinaryNotFound(t *testing.T) { + s := &Supervisor{Config: config.RuntimeConfig{Server: "127.0.0.1:1", Binary: filepath.Join(t.TempDir(), "nope")}, Logger: quietLogger()} + err := s.Run(context.Background()) + if !errors.Is(err, ErrBinaryNotFound) { + t.Fatalf("Run = %v, want ErrBinaryNotFound", err) + } + s = &Supervisor{Config: config.RuntimeConfig{Server: "", Binary: fakeCrux(t)}, Logger: quietLogger()} + if err := s.Run(context.Background()); err == nil || errors.Is(err, ErrBinaryNotFound) { + t.Fatalf("empty server = %v, want address error", err) + } +} + +func TestSupervisorRestartsAfterExit(t *testing.T) { + rec := cruxtest.NewRecorder() + srv := serve.New(rec) + srv.Token = "tok" + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(ts.Close) + + s := &Supervisor{ + Config: config.RuntimeConfig{Server: ts.URL, Token: "tok", Binary: fakeCrux(t)}, + Logger: quietLogger(), + MinBackoff: 10 * time.Millisecond, + MaxBackoff: 20 * time.Millisecond, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- s.Run(ctx) }() + + waitFor(t, "first child ready", func() bool { st := s.State(); return st.Ready && st.Running && st.PID > 0 }) + first := s.State() + if first.Starts != 1 || first.Binary == "" { + t.Fatalf("state after first start = %+v", first) + } + if err := syscall.Kill(first.PID, syscall.SIGKILL); err != nil { + t.Fatal(err) + } + waitFor(t, "restart", func() bool { st := s.State(); return st.Starts == 2 && st.Ready }) + second := s.State() + if second.PID == first.PID || second.LastExit == "" || second.LastExitAt.IsZero() { + t.Fatalf("state after restart = %+v (first pid %d)", second, first.PID) + } + cancel() + if err := <-done; err != nil { + t.Fatalf("Run after cancel = %v", err) + } + waitFor(t, "child reaped", func() bool { return !s.State().Running }) +} + +func TestSupervisorKillsChildThatNeverBecomesReady(t *testing.T) { + // Nothing listens on the configured address, so readiness must time out + // and the child be killed and restarted. + s := &Supervisor{ + Config: config.RuntimeConfig{Server: "127.0.0.1:1", Binary: fakeCrux(t)}, + Logger: quietLogger(), + MinBackoff: 5 * time.Millisecond, + MaxBackoff: 10 * time.Millisecond, + ReadyTimeout: 50 * time.Millisecond, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- s.Run(ctx) }() + waitFor(t, "two failed starts", func() bool { return s.State().Starts >= 2 }) + if st := s.State(); st.Ready || st.LastExit == "" { + t.Fatalf("state = %+v", st) + } + cancel() + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestSupervisorStartFailureBacksOff(t *testing.T) { + var calls atomic.Int32 + s := &Supervisor{ + Config: config.RuntimeConfig{Server: "127.0.0.1:1", Binary: fakeCrux(t)}, + Logger: quietLogger(), + MinBackoff: 5 * time.Millisecond, + MaxBackoff: 10 * time.Millisecond, + start: func(*exec.Cmd) error { + calls.Add(1) + return errors.New("boom") + }, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- s.Run(ctx) }() + waitFor(t, "retries", func() bool { return calls.Load() >= 3 }) + if st := s.State(); st.LastExit != "boom" || st.Starts != 0 || st.NextStart.IsZero() { + t.Fatalf("state = %+v", st) + } + cancel() + <-done +} + +func TestSupervisorDefaults(t *testing.T) { + var s Supervisor + s.defaults() + if s.MinBackoff != DefaultMinBackoff || s.MaxBackoff != DefaultMaxBackoff || s.ResetAfter != DefaultResetAfter || s.ReadyTimeout != DefaultReadyTimeout { + t.Fatalf("defaults = %s %s %s %s", s.MinBackoff, s.MaxBackoff, s.ResetAfter, s.ReadyTimeout) + } + s = Supervisor{MinBackoff: 2 * time.Hour} + s.defaults() + if s.MaxBackoff != 2*time.Hour { + t.Fatalf("max below min must clamp, got %s", s.MaxBackoff) + } + if s.logger() == nil { + t.Fatal("nil logger") + } +} diff --git a/src/crux/runtime_test.go b/src/crux/runtime_test.go new file mode 100644 index 0000000000..91f04a3ce5 --- /dev/null +++ b/src/crux/runtime_test.go @@ -0,0 +1,79 @@ +package crux_test + +import ( + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agent" +) + +// TestValueAliasesTrackAgent pins that the façade's constants are the ones +// the manager writes (crux defines them, agent aliases them), so a consumer +// comparing crux.StateRunning against a Process.State can never drift. +func TestValueAliasesTrackAgent(t *testing.T) { + if crux.StateRunning != agent.StateRunning || crux.StatePaused != agent.StatePaused || crux.StateStopped != agent.StateStopped { + t.Fatal("process state constants diverged from pkg/agent") + } + if crux.KickPhasePending != agent.KickPhasePending || crux.KickPhaseFailed != agent.KickPhaseFailed { + t.Fatal("kick phase constants diverged from pkg/agent") + } + if crux.StartFailureCredentialRejected != agent.StartFailureCredentialRejected { + t.Fatal("start failure constants diverged from pkg/agent") + } + var p crux.Process + p.State = crux.StateRunning + if p.State != agent.StateRunning { + t.Fatal("Process.State does not share agent.AgentProcess's State type") + } + p.KickOutcome = crux.KickOutcome{Kind: crux.KickOutcomeEnded, KickAt: time.Unix(1, 0)} + kickAt := time.Unix(1, 0) + if !p.KickOutcome.Settled(&kickAt) || p.KickOutcome.Kind != agent.KickOutcomeEnded { + t.Fatal("KickOutcome alias does not share agent.AgentProcess's KickOutcome type") + } + p.RestartEvents = []crux.RestartEvent{{Reason: "x"}} + p.TurnLoss = crux.TurnLoss{Recent: []crux.TurnInterruption{{Reason: "restart"}}} + p.KickHistory = []crux.KickRecord{{Agent: "a"}} + p.SubAgentModels = []crux.SubAgentModel{{Model: "m"}} + p.StartFailureClass = string(crux.StartFailureNoOutput) + if p.StartFailureClass != string(agent.StartFailureNoOutput) || !(crux.KickDispatch{Phase: crux.KickPhasePending}).Pending() { + t.Fatal("start-failure or kick-dispatch aliases diverged from pkg/agent") + } +} + +// TestProviderStatusAliasesTrackTypes pins that the provider-status surface +// re-exported here is the same one pkg/agent aliases, so a hub comparing +// crux.BackendAuthOK against a Process.BackendAuth.Status the runtime wrote +// can never drift. +func TestProviderStatusAliasesTrackTypes(t *testing.T) { + if crux.BackendAuthOK != agent.BackendAuthOK || crux.BackendAuthForbidden != agent.BackendAuthForbidden { + t.Fatal("backend auth constants diverged from pkg/agent") + } + if crux.ProviderAuthNeedsLogin != agent.ProviderAuthNeedsLogin || crux.ProviderAvailUnknown != agent.ProviderAvailUnknown { + t.Fatal("provider state constants diverged from pkg/agent") + } + var p crux.Process + p.BackendAuth = crux.BackendAuthState{Status: crux.BackendAuthQuota} + if p.BackendAuth.Status != agent.BackendAuthQuota { + t.Fatal("BackendAuthState alias does not share agent.AgentProcess's BackendAuth type") + } + got := crux.RollUpProviderStatus([]crux.ProviderAgentEvidence{{Agent: "a", Provider: "copilot", Enabled: true, AuthStatus: crux.BackendAuthUnlicensed}}) + want := agent.RollUpProviderStatus([]agent.ProviderAgentEvidence{{Agent: "a", Provider: "copilot", Enabled: true, AuthStatus: crux.BackendAuthUnlicensed}}) + if len(got) != 1 || len(want) != 1 || got[0].AuthState != want[0].AuthState || got[0].Evidence != want[0].Evidence { + t.Fatalf("RollUpProviderStatus = %+v, agent wrapper = %+v", got, want) + } + if got[0].AuthState != crux.ProviderAuthNeedsLogin { + t.Fatalf("AuthState = %q, want %q", got[0].AuthState, crux.ProviderAuthNeedsLogin) + } +} + +func TestFatalNetworkErrorPatternsIsACopy(t *testing.T) { + got := crux.FatalNetworkErrorPatterns() + if len(got) != len(agent.FatalNetworkErrorPatterns) || len(got) == 0 { + t.Fatalf("FatalNetworkErrorPatterns = %v, want the runtime's %v", got, agent.FatalNetworkErrorPatterns) + } + got[0] = "mutated" + if agent.FatalNetworkErrorPatterns[0] == "mutated" { + t.Fatal("FatalNetworkErrorPatterns must return a copy") + } +} diff --git a/src/pkg/sandbox/credential_name_test.go b/src/crux/sandbox/credential_name_test.go similarity index 100% rename from src/pkg/sandbox/credential_name_test.go rename to src/crux/sandbox/credential_name_test.go diff --git a/src/pkg/sandbox/sandbox.go b/src/crux/sandbox/sandbox.go similarity index 100% rename from src/pkg/sandbox/sandbox.go rename to src/crux/sandbox/sandbox.go diff --git a/src/pkg/sandbox/sandbox_coverage_test.go b/src/crux/sandbox/sandbox_coverage_test.go similarity index 100% rename from src/pkg/sandbox/sandbox_coverage_test.go rename to src/crux/sandbox/sandbox_coverage_test.go diff --git a/src/pkg/sandbox/sandbox_launcher_test.go b/src/crux/sandbox/sandbox_launcher_test.go similarity index 100% rename from src/pkg/sandbox/sandbox_launcher_test.go rename to src/crux/sandbox/sandbox_launcher_test.go diff --git a/src/pkg/sandbox/sandbox_test.go b/src/crux/sandbox/sandbox_test.go similarity index 100% rename from src/pkg/sandbox/sandbox_test.go rename to src/crux/sandbox/sandbox_test.go diff --git a/src/pkg/sandbox/worktree.go b/src/crux/sandbox/worktree.go similarity index 100% rename from src/pkg/sandbox/worktree.go rename to src/crux/sandbox/worktree.go diff --git a/src/pkg/sandbox/worktree_test.go b/src/crux/sandbox/worktree_test.go similarity index 100% rename from src/pkg/sandbox/worktree_test.go rename to src/crux/sandbox/worktree_test.go diff --git a/src/crux/scripts/smoke.sh b/src/crux/scripts/smoke.sh new file mode 100755 index 0000000000..18ff5aa244 --- /dev/null +++ b/src/crux/scripts/smoke.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Smoke test for the crux binary: start `crux serve` on a private unix socket +# with an empty runtime, drive it over the Crux API with `crux --server` +# (ps) and a raw GET of the health route, then stop it and check the server +# exited cleanly. Needs no agents, tmux or network; safe in CI. +# +# CRUX_BIN binary under test (default: bin/crux next to this module, +# built here with `go build` when it does not exist yet) +# SMOKE_DIR scratch dir for the socket and work dir (default: .smoke here; +# keep it short, a unix socket path is limited to ~100 bytes) +set -euo pipefail + +here=$(cd "$(dirname "$0")/.." && pwd) +CRUX_BIN=${CRUX_BIN:-$here/bin/crux} +SMOKE_DIR=${SMOKE_DIR:-$here/.smoke} +SOCK=$SMOKE_DIR/sock/crux.sock +WORK=$SMOKE_DIR/work +# How long to wait for the socket to appear, in tenths of a second. +WAIT_TICKS=${WAIT_TICKS:-100} + +rm -rf "$SMOKE_DIR" +mkdir -p "$WORK" +chmod 0700 "$SMOKE_DIR" + +fail() { echo "smoke: FAIL: $*" >&2; exit 1; } + +if [ ! -x "$CRUX_BIN" ]; then + echo "smoke: building $CRUX_BIN" + (cd "$here" && go build -o "$CRUX_BIN" ./cmd/crux) || fail "go build ./cmd/crux failed" +fi + +echo "smoke: $("$CRUX_BIN" --version)" + +export HIVE_WORK_DIR=$WORK +unset CRUX_SERVER CRUX_TOKEN CRUX_DEFS CRUX_SOCKET + +"$CRUX_BIN" serve --socket "$SOCK" >"$SMOKE_DIR/serve.out" 2>"$SMOKE_DIR/serve.err" & +SERVE_PID=$! +cleanup() { + if kill -0 "$SERVE_PID" 2>/dev/null; then + kill "$SERVE_PID" 2>/dev/null || true + wait "$SERVE_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +for _ in $(seq "$WAIT_TICKS"); do + [ -S "$SOCK" ] && break + kill -0 "$SERVE_PID" 2>/dev/null || { cat "$SMOKE_DIR/serve.err" >&2; fail "crux serve exited before listening"; } + sleep 0.1 +done +[ -S "$SOCK" ] || { cat "$SMOKE_DIR/serve.err" >&2; fail "no socket at $SOCK after $((WAIT_TICKS / 10))s"; } +echo "smoke: serving on $SOCK" + +# Remote mode over the socket: an empty runtime lists no agents and exits 0. +"$CRUX_BIN" --server "$SOCK" ps >"$SMOKE_DIR/ps.out" || { cat "$SMOKE_DIR/ps.out" >&2; fail "crux --server ps failed"; } +echo "smoke: ps ok ($(wc -l <"$SMOKE_DIR/ps.out" | tr -d ' ') line(s))" + +# The JSON form must be a (possibly empty) list. +"$CRUX_BIN" --server "$SOCK" ps --json >"$SMOKE_DIR/ps.json" || fail "crux --server ps --json failed" +case "$(tr -d ' \n' <"$SMOKE_DIR/ps.json")" in + \[*\]|null) ;; + *) cat "$SMOKE_DIR/ps.json" >&2; fail "ps --json did not print a list" ;; +esac + +# Health: the API's GET /v1/health, the equivalent of a `crux health` verb. +if command -v curl >/dev/null 2>&1; then + health=$(curl -fsS --unix-socket "$SOCK" http://crux/v1/health) || fail "GET /v1/health failed" + echo "smoke: health $health" +else + echo "smoke: curl not found; skipping GET /v1/health" +fi + +# A missing token must not be needed on the socket; an unknown agent is a +# clean usage-class error, not a crash. +if "$CRUX_BIN" --server "$SOCK" stop no-such-agent >/dev/null 2>"$SMOKE_DIR/stop.err"; then + fail "stop of an unknown agent succeeded" +fi +kill -0 "$SERVE_PID" 2>/dev/null || fail "crux serve died during the run" + +kill -INT "$SERVE_PID" +trap - EXIT +if ! wait "$SERVE_PID"; then + cat "$SMOKE_DIR/serve.err" >&2 + fail "crux serve exited non-zero on SIGINT" +fi +grep -q "serve stopped" "$SMOKE_DIR/serve.out" || { cat "$SMOKE_DIR/serve.out" >&2; fail "serve did not report a clean stop"; } +[ -S "$SOCK" ] && fail "socket $SOCK left behind after stop" +echo "smoke: PASS" diff --git a/src/crux/serve/families.go b/src/crux/serve/families.go new file mode 100644 index 0000000000..4312ca3e02 --- /dev/null +++ b/src/crux/serve/families.go @@ -0,0 +1,1087 @@ +package serve + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "sort" + "strings" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/trajectory" + "github.com/hivecommons/hive/crux/watchdog" +) + +// snapshotPaneLines bounds the pane capture a snapshot carries; the +// runtime's filter already prefers the current CLI session's lines. +const snapshotPaneLines = 10000 + +// watchdogSourcer is what a runtime implements to expose the watchdog seam +// (*agent.Manager does); a runtime without it answers CodeNotSupported on +// the watchdog routes rather than inventing observations. +type watchdogSourcer interface { + WatchdogSource() watchdog.Source +} + +// registerFamilies adds the stage-3 routes (every crux.Runtime family not +// covered by the CLI-shaped routes) to mux. +func (s *Server) registerFamilies(mux *http.ServeMux) { + for _, r := range []struct { + method, path string + h http.HandlerFunc + }{ + // Lifecycle + {"POST", api.PathAgents + "/{name}/start", s.handleStart}, + {"POST", api.PathAgents + "/{name}/restart", s.handleRestart}, + {"POST", api.PathAgents + "/{name}/pause", s.handlePause}, + {"POST", api.PathAgents + "/{name}/resume", s.handleResume}, + {"DELETE", api.PathAgents + "/{name}/restart-count", s.handleResetRestartCount}, + // Observer + {"GET", api.PathAgents + "/{name}/snapshot", s.handleSnapshot}, + {"GET", api.PathSnapshots, s.handleSnapshots}, + // Kicker + {"POST", api.PathAgents + "/{name}/kick/validate", s.handleKickValidate}, + {"GET", api.PathAgents + "/{name}/kick", s.handleKickDispatch}, + {"GET", api.PathAgents + "/{name}/kicklogs", s.handleKickLogs}, + {"GET", api.PathAgents + "/{name}/kicklogs/{id}", s.handleKickLog}, + // Definitions + {"POST", api.PathDefinitions, s.handleAddDefinition}, + {"DELETE", api.PathAgents + "/{name}", s.handleRemoveAgent}, + {"PUT", api.PathAgents + "/{name}/definition", s.handleUpdateDefinition}, + {"POST", api.PathReconcile, s.handleReconcile}, + {"PUT", api.PathAgents + "/{name}/overrides", s.handleOverrides}, + {"PUT", api.PathAgents + "/{name}/pins/{kind}", s.handlePin}, + {"DELETE", api.PathAgents + "/{name}/pins/{kind}", s.handleUnpin}, + {"POST", api.PathModesClear, s.handleModesClear}, + {"POST", api.PathModesSync, s.handleModesSync}, + {"POST", api.PathInference, s.handleInference}, + // Fleet + {"GET", api.PathFleet, s.handleFleet}, + {"GET", api.PathBreaker, s.handleBreakerState}, + {"PUT", api.PathBreaker, s.handleRestoreBreaker}, + {"POST", api.PathBreakerOn, s.handleBreakerEngage}, + {"POST", api.PathBreakerOff, s.handleBreakerRelease}, + {"PUT", api.PathACMM, s.handleACMM}, + {"POST", api.PathRelaunchBob, s.handleRelaunchBob}, + {"POST", api.PathStartupQueued, s.handleStartupQueued}, + {"POST", api.PathRestartCrashed, s.handleRestartCrashed}, + {"POST", api.PathTokensRefresh, s.handleTokensRefresh}, + {"POST", api.PathKickLogsArchive, s.handleKickLogsArchive}, + // Credentials + {"POST", api.PathAgents + "/{name}/login-code", s.handleLoginCode}, + {"GET", api.PathCopilotCred, s.handleCopilotGet}, + {"PUT", api.PathCopilotCred, s.handleCopilotPut}, + {"POST", api.PathClaudeReload, s.handleClaudeReload}, + {"GET", api.PathClaudeCred, s.handleClaudeGet}, + {"GET", api.PathOmpHomes, s.handleOmpHomes}, + // Resumer + {"GET", api.PathAgents + "/{name}/session", s.handleSession}, + {"POST", api.PathAgents + "/{name}/session/kick", s.handleResumeKick}, + // Supervisor + {"GET", api.PathAgents + "/{name}/supervision", s.handleSupervision}, + {"POST", api.PathAgents + "/{name}/tokens/refresh", s.handleAgentTokenRefresh}, + {"GET", api.PathBackends + "/{backend}/auth", s.handleBackendAuth}, + {"POST", api.PathAgents + "/{name}/nudge", s.handleNudge}, + {"POST", api.PathAudit, s.handleAudit}, + {"GET", api.PathTrajectory, s.handleTrajectory}, + // Restorer + {"POST", api.PathAgents + "/{name}/seed", s.handleSeed}, + // Authorizer + {"POST", api.PathAuthorize, s.handleAuthorize}, + // Watchdog seam + {"GET", api.PathWatchdogAgents, s.handleWatchdogAgents}, + {"GET", api.PathAgents + "/{name}/watchdog", s.handleWatchdogObserve}, + {"PUT", api.PathAgents + "/{name}/watchdog/conditions", s.handleWatchdogConditions}, + } { + mux.HandleFunc(r.method+" "+r.path, r.h) + } +} + +// resolve resolves the {name} path value and reports whether the runtime +// holds that agent; when it does not, a 404 has been written. +func (s *Server) resolve(w http.ResponseWriter, r *http.Request) (string, bool) { + name := s.Runtime.ResolveAgent(r.PathValue("name")) + if p, err := s.Runtime.GetStatusFast(name); err != nil || p == nil { + s.writeError(w, http.StatusNotFound, api.CodeNotFound, fmt.Sprintf("agent %q not found", name)) + return name, false + } + return name, true +} + +// readOptionalJSON is readJSON for routes whose body may be empty. +func (s *Server) readOptionalJSON(w http.ResponseWriter, r *http.Request, v any) bool { + if r.Body == nil || r.ContentLength == 0 { + return true + } + return s.readJSON(w, r, v) +} + +func (s *Server) noContent(w http.ResponseWriter) { + w.WriteHeader(http.StatusNoContent) +} + +// ---- Lifecycle ---- + +func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + if err := s.Runtime.Start(s.LaunchContext, name); err != nil { + s.writeRuntimeError(w, name, "start", err) + return + } + s.publish(api.Event{Type: api.EventStarted, Agent: name}) + s.writeJSON(w, http.StatusOK, s.agentResponse(name)) +} + +func (s *Server) handleRestart(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + var req api.RestartRequest + if !s.readOptionalJSON(w, r, &req) { + return + } + var err error + if req.Bootstrap != "" { + err = s.Runtime.RestartWithBootstrap(s.LaunchContext, name, req.Bootstrap) + } else { + err = s.Runtime.Restart(s.LaunchContext, name) + } + if err != nil { + s.writeRuntimeError(w, name, "restart", err) + return + } + s.writeJSON(w, http.StatusOK, s.agentResponse(name)) +} + +func (s *Server) handlePause(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + var req api.PauseRequest + if !s.readJSON(w, r, &req) { + return + } + if strings.TrimSpace(req.Trigger) == "" { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "trigger is required") + return + } + var err error + switch { + case req.Cause != nil: + err = s.Runtime.PauseByCause(name, req.Trigger, req.Reason, req.By, crux.PauseCausation{ + Depth: req.Cause.Depth, HookName: req.Cause.HookName, OriginTransition: req.Cause.OriginTransition, + }) + case req.By != "": + err = s.Runtime.PauseBy(name, req.Trigger, req.Reason, req.By) + default: + err = s.Runtime.Pause(name, req.Trigger, req.Reason) + } + if err != nil { + s.writeRuntimeError(w, name, "pause", err) + return + } + s.writeJSON(w, http.StatusOK, s.agentResponse(name)) +} + +func (s *Server) handleResume(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + var req api.ResumeRequest + if !s.readJSON(w, r, &req) { + return + } + if strings.TrimSpace(req.Trigger) == "" { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "trigger is required") + return + } + if err := s.Runtime.Resume(s.LaunchContext, name, req.Trigger, req.Reason); err != nil { + s.writeRuntimeError(w, name, "resume", err) + return + } + s.writeJSON(w, http.StatusOK, s.agentResponse(name)) +} + +func (s *Server) handleResetRestartCount(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + if err := s.Runtime.ResetRestartCount(name); err != nil { + s.writeRuntimeError(w, name, "reset restart count", err) + return + } + s.noContent(w) +} + +// ---- Observer ---- + +// snapshotOf projects a runtime snapshot onto the wire, without the output +// buffer (a ring buffer with a mutex; the logs route serves its content). +func snapshotOf(name string, p *crux.Process) (api.Snapshot, error) { + c := *p + c.OutputBuffer = nil + raw, err := json.Marshal(&c) + if err != nil { + return api.Snapshot{}, err + } + return api.Snapshot{Agent: name, Process: raw, Pane: p.PaneLines(snapshotPaneLines)}, nil +} + +func (s *Server) handleSnapshot(w http.ResponseWriter, r *http.Request) { + name := s.Runtime.ResolveAgent(r.PathValue("name")) + var ( + p *crux.Process + err error + ) + if isTrue(r.URL.Query().Get(api.QueryFast)) { + p, err = s.Runtime.GetStatusFast(name) + } else { + p, err = s.Runtime.GetStatus(name) + } + if err != nil || p == nil { + s.writeError(w, http.StatusNotFound, api.CodeNotFound, fmt.Sprintf("agent %q not found", name)) + return + } + snap, err := snapshotOf(name, p) + if err != nil { + s.writeError(w, http.StatusInternalServerError, api.CodeRuntime, "snapshot "+name+": "+err.Error()) + return + } + s.writeJSON(w, http.StatusOK, snap) +} + +func (s *Server) handleSnapshots(w http.ResponseWriter, _ *http.Request) { + statuses := s.Runtime.AllStatuses() + out := api.SnapshotList{Snapshots: make([]api.Snapshot, 0, len(statuses))} + for name, p := range statuses { + if p == nil { + continue + } + snap, err := snapshotOf(name, p) + if err != nil { + s.writeError(w, http.StatusInternalServerError, api.CodeRuntime, "snapshot "+name+": "+err.Error()) + return + } + out.Snapshots = append(out.Snapshots, snap) + } + sort.Slice(out.Snapshots, func(i, j int) bool { return out.Snapshots[i].Agent < out.Snapshots[j].Agent }) + s.writeJSON(w, http.StatusOK, out) +} + +// ---- Kicker ---- + +func (s *Server) handleKickValidate(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + if err := s.Runtime.ValidateKick(name); err != nil { + s.writeError(w, http.StatusConflict, api.CodeKickRefused, err.Error()) + return + } + s.noContent(w) +} + +func (s *Server) handleKickDispatch(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + out := api.KickDispatchResponse{Agent: name} + if d, ok := s.Runtime.KickDispatchState(name); ok { + out.Dispatch = &api.KickDispatch{Agent: d.Agent, Phase: d.Phase, Error: d.Error, QueuedAt: d.QueuedAt, SettledAt: d.SettledAt} + } + s.writeJSON(w, http.StatusOK, out) +} + +func (s *Server) handleKickLogs(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + infos, err := s.Runtime.ListKickLogs(name) + if err != nil { + s.writeRuntimeError(w, name, "kick logs", err) + return + } + out := api.KickLogList{Agent: name, Logs: make([]api.KickLogInfo, 0, len(infos))} + for _, in := range infos { + out.Logs = append(out.Logs, api.KickLogInfo{ID: in.ID, Timestamp: in.Timestamp, SizeBytes: in.SizeBytes, Reason: in.Reason, Source: in.Source}) + } + s.writeJSON(w, http.StatusOK, out) +} + +func (s *Server) handleKickLog(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + id := r.PathValue("id") + content, err := s.Runtime.ReadKickLog(name, id) + if err != nil { + s.writeRuntimeError(w, name, "kick log "+id, err) + return + } + s.writeJSON(w, http.StatusOK, api.KickLog{Agent: name, ID: id, Content: content}) +} + +// ---- Definitions ---- + +// decodeVerbatim parses a host-supplied config.AgentConfig without filling +// defaults: the host owns the complete definition. +func decodeVerbatim(raw json.RawMessage) (config.AgentConfig, error) { + var cfg config.AgentConfig + if len(raw) == 0 || string(raw) == "null" { + return cfg, errors.New("definition is required") + } + err := json.Unmarshal(raw, &cfg) + return cfg, err +} + +func (s *Server) handleAddDefinition(w http.ResponseWriter, r *http.Request) { + var req api.DefinitionRequest + if !s.readJSON(w, r, &req) { + return + } + if strings.TrimSpace(req.Name) == "" { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "name is required") + return + } + cfg, err := decodeVerbatim(req.Definition) + if err != nil { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "definition: "+err.Error()) + return + } + status := http.StatusCreated + if _, err := s.Runtime.GetStatusFast(req.Name); err == nil { + status = http.StatusOK + } + s.Runtime.AddAgent(req.Name, cfg) + p, err := s.Runtime.GetStatusFast(req.Name) + if err != nil || p == nil { + s.writeError(w, http.StatusUnprocessableEntity, api.CodeBadRequest, + fmt.Sprintf("agent %q was not accepted by the runtime", req.Name)) + return + } + s.writeJSON(w, status, toWire(p)) +} + +func (s *Server) handleRemoveAgent(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + s.Runtime.RemoveAgent(name) + s.noContent(w) +} + +func (s *Server) handleUpdateDefinition(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + var req api.DefinitionRequest + if !s.readJSON(w, r, &req) { + return + } + cfg, err := decodeVerbatim(req.Definition) + if err != nil { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "definition: "+err.Error()) + return + } + if err := s.Runtime.UpdateConfig(name, cfg); err != nil { + s.writeRuntimeError(w, name, "update definition", err) + return + } + s.writeJSON(w, http.StatusOK, s.agentResponse(name)) +} + +func (s *Server) handleReconcile(w http.ResponseWriter, r *http.Request) { + var req api.ReconcileRequest + if !s.readJSON(w, r, &req) { + return + } + configs := make(map[string]config.AgentConfig, len(req.Definitions)) + for name, raw := range req.Definitions { + cfg, err := decodeVerbatim(raw) + if err != nil { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, fmt.Sprintf("definition %q: %v", name, err)) + return + } + configs[name] = cfg + } + s.writeJSON(w, http.StatusOK, api.Names{Names: names(s.Runtime.ReconcileAgents(configs))}) +} + +func names(in []string) []string { + if in == nil { + return []string{} + } + return in +} + +func (s *Server) handleOverrides(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + var req api.OverridesRequest + if !s.readJSON(w, r, &req) { + return + } + if req.Model == nil && req.Backend == nil { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "model or backend is required") + return + } + if req.Model != nil { + if err := s.Runtime.SetModelOverride(name, *req.Model); err != nil { + s.writeRuntimeError(w, name, "model override", err) + return + } + } + if req.Backend != nil { + if err := s.Runtime.SetBackendOverride(name, *req.Backend); err != nil { + s.writeRuntimeError(w, name, "backend override", err) + return + } + } + s.writeJSON(w, http.StatusOK, s.agentResponse(name)) +} + +func (s *Server) handlePin(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + var req api.PinRequest + if !s.readJSON(w, r, &req) { + return + } + if strings.TrimSpace(req.Value) == "" { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "value is required") + return + } + var err error + switch kind := r.PathValue("kind"); kind { + case api.PinCLI: + err = s.Runtime.PinCLI(name, req.Value) + case api.PinModel: + err = s.Runtime.PinModel(name, req.Value) + default: + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, fmt.Sprintf("pin kind must be %s or %s", api.PinCLI, api.PinModel)) + return + } + if err != nil { + s.writeRuntimeError(w, name, "pin", err) + return + } + s.noContent(w) +} + +func (s *Server) handleUnpin(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + var err error + switch kind := r.PathValue("kind"); kind { + case api.PinCLI: + err = s.Runtime.UnpinCLI(name) + case api.PinModel: + err = s.Runtime.UnpinModel(name) + default: + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, fmt.Sprintf("pin kind must be %s or %s", api.PinCLI, api.PinModel)) + return + } + if err != nil { + s.writeRuntimeError(w, name, "unpin", err) + return + } + s.noContent(w) +} + +func (s *Server) handleModesClear(w http.ResponseWriter, r *http.Request) { + var req api.Names + if !s.readJSON(w, r, &req) { + return + } + s.Runtime.ClearModeOverrides(req.Names) + s.noContent(w) +} + +func (s *Server) handleModesSync(w http.ResponseWriter, r *http.Request) { + var req api.Level + if !s.readJSON(w, r, &req) { + return + } + s.Runtime.SyncModeFiles(req.Level) + s.noContent(w) +} + +func (s *Server) handleInference(w http.ResponseWriter, r *http.Request) { + var req api.Backend + if !s.readJSON(w, r, &req) { + return + } + s.Runtime.RefreshInferenceRoutes(req.Backend) + s.noContent(w) +} + +// ---- Fleet ---- + +func (s *Server) handleFleet(w http.ResponseWriter, _ *http.Request) { + s.writeJSON(w, http.StatusOK, api.FleetInfo{ + ACMMLevel: s.Runtime.GetACMMLevel(), + SharedDevUID: s.Runtime.SharedDevUID(), + AgentsWithModel: s.Runtime.CountAgentsWithModel(), + ConsentWedged: names(s.Runtime.ConsentWedgedAgents()), + }) +} + +func (s *Server) handleBreakerState(w http.ResponseWriter, _ *http.Request) { + engaged, paused := s.Runtime.BreakerState() + s.writeJSON(w, http.StatusOK, api.Breaker{Engaged: engaged, Paused: names(paused)}) +} + +func (s *Server) handleRestoreBreaker(w http.ResponseWriter, r *http.Request) { + var req api.Breaker + if !s.readJSON(w, r, &req) { + return + } + s.Runtime.RestoreBreaker(req.Engaged, req.Paused) + s.noContent(w) +} + +func (s *Server) handleBreakerEngage(w http.ResponseWriter, _ *http.Request) { + s.writeJSON(w, http.StatusOK, api.Names{Names: names(s.Runtime.EngageBreaker())}) +} + +func (s *Server) handleBreakerRelease(w http.ResponseWriter, _ *http.Request) { + s.writeJSON(w, http.StatusOK, api.Names{Names: names(s.Runtime.ReleaseBreaker(s.LaunchContext))}) +} + +func (s *Server) handleACMM(w http.ResponseWriter, r *http.Request) { + var req api.Level + if !s.readJSON(w, r, &req) { + return + } + s.Runtime.SetACMMLevel(req.Level) + s.noContent(w) +} + +func (s *Server) handleRelaunchBob(w http.ResponseWriter, _ *http.Request) { + s.writeJSON(w, http.StatusOK, api.Names{Names: names(s.Runtime.RelaunchBobAgentsAwaitingKey(s.LaunchContext))}) +} + +func (s *Server) handleStartupQueued(w http.ResponseWriter, r *http.Request) { + var req api.Names + if !s.readJSON(w, r, &req) { + return + } + s.Runtime.MarkStartupLaunchQueued(req.Names) + s.noContent(w) +} + +func (s *Server) handleRestartCrashed(w http.ResponseWriter, _ *http.Request) { + s.writeJSON(w, http.StatusOK, api.Names{Names: names(s.Runtime.CheckAndRestartCrashedAgents(s.LaunchContext))}) +} + +func (s *Server) handleTokensRefresh(w http.ResponseWriter, _ *http.Request) { + s.Runtime.RefreshAgentTokens(s.LaunchContext) + s.noContent(w) +} + +func (s *Server) handleKickLogsArchive(w http.ResponseWriter, r *http.Request) { + var req api.Reason + if !s.readJSON(w, r, &req) { + return + } + s.Runtime.ArchiveAllKickLogs(req.Reason) + s.noContent(w) +} + +// ---- Credentials ---- + +func (s *Server) handleLoginCode(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + var req api.LoginCodeRequest + if !s.readJSON(w, r, &req) { + return + } + if strings.TrimSpace(req.Code) == "" { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "code is required") + return + } + if err := s.Runtime.SubmitLoginCode(name, req.Code); err != nil { + s.writeRuntimeError(w, name, "login code", err) + return + } + s.noContent(w) +} + +func (s *Server) handleCopilotGet(w http.ResponseWriter, _ *http.Request) { + s.writeJSON(w, http.StatusOK, api.CopilotCredential{Token: s.Runtime.CopilotToken(), Source: s.Runtime.CopilotTokenSource()}) +} + +func (s *Server) handleCopilotPut(w http.ResponseWriter, r *http.Request) { + var req api.CopilotCredential + if !s.readJSON(w, r, &req) { + return + } + if req.Activate { + if err := s.Runtime.ActivateCopilotToken(req.Token); err != nil { + s.writeError(w, http.StatusUnprocessableEntity, api.CodeRuntime, "activate copilot token: "+err.Error()) + return + } + } else { + s.Runtime.SetCopilotToken(req.Token) + } + s.noContent(w) +} + +func (s *Server) handleClaudeReload(w http.ResponseWriter, _ *http.Request) { + s.Runtime.ReloadClaudeToken() + s.noContent(w) +} + +func (s *Server) handleClaudeGet(w http.ResponseWriter, _ *http.Request) { + s.writeJSON(w, http.StatusOK, api.ClaudeCredential{CandidatePaths: names(s.Runtime.ClaudeCredentialCandidatePaths())}) +} + +func (s *Server) handleOmpHomes(w http.ResponseWriter, _ *http.Request) { + s.writeJSON(w, http.StatusOK, api.OmpHomes{Homes: names(s.Runtime.OmpAgentHomes())}) +} + +// ---- Resumer ---- + +func (s *Server) handleSession(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + out := api.Session{Agent: name, Missing: s.Runtime.SessionMissing(name)} + out.SessionID, out.HasSession = s.Runtime.SessionID(name) + if h, ok := s.Runtime.ResumeHandle(name); ok { + out.Handle = &api.ResumeHandle{Backend: h.Backend, SessionID: h.SessionID, Transcript: h.Transcript, Command: h.Command, LastActive: h.LastActive, CapturedAt: h.CapturedAt} + } + s.writeJSON(w, http.StatusOK, out) +} + +// handleResumeKick maps the Resumer sentinels onto statuses: busy is a 409 +// (retry later), a gone session a 410, an agent that cannot resume a 422. +func (s *Server) handleResumeKick(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + var req api.ResumeKickRequest + if !s.readJSON(w, r, &req) { + return + } + if strings.TrimSpace(req.Message) == "" || strings.TrimSpace(req.SessionID) == "" { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "message and session_id are required") + return + } + err := s.Runtime.SendResumeKick(name, req.Message, req.SessionID) + switch { + case err == nil: + s.publish(api.Event{Type: api.EventKicked, Agent: name, Detail: "resume"}) + s.writeJSON(w, http.StatusOK, s.agentResponse(name)) + case errors.Is(err, crux.ErrResumeBusy): + s.writeError(w, http.StatusConflict, api.CodeResumeBusy, err.Error()) + case errors.Is(err, crux.ErrResumeSessionGone): + s.writeError(w, http.StatusGone, api.CodeResumeSessionGone, err.Error()) + case errors.Is(err, crux.ErrResumeUnsupported): + s.writeError(w, http.StatusUnprocessableEntity, api.CodeResumeUnsupported, err.Error()) + default: + s.writeRuntimeError(w, name, "resume kick", err) + } +} + +// ---- Supervisor ---- + +func (s *Server) handleSupervision(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + rt := s.Runtime + out := api.Supervision{ + Agent: name, + MayStart: rt.GovernorMayStart(name), + HasValidCredential: rt.AgentHasValidCredential(name), + CrashOOMSuspected: rt.CrashOOMSuspected(name), + } + out.CredentialRefreshable, out.CredentialKnown = rt.AgentCredentialRefreshable(name) + out.AuthAvailable, out.AuthKnown = rt.AgentAuthAvailable(name) + if issue, pr, merge, ok := rt.AgentCapabilities(name); ok { + out.Capabilities = &api.Capabilities{CanOpenIssue: issue, CanOpenPR: pr, CanMerge: merge} + } + if b, ok := rt.EffectiveBackend(name); ok { + out.Backend = &b + } + if b, m, e, ok := rt.InvocationMetadata(name); ok { + out.Invocation = &api.Invocation{Backend: b, Model: m, Effort: e} + } + if total, last24h, at, reason, ok := rt.RestartTelemetry(name); ok { + out.Restarts = &api.RestartTelemetry{Total: total, Last24h: last24h, LastRestartAt: at, LastReason: reason} + } + if f, ok := rt.StartFailureState(name); ok { + out.StartFailure = &api.StartFailureSnapshot{Reason: f.Reason, Count: f.Count, LastAt: f.LastAt, LastExitCode: f.LastExitCode, LastSignal: f.LastSignal, Blocked: f.Blocked} + } + if d, provider, reason, ok := rt.ProviderErrorBackoffRemaining(name); ok { + out.ProviderBackoff = &api.ProviderBackoff{RemainingMS: d.Milliseconds(), Provider: provider, Reason: reason} + } + s.writeJSON(w, http.StatusOK, out) +} + +func (s *Server) handleAgentTokenRefresh(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + if err := s.Runtime.RefreshAgentTokenFor(s.LaunchContext, name); err != nil { + s.writeRuntimeError(w, name, "refresh token", err) + return + } + s.noContent(w) +} + +func (s *Server) handleBackendAuth(w http.ResponseWriter, r *http.Request) { + backend := r.PathValue("backend") + available, known := s.Runtime.BackendAuthAvailable(backend) + s.writeJSON(w, http.StatusOK, api.BackendAuth{Backend: backend, Available: available, Known: known}) +} + +func (s *Server) handleNudge(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + var req api.NudgeRequest + if !s.readJSON(w, r, &req) { + return + } + if strings.TrimSpace(req.Message) == "" { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "message is required") + return + } + s.Runtime.SendGitHubAPIBudgetNudge(name, req.Message) + s.noContent(w) +} + +func (s *Server) handleAudit(w http.ResponseWriter, r *http.Request) { + var req api.AuditRequest + if !s.readJSON(w, r, &req) { + return + } + if strings.TrimSpace(req.Action) == "" { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "action is required") + return + } + s.Runtime.RecordAudit(req.Action, req.Agent, req.Fields) + s.noContent(w) +} + +func (s *Server) handleTrajectory(w http.ResponseWriter, _ *http.Request) { + views := s.Runtime.TrajectoryAgents() + out := api.TrajectoryList{Agents: make([]api.TrajectoryAgent, 0, len(views))} + for _, v := range views { + out.Agents = append(out.Agents, trajectoryWire(v)) + } + s.writeJSON(w, http.StatusOK, out) +} + +func trajectoryWire(v trajectory.AgentView) api.TrajectoryAgent { + return api.TrajectoryAgent{Name: v.Name, Running: v.Running, Intent: v.Intent, Transcript: v.Transcript, Idle: v.Idle, LastOutputAt: v.LastOutputAt} +} + +// ---- Restorer ---- + +func (s *Server) handleSeed(w http.ResponseWriter, r *http.Request) { + name, ok := s.resolve(w, r) + if !ok { + return + } + var req api.SeedRequest + if !s.readJSON(w, r, &req) { + return + } + var loss crux.TurnLoss + if len(req.TurnLoss) > 0 { + if err := json.Unmarshal(req.TurnLoss, &loss); err != nil { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "turn_loss: "+err.Error()) + return + } + } + if req.Pause != nil { + s.Runtime.SeedPauseState(name, req.Pause.PausedAt, req.Pause.Trigger, req.Pause.Reason, req.Pause.By) + } + if req.Restarts != nil { + events := make([]crux.RestartEvent, 0, len(req.Restarts.Events)) + for _, e := range req.Restarts.Events { + events = append(events, crux.RestartEvent{At: e.At, Reason: e.Reason}) + } + s.Runtime.SeedRestartTelemetry(name, req.Restarts.Count, events, req.Restarts.LastReason) + } + if len(req.TurnLoss) > 0 { + s.Runtime.SeedTurnLoss(name, loss) + } + if req.LastKick != nil { + s.Runtime.SeedLastKick(name, *req.LastKick) + } + if req.KickHistory != nil { + records := make([]crux.KickRecord, 0, len(*req.KickHistory)) + for _, k := range *req.KickHistory { + records = append(records, crux.KickRecord{Timestamp: k.Timestamp, Agent: k.Agent, Snippet: k.Snippet}) + } + s.Runtime.SeedKickHistory(name, records) + } + s.noContent(w) +} + +// ---- Authorizer ---- + +// handleAuthorize answers 204 when the runtime permits the action and 403 +// with the runtime's reason when it does not. An unknown action is a 400, +// which a client must treat as a denial. +func (s *Server) handleAuthorize(w http.ResponseWriter, r *http.Request) { + var req api.AuthorizeRequest + if !s.readJSON(w, r, &req) { + return + } + if strings.TrimSpace(req.Agent) == "" { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "agent is required") + return + } + var err error + switch req.Action { + case api.AuthorizePROpen: + err = s.Runtime.AuthorizePROpen(req.Agent, req.FileUID) + case api.AuthorizePushBranch: + err = s.Runtime.AuthorizePushBranch(req.Agent, req.FileUID) + case api.AuthorizeIssueOpen: + err = s.Runtime.AuthorizeIssueOpen(req.Agent, req.FileUID, req.Kind) + case api.AuthorizeMerge: + err = s.Runtime.AuthorizeMerge(req.Agent, req.FileUID, req.Repo) + case api.AuthorizeReviewRequest: + err = s.Runtime.AuthorizeReviewRequest(req.Agent, req.FileUID) + default: + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, fmt.Sprintf("unknown action %q", req.Action)) + return + } + if err != nil { + s.writeError(w, http.StatusForbidden, api.CodeForbidden, err.Error()) + return + } + s.noContent(w) +} + +// ---- Watchdog seam ---- + +func (s *Server) watchdogSource(w http.ResponseWriter) (watchdog.Source, bool) { + ws, ok := s.Runtime.(watchdogSourcer) + if !ok { + s.writeError(w, http.StatusNotImplemented, api.CodeNotSupported, "runtime does not expose a watchdog source") + return nil, false + } + src := ws.WatchdogSource() + if src == nil { + s.writeError(w, http.StatusNotImplemented, api.CodeNotSupported, "runtime does not expose a watchdog source") + return nil, false + } + return src, true +} + +func (s *Server) handleWatchdogAgents(w http.ResponseWriter, _ *http.Request) { + src, ok := s.watchdogSource(w) + if !ok { + return + } + s.writeJSON(w, http.StatusOK, api.Names{Names: names(src.AgentNames())}) +} + +func (s *Server) handleWatchdogObserve(w http.ResponseWriter, r *http.Request) { + src, ok := s.watchdogSource(w) + if !ok { + return + } + name := s.Runtime.ResolveAgent(r.PathValue("name")) + o, err := src.Observe(name) + if err != nil { + s.writeRuntimeError(w, name, "observe", err) + return + } + out := api.WatchdogObservation{ + Agent: name, Backend: o.Backend, SessionExists: o.SessionExists, Pane: o.Pane, + HasCLIMarker: o.HasCLIMarker, ShowsLoginPrompt: o.ShowsLoginPrompt, LastChange: o.LastChange, + StartedAt: o.StartedAt, AuthAvailable: o.AuthAvailable, AuthKnown: o.AuthKnown, + CredentialProven: o.CredentialProven, ProviderErrorClass: o.ProviderErrorClass, ProviderErrorLine: o.ProviderErrorLine, + } + if t, ok := src.LastProduction(name); ok { + out.LastProduction = &t + } + s.writeJSON(w, http.StatusOK, out) +} + +func (s *Server) handleWatchdogConditions(w http.ResponseWriter, r *http.Request) { + src, ok := s.watchdogSource(w) + if !ok { + return + } + name, ok := s.resolve(w, r) + if !ok { + return + } + var req api.Conditions + if !s.readJSON(w, r, &req) { + return + } + conds := make([]crux.Condition, 0, len(req.Conditions)) + for _, c := range req.Conditions { + conds = append(conds, crux.Condition{ + Type: crux.ConditionType(c.Type), Status: crux.ConditionStatus(c.Status), + Reason: c.Reason, Message: c.Message, LastTransitionTime: c.LastTransitionTime, + }) + } + src.SetConditions(name, conds) + s.noContent(w) +} + +// ---- Host observers → /v1/events ---- + +// backgroundLoops are the crux.Background methods, in the order Serve starts +// them and Health reports them. +var backgroundLoops = []string{"StartAgentTokenRefresh", "StartCredentialWatchdog", "StartCopilotSessionRefresh", "StartTranscriptActivityRefresh"} + +// publishData publishes an event whose Data is v's JSON. +func (s *Server) publishData(typ, agent string, v any) { + raw, err := json.Marshal(v) + if err != nil { + s.Logger.Warn("crux serve: encoding event", "type", typ, "err", err) + return + } + s.publish(api.Event{Type: typ, Agent: agent, Data: raw}) +} + +// installHostObservers routes the crux.Host observer callbacks onto the +// event stream so a sidecar host sees what an in-process host would be +// called with. AuditSink and ConditionReporter are installed as tees: +// events always flow, and the Server's own sinks (when set) see the calls +// too. It returns the uninstaller. +func (s *Server) installHostObservers() func() { + host, ok := s.Runtime.(crux.Host) + if !ok { + return func() {} + } + host.SetKickOutcomeObserver(func(agent string, o crux.KickOutcome) { + s.publishData(api.EventKickOutcome, agent, api.KickOutcome{Kind: o.Kind, Reason: o.Reason, At: o.At, KickAt: o.KickAt}) + }) + host.SetPauseTransitionObserver(func(ev crux.PauseTransitionEvent) { + s.publishData(api.EventPauseTransition, ev.Agent, api.PauseTransition{ + Paused: ev.Paused, Trigger: ev.Trigger, Reason: ev.Reason, By: ev.By, At: ev.At, + Causation: api.PauseCausation{Depth: ev.Causation.Depth, HookName: ev.Causation.HookName, OriginTransition: ev.Causation.OriginTransition}, + }) + }) + host.SetRelaunchObserver(func(agent string) { + s.publish(api.Event{Type: api.EventRelaunch, Agent: agent}) + }) + host.SetPersistPauseCallback(func(agent string, paused bool) { + s.publishData(api.EventPausePersist, agent, api.PausePersist{Paused: paused}) + }) + host.SetRecordPromptCallback(func(agent, trigger, prompt string) { + s.publishData(api.EventPrompt, agent, api.Prompt{Trigger: trigger, Prompt: prompt}) + }) + host.SetSandboxAuditCallback(func(agent, action, detail string) { + s.publishData(api.EventSandboxAudit, agent, api.SandboxAudit{Action: action, Detail: detail}) + }) + host.SetAuditSink(auditTee{s}) + host.SetConditionReporter(conditionTee{s}) + return func() { + host.SetKickOutcomeObserver(nil) + host.SetPauseTransitionObserver(nil) + host.SetRelaunchObserver(nil) + host.SetPersistPauseCallback(nil) + host.SetRecordPromptCallback(nil) + host.SetSandboxAuditCallback(nil) + } +} + +type auditTee struct{ s *Server } + +func (t auditTee) Record(actor, action, agentName string, fields map[string]any) { + t.s.publishData(api.EventAudit, agentName, api.Audit{Actor: actor, Action: action, Fields: fields}) + if t.s.AuditSink != nil { + t.s.AuditSink.Record(actor, action, agentName, fields) + } +} + +type conditionTee struct{ s *Server } + +// Conditions is the server's own condition reporter: every report reaches +// the event stream (EventCondition) and ConditionReporter when set. It is +// how a condition about the serving process itself, rather than one the +// runtime raises, gets onto the same path (hivecap.Watch uses it for +// hive_callback_unavailable). +func (s *Server) Conditions() crux.ConditionReporter { return conditionTee{s} } + +// StreamConditions reports only to the event stream, never to +// ConditionReporter. It is for a condition that must not travel over the +// forwarding path because it describes that path being down +// (hivecap.Watch's hive_callback_unavailable, when ConditionReporter is the +// hive callback itself). +func (s *Server) StreamConditions() crux.ConditionReporter { return streamConditions{s} } + +type streamConditions struct{ s *Server } + +func (t streamConditions) Observe(class, key, message, diagnostics string) { + t.s.publishData(api.EventCondition, "", api.ConditionEvent{Action: api.ConditionObserve, Class: class, Key: key, Message: message, Diagnostics: diagnostics}) +} + +func (t streamConditions) Clear(class, key string) { + t.s.publishData(api.EventCondition, "", api.ConditionEvent{Action: api.ConditionClear, Class: class, Key: key}) +} + +func (t conditionTee) Observe(class, key, message, diagnostics string) { + t.s.publishData(api.EventCondition, "", api.ConditionEvent{Action: api.ConditionObserve, Class: class, Key: key, Message: message, Diagnostics: diagnostics}) + if t.s.ConditionReporter != nil { + t.s.ConditionReporter.Observe(class, key, message, diagnostics) + } +} + +func (t conditionTee) Clear(class, key string) { + t.s.publishData(api.EventCondition, "", api.ConditionEvent{Action: api.ConditionClear, Class: class, Key: key}) + if t.s.ConditionReporter != nil { + t.s.ConditionReporter.Clear(class, key) + } +} + +// startBackground runs the crux.Background loops under ctx when the Server +// is configured to, and returns the names Health reports. +func (s *Server) startBackground(ctx context.Context) { + if !s.Background { + return + } + rt := s.Runtime + go rt.StartAgentTokenRefresh(ctx) + go rt.StartCredentialWatchdog(ctx) + go rt.StartCopilotSessionRefresh(ctx) + go rt.StartTranscriptActivityRefresh(ctx) + s.mu.Lock() + s.background = append([]string(nil), backgroundLoops...) + s.mu.Unlock() +} diff --git a/src/crux/serve/families_test.go b/src/crux/serve/families_test.go new file mode 100644 index 0000000000..0312f8782b --- /dev/null +++ b/src/crux/serve/families_test.go @@ -0,0 +1,951 @@ +package serve + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/cruxtest" + "github.com/hivecommons/hive/crux/trajectory" + "github.com/hivecommons/hive/crux/watchdog" +) + +// famServer is a Server over a cruxtest.Recorder behind httptest, with +// LaunchContext set the way Serve would (no listener plumbing needed for +// handler tests). +type famServer struct { + t *testing.T + rt *cruxtest.Recorder + srv *Server + ts *httptest.Server +} + +func newFamServer(t *testing.T) *famServer { + t.Helper() + rt := cruxtest.NewRecorder() + srv := New(rt) + srv.Token = famToken + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(ts.Close) + return &famServer{t: t, rt: rt, srv: srv, ts: ts} +} + +// famToken is the bearer token handler tests use; httptest serves over TCP, +// which the server refuses without one. +const famToken = "fam-token" + +func (f *famServer) request(method, path string, rd io.Reader) *http.Request { + f.t.Helper() + req, err := http.NewRequest(method, f.ts.URL+path, rd) + if err != nil { + f.t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+famToken) + return req +} + +func (f *famServer) do(method, path string, body any) (int, []byte) { + f.t.Helper() + var rd io.Reader + switch b := body.(type) { + case nil: + case string: + rd = strings.NewReader(b) + default: + data, err := json.Marshal(body) + if err != nil { + f.t.Fatal(err) + } + rd = bytes.NewReader(data) + } + resp, err := http.DefaultClient.Do(f.request(method, path, rd)) + if err != nil { + f.t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + data, _ := io.ReadAll(resp.Body) + return resp.StatusCode, data +} + +func (f *famServer) expect(method, path string, body any, status int, into any) []byte { + f.t.Helper() + got, data := f.do(method, path, body) + if got != status { + f.t.Fatalf("%s %s: status %d want %d; body %s", method, path, got, status, data) + } + if into != nil { + if err := json.Unmarshal(data, into); err != nil { + f.t.Fatalf("%s %s: %v in %s", method, path, err, data) + } + } + return data +} + +func (f *famServer) expectCode(method, path string, body any, status int, code string) { + f.t.Helper() + var e api.Error + f.expect(method, path, body, status, &e) + if e.Code != code { + f.t.Fatalf("%s %s: code %q want %q (%s)", method, path, e.Code, code, e.Error) + } +} + +func (f *famServer) called(method string) cruxtest.Call { + f.t.Helper() + c, ok := f.rt.Last(method) + if !ok { + f.t.Fatalf("%s was not called; calls: %v", method, f.rt.Methods()) + } + return c +} + +// TestEveryRouteIsRegistered walks api.RuntimeRoutes and api.WatchdogRoutes +// and checks each one reaches a handler: an unregistered pattern gets the +// mux's plain-text 404 or 405, a registered one answers JSON (any status). +// This is the server half of the parity test; crux/api/client holds the +// interface half. +func TestEveryRouteIsRegistered(t *testing.T) { + f := newFamServer(t) + f.rt.Add("alpha", "a1", crux.StateRunning) + routes := map[string]api.Route{} + for k, v := range api.RuntimeRoutes { + routes[k] = v + } + for k, v := range api.WatchdogRoutes { + routes[k] = v + } + fill := strings.NewReplacer("{name}", "alpha", "{id}", "k1", "{kind}", api.PinCLI, "{backend}", "claude") + seen := map[api.Route]bool{} + for method, route := range routes { + if seen[route] { + continue + } + seen[route] = true + if route.Path == api.PathEvents { + continue // streams; covered by TestHostObserversReachTheEventStream + } + var body any + if route.Method == http.MethodPost || route.Method == http.MethodPut { + body = "{}" + } + req := f.request(route.Method, fill.Replace(route.Path), strings.NewReader("")) + if body != nil { + req.Body = io.NopCloser(strings.NewReader("{}")) + req.ContentLength = 2 + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + ct := resp.Header.Get("Content-Type") + if resp.StatusCode == http.StatusNotFound && ct != api.ContentTypeJSON || resp.StatusCode == http.StatusMethodNotAllowed { + t.Errorf("%s: %s %s is not registered (status %d, %s)", method, route.Method, route.Path, resp.StatusCode, ct) + } + } +} + +func TestLifecycleRoutes(t *testing.T) { + f := newFamServer(t) + f.rt.Add("alpha", "a1", crux.StateStopped) + + var resp api.AgentResponse + f.expect(http.MethodPost, api.StartPath("a1"), nil, http.StatusOK, &resp) + if resp.Agent != "alpha" || resp.Process.State != string(crux.StateRunning) { + t.Fatalf("start: %+v", resp) + } + f.expectCode(http.MethodPost, api.StartPath("nope"), nil, http.StatusNotFound, api.CodeNotFound) + + f.expect(http.MethodPost, api.RestartPath("alpha"), nil, http.StatusOK, &resp) + f.called("Restart") + f.expect(http.MethodPost, api.RestartPath("alpha"), api.RestartRequest{Bootstrap: "hello"}, http.StatusOK, &resp) + if c := f.called("RestartWithBootstrap"); c.Args[1] != "hello" { + t.Fatalf("bootstrap: %v", c) + } + f.rt.Err = cruxtest.ErrRecorder + f.expectCode(http.MethodPost, api.RestartPath("alpha"), nil, http.StatusInternalServerError, api.CodeRuntime) + f.expectCode(http.MethodPost, api.StartPath("alpha"), nil, http.StatusInternalServerError, api.CodeRuntime) + f.rt.Err = nil + + f.expect(http.MethodPost, api.PausePath("alpha"), api.PauseRequest{Trigger: "op", Reason: "r"}, http.StatusOK, &resp) + f.called("Pause") + if !resp.Process.Paused { + t.Fatal("pause not reflected") + } + f.expect(http.MethodPost, api.PausePath("alpha"), api.PauseRequest{Trigger: "op", Reason: "r", By: "me"}, http.StatusOK, &resp) + f.called("PauseBy") + f.expect(http.MethodPost, api.PausePath("alpha"), api.PauseRequest{Trigger: "op", Reason: "r", Cause: &api.PauseCausation{Depth: 2, HookName: "h"}}, http.StatusOK, &resp) + if c := f.called("PauseByCause"); c.Args[4].(crux.PauseCausation).Depth != 2 { + t.Fatalf("cause: %v", c) + } + f.expectCode(http.MethodPost, api.PausePath("alpha"), api.PauseRequest{Reason: "r"}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PausePath("alpha"), "{bad", http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.ResumePath("alpha"), api.ResumeRequest{}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.ResumePath("alpha"), "{bad", http.StatusBadRequest, api.CodeBadRequest) + f.expect(http.MethodPost, api.ResumePath("alpha"), api.ResumeRequest{Trigger: "op", Reason: "r"}, http.StatusOK, &resp) + if resp.Process.Paused { + t.Fatal("resume not reflected") + } + f.rt.Err = cruxtest.ErrRecorder + f.expectCode(http.MethodPost, api.PausePath("alpha"), api.PauseRequest{Trigger: "op"}, http.StatusInternalServerError, api.CodeRuntime) + f.expectCode(http.MethodPost, api.ResumePath("alpha"), api.ResumeRequest{Trigger: "op"}, http.StatusInternalServerError, api.CodeRuntime) + f.expectCode(http.MethodDelete, api.RestartCountPath("alpha"), nil, http.StatusInternalServerError, api.CodeRuntime) + f.rt.Err = nil + f.rt.Procs["alpha"].RestartCount = 4 + f.expect(http.MethodDelete, api.RestartCountPath("alpha"), nil, http.StatusNoContent, nil) + if f.rt.Procs["alpha"].RestartCount != 0 { + t.Fatal("restart count not reset") + } +} + +func TestSnapshotRoutes(t *testing.T) { + f := newFamServer(t) + p := f.rt.Add("alpha", "a1", crux.StateRunning) + p.RestartCount = 7 + p.OutputBuffer = crux.NewRingBuffer(4) + f.rt.Pane["alpha"] = []string{"one", "two"} + + var snap api.Snapshot + f.expect(http.MethodGet, api.SnapshotPath("a1"), nil, http.StatusOK, &snap) + f.called("GetStatus") + var proc crux.Process + if err := json.Unmarshal(snap.Process, &proc); err != nil { + t.Fatal(err) + } + if snap.Agent != "alpha" || proc.RestartCount != 7 || proc.OutputBuffer != nil || len(snap.Pane) != 2 { + t.Fatalf("snapshot: %+v %+v", snap, proc) + } + snap = api.Snapshot{} + f.expect(http.MethodGet, api.SnapshotPath("alpha")+"?"+api.QueryFast+"=1", nil, http.StatusOK, &snap) + f.called("GetStatusFast") + if len(snap.Pane) != 0 { + t.Fatal("fast snapshot carries a pane") + } + f.expectCode(http.MethodGet, api.SnapshotPath("nope"), nil, http.StatusNotFound, api.CodeNotFound) + + f.rt.Add("beta", "b1", crux.StateStopped) + var list api.SnapshotList + f.expect(http.MethodGet, api.PathSnapshots, nil, http.StatusOK, &list) + if len(list.Snapshots) != 2 || list.Snapshots[0].Agent != "alpha" || list.Snapshots[1].Agent != "beta" { + t.Fatalf("snapshots: %+v", list) + } +} + +func TestKickerRoutes(t *testing.T) { + f := newFamServer(t) + f.rt.Add("alpha", "a1", crux.StateRunning) + f.rt.KickStarted = true + + var resp api.AgentResponse + f.expect(http.MethodPost, api.KickPath("alpha"), api.KickRequest{Message: "go", Async: true}, http.StatusAccepted, &resp) + f.called("SendKickAsync") + if resp.Started == nil || !*resp.Started { + t.Fatalf("async: %+v", resp) + } + f.rt.KickErr = errors.New("agent alpha is paused") + f.expectCode(http.MethodPost, api.KickPath("alpha"), api.KickRequest{Message: "go", Async: true}, http.StatusInternalServerError, api.CodeRuntime) + f.expectCode(http.MethodPost, api.KickValidatePath("alpha"), nil, http.StatusConflict, api.CodeKickRefused) + f.rt.KickErr = nil + f.expect(http.MethodPost, api.KickValidatePath("alpha"), nil, http.StatusNoContent, nil) + f.expectCode(http.MethodPost, api.KickValidatePath("nope"), nil, http.StatusNotFound, api.CodeNotFound) + + var disp api.KickDispatchResponse + f.expect(http.MethodGet, api.KickPath("alpha"), nil, http.StatusOK, &disp) + if disp.Dispatch != nil { + t.Fatal("no dispatch expected") + } + f.rt.Dispatch["alpha"] = crux.KickDispatch{Agent: "alpha", Phase: crux.KickPhaseDelivered, QueuedAt: time.Unix(10, 0)} + f.expect(http.MethodGet, api.KickPath("alpha"), nil, http.StatusOK, &disp) + if disp.Dispatch == nil || disp.Dispatch.Phase != crux.KickPhaseDelivered { + t.Fatalf("dispatch: %+v", disp) + } + + f.rt.KickLogs["alpha"] = []crux.KickLogInfo{{ID: "k1", Reason: "kick", SizeBytes: 3}} + f.rt.KickLogContent["alpha"] = map[string]string{"k1": "abc"} + var logs api.KickLogList + f.expect(http.MethodGet, api.KickLogsPath("alpha"), nil, http.StatusOK, &logs) + if len(logs.Logs) != 1 || logs.Logs[0].ID != "k1" || logs.Logs[0].SizeBytes != 3 { + t.Fatalf("kick logs: %+v", logs) + } + var log api.KickLog + f.expect(http.MethodGet, api.KickLogPath("alpha", "k1"), nil, http.StatusOK, &log) + if log.Content != "abc" { + t.Fatalf("kick log: %+v", log) + } + f.expectCode(http.MethodGet, api.KickLogPath("alpha", "k2"), nil, http.StatusNotFound, api.CodeNotFound) + f.rt.Err = cruxtest.ErrRecorder + f.expectCode(http.MethodGet, api.KickLogsPath("alpha"), nil, http.StatusInternalServerError, api.CodeRuntime) +} + +func TestDefinitionRoutes(t *testing.T) { + f := newFamServer(t) + def := json.RawMessage(`{"id":"b1","backend":"claude","model":"opus"}`) + + // Verbatim: enabled was not sent, so it stays false (unlike POST /v1/agents). + var proc api.Process + f.expect(http.MethodPost, api.PathDefinitions, api.DefinitionRequest{Name: "beta", Definition: def}, http.StatusCreated, &proc) + if proc.Name != "beta" || proc.ID != "b1" || f.rt.Procs["beta"].Config.Enabled { + t.Fatalf("add: %+v %+v", proc, f.rt.Procs["beta"].Config) + } + f.expect(http.MethodPost, api.PathDefinitions, api.DefinitionRequest{Name: "beta", Definition: def}, http.StatusOK, &proc) + f.expectCode(http.MethodPost, api.PathDefinitions, api.DefinitionRequest{Name: "", Definition: def}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathDefinitions, api.DefinitionRequest{Name: "x"}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathDefinitions, api.DefinitionRequest{Name: "x", Definition: json.RawMessage(`[1]`)}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathDefinitions, "nope", http.StatusBadRequest, api.CodeBadRequest) + + var resp api.AgentResponse + f.expect(http.MethodPut, api.DefinitionPath("beta"), api.DefinitionRequest{Definition: json.RawMessage(`{"id":"b1","model":"sonnet"}`)}, http.StatusOK, &resp) + if f.rt.Procs["beta"].Config.Model != "sonnet" { + t.Fatal("update not applied") + } + f.expectCode(http.MethodPut, api.DefinitionPath("beta"), api.DefinitionRequest{}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPut, api.DefinitionPath("beta"), "x", http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPut, api.DefinitionPath("nope"), api.DefinitionRequest{Definition: def}, http.StatusNotFound, api.CodeNotFound) + f.rt.Err = cruxtest.ErrRecorder + f.expectCode(http.MethodPut, api.DefinitionPath("beta"), api.DefinitionRequest{Definition: def}, http.StatusInternalServerError, api.CodeRuntime) + f.rt.Err = nil + + f.rt.Reconciled = []string{"beta"} + var names api.Names + f.expect(http.MethodPost, api.PathReconcile, api.ReconcileRequest{Definitions: map[string]json.RawMessage{"beta": def}}, http.StatusOK, &names) + if len(names.Names) != 1 || names.Names[0] != "beta" { + t.Fatalf("reconcile: %+v", names) + } + if c := f.called("ReconcileAgents"); !reflect.DeepEqual(c.Args[0], []string{"beta"}) { + t.Fatalf("reconcile args: %v", c) + } + f.expectCode(http.MethodPost, api.PathReconcile, api.ReconcileRequest{Definitions: map[string]json.RawMessage{"beta": json.RawMessage(`[]`)}}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathReconcile, "x", http.StatusBadRequest, api.CodeBadRequest) + + model, backend := "haiku", "copilot" + f.expect(http.MethodPut, api.OverridesPath("beta"), api.OverridesRequest{Model: &model, Backend: &backend}, http.StatusOK, &resp) + if resp.Process.Model != "haiku" || resp.Process.Backend != "copilot" { + t.Fatalf("overrides: %+v", resp.Process) + } + f.expectCode(http.MethodPut, api.OverridesPath("beta"), api.OverridesRequest{}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPut, api.OverridesPath("beta"), "x", http.StatusBadRequest, api.CodeBadRequest) + f.rt.Err = cruxtest.ErrRecorder + f.expectCode(http.MethodPut, api.OverridesPath("beta"), api.OverridesRequest{Model: &model}, http.StatusInternalServerError, api.CodeRuntime) + f.expectCode(http.MethodPut, api.OverridesPath("beta"), api.OverridesRequest{Backend: &backend}, http.StatusInternalServerError, api.CodeRuntime) + f.rt.Err = nil + + f.expect(http.MethodPut, api.PinPath("beta", api.PinCLI), api.PinRequest{Value: "1.2"}, http.StatusNoContent, nil) + f.called("PinCLI") + f.expect(http.MethodPut, api.PinPath("beta", api.PinModel), api.PinRequest{Value: "opus"}, http.StatusNoContent, nil) + f.called("PinModel") + f.expectCode(http.MethodPut, api.PinPath("beta", "other"), api.PinRequest{Value: "x"}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPut, api.PinPath("beta", api.PinCLI), api.PinRequest{}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPut, api.PinPath("beta", api.PinCLI), "x", http.StatusBadRequest, api.CodeBadRequest) + f.expect(http.MethodDelete, api.PinPath("beta", api.PinCLI), nil, http.StatusNoContent, nil) + f.called("UnpinCLI") + f.expect(http.MethodDelete, api.PinPath("beta", api.PinModel), nil, http.StatusNoContent, nil) + f.called("UnpinModel") + f.expectCode(http.MethodDelete, api.PinPath("beta", "other"), nil, http.StatusBadRequest, api.CodeBadRequest) + f.rt.Err = cruxtest.ErrRecorder + f.expectCode(http.MethodPut, api.PinPath("beta", api.PinCLI), api.PinRequest{Value: "1"}, http.StatusInternalServerError, api.CodeRuntime) + f.expectCode(http.MethodDelete, api.PinPath("beta", api.PinCLI), nil, http.StatusInternalServerError, api.CodeRuntime) + f.rt.Err = nil + + f.expect(http.MethodPost, api.PathModesClear, api.Names{Names: []string{"beta"}}, http.StatusNoContent, nil) + f.called("ClearModeOverrides") + f.expect(http.MethodPost, api.PathModesSync, api.Level{Level: 3}, http.StatusNoContent, nil) + if c := f.called("SyncModeFiles"); c.Args[0] != 3 { + t.Fatalf("sync: %v", c) + } + f.expect(http.MethodPost, api.PathInference, api.Backend{Backend: "claude"}, http.StatusNoContent, nil) + f.called("RefreshInferenceRoutes") + for _, p := range []string{api.PathModesClear, api.PathModesSync, api.PathInference} { + f.expectCode(http.MethodPost, p, "x", http.StatusBadRequest, api.CodeBadRequest) + } + + f.expect(http.MethodDelete, api.AgentPath("beta"), nil, http.StatusNoContent, nil) + if _, ok := f.rt.Procs["beta"]; ok { + t.Fatal("remove not applied") + } + f.expectCode(http.MethodDelete, api.AgentPath("beta"), nil, http.StatusNotFound, api.CodeNotFound) +} + +func TestFleetRoutes(t *testing.T) { + f := newFamServer(t) + f.rt.ACMM = 2 + f.rt.Shared = true + f.rt.WithModel = 5 + f.rt.Wedged = []string{"w"} + var info api.FleetInfo + f.expect(http.MethodGet, api.PathFleet, nil, http.StatusOK, &info) + if info.ACMMLevel != 2 || !info.SharedDevUID || info.AgentsWithModel != 5 || len(info.ConsentWedged) != 1 { + t.Fatalf("fleet: %+v", info) + } + + f.rt.BreakerPaused = []string{"alpha"} + f.rt.Resumed = []string{"alpha"} + var names api.Names + f.expect(http.MethodPost, api.PathBreakerOn, nil, http.StatusOK, &names) + var br api.Breaker + f.expect(http.MethodGet, api.PathBreaker, nil, http.StatusOK, &br) + if !br.Engaged || len(br.Paused) != 1 || len(names.Names) != 1 { + t.Fatalf("breaker: %+v %+v", br, names) + } + f.expect(http.MethodPost, api.PathBreakerOff, nil, http.StatusOK, &names) + f.expect(http.MethodGet, api.PathBreaker, nil, http.StatusOK, &br) + if br.Engaged { + t.Fatal("breaker still engaged") + } + f.expect(http.MethodPut, api.PathBreaker, api.Breaker{Engaged: true, Paused: []string{"x", "y"}}, http.StatusNoContent, nil) + f.expect(http.MethodGet, api.PathBreaker, nil, http.StatusOK, &br) + if !br.Engaged || len(br.Paused) != 2 { + t.Fatalf("restore: %+v", br) + } + f.expectCode(http.MethodPut, api.PathBreaker, "x", http.StatusBadRequest, api.CodeBadRequest) + + f.expect(http.MethodPut, api.PathACMM, api.Level{Level: 4}, http.StatusNoContent, nil) + if f.rt.ACMM != 4 { + t.Fatal("acmm") + } + f.expectCode(http.MethodPut, api.PathACMM, "x", http.StatusBadRequest, api.CodeBadRequest) + f.rt.Relaunched = []string{"bob"} + f.expect(http.MethodPost, api.PathRelaunchBob, nil, http.StatusOK, &names) + if len(names.Names) != 1 { + t.Fatal("relaunch") + } + f.expect(http.MethodPost, api.PathStartupQueued, api.Names{Names: []string{"a", "b"}}, http.StatusNoContent, nil) + if c := f.called("MarkStartupLaunchQueued"); !reflect.DeepEqual(c.Args[0], []string{"a", "b"}) { + t.Fatalf("queued: %v", c) + } + f.expectCode(http.MethodPost, api.PathStartupQueued, "x", http.StatusBadRequest, api.CodeBadRequest) + f.rt.Crashed = []string{"c"} + f.expect(http.MethodPost, api.PathRestartCrashed, nil, http.StatusOK, &names) + if len(names.Names) != 1 || names.Names[0] != "c" { + t.Fatal("crashed") + } + f.expect(http.MethodPost, api.PathTokensRefresh, nil, http.StatusNoContent, nil) + f.called("RefreshAgentTokens") + f.expect(http.MethodPost, api.PathKickLogsArchive, api.Reason{Reason: "shutdown"}, http.StatusNoContent, nil) + if c := f.called("ArchiveAllKickLogs"); c.Args[0] != "shutdown" { + t.Fatalf("archive: %v", c) + } + f.expectCode(http.MethodPost, api.PathKickLogsArchive, "x", http.StatusBadRequest, api.CodeBadRequest) +} + +func TestCredentialRoutes(t *testing.T) { + f := newFamServer(t) + f.rt.Add("alpha", "a1", crux.StateRunning) + f.expect(http.MethodPost, api.LoginCodePath("alpha"), api.LoginCodeRequest{Code: "1234"}, http.StatusNoContent, nil) + if c := f.called("SubmitLoginCode"); c.Args[1] != "1234" { + t.Fatalf("login: %v", c) + } + f.expectCode(http.MethodPost, api.LoginCodePath("alpha"), api.LoginCodeRequest{}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.LoginCodePath("alpha"), "x", http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.LoginCodePath("nope"), api.LoginCodeRequest{Code: "1"}, http.StatusNotFound, api.CodeNotFound) + f.rt.Err = cruxtest.ErrRecorder + f.expectCode(http.MethodPost, api.LoginCodePath("alpha"), api.LoginCodeRequest{Code: "1"}, http.StatusInternalServerError, api.CodeRuntime) + f.expectCode(http.MethodPut, api.PathCopilotCred, api.CopilotCredential{Token: "t", Activate: true}, http.StatusUnprocessableEntity, api.CodeRuntime) + f.rt.Err = nil + + f.rt.CopilotSource = "env" + f.expect(http.MethodPut, api.PathCopilotCred, api.CopilotCredential{Token: "tok"}, http.StatusNoContent, nil) + f.called("SetCopilotToken") + var cred api.CopilotCredential + f.expect(http.MethodGet, api.PathCopilotCred, nil, http.StatusOK, &cred) + if cred.Token != "tok" || cred.Source != "env" { + t.Fatalf("copilot: %+v", cred) + } + f.expect(http.MethodPut, api.PathCopilotCred, api.CopilotCredential{Token: "tok2", Activate: true}, http.StatusNoContent, nil) + f.called("ActivateCopilotToken") + f.expectCode(http.MethodPut, api.PathCopilotCred, "x", http.StatusBadRequest, api.CodeBadRequest) + + f.expect(http.MethodPost, api.PathClaudeReload, nil, http.StatusNoContent, nil) + f.called("ReloadClaudeToken") + f.rt.ClaudePaths = []string{"/a", "/b"} + var cl api.ClaudeCredential + f.expect(http.MethodGet, api.PathClaudeCred, nil, http.StatusOK, &cl) + if len(cl.CandidatePaths) != 2 { + t.Fatalf("claude: %+v", cl) + } +} + +func TestResumerRoutes(t *testing.T) { + f := newFamServer(t) + f.rt.Add("alpha", "a1", crux.StateRunning) + var sess api.Session + f.expect(http.MethodGet, api.SessionPath("alpha"), nil, http.StatusOK, &sess) + if sess.HasSession || sess.Handle != nil || sess.Missing { + t.Fatalf("empty session: %+v", sess) + } + f.rt.Sessions["alpha"] = "s1" + f.rt.Missing["alpha"] = true + f.rt.Handles["alpha"] = crux.ResumeHandle{Backend: "claude", SessionID: "s1", CapturedAt: time.Unix(5, 0)} + f.expect(http.MethodGet, api.SessionPath("alpha"), nil, http.StatusOK, &sess) + if !sess.HasSession || sess.SessionID != "s1" || !sess.Missing || sess.Handle == nil || sess.Handle.Backend != "claude" { + t.Fatalf("session: %+v", sess) + } + f.expectCode(http.MethodGet, api.SessionPath("nope"), nil, http.StatusNotFound, api.CodeNotFound) + + req := api.ResumeKickRequest{Message: "continue", SessionID: "s1"} + var resp api.AgentResponse + f.expect(http.MethodPost, api.SessionKickPath("alpha"), req, http.StatusOK, &resp) + if c := f.called("SendResumeKick"); c.Args[2] != "s1" { + t.Fatalf("resume kick: %v", c) + } + f.expectCode(http.MethodPost, api.SessionKickPath("alpha"), api.ResumeKickRequest{Message: "x"}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.SessionKickPath("alpha"), "x", http.StatusBadRequest, api.CodeBadRequest) + for _, tc := range []struct { + err error + status int + code string + }{ + {crux.ErrResumeBusy, http.StatusConflict, api.CodeResumeBusy}, + {crux.ErrResumeSessionGone, http.StatusGone, api.CodeResumeSessionGone}, + {crux.ErrResumeUnsupported, http.StatusUnprocessableEntity, api.CodeResumeUnsupported}, + {errors.New("tmux exploded"), http.StatusInternalServerError, api.CodeRuntime}, + } { + f.rt.ResumeErr = tc.err + f.expectCode(http.MethodPost, api.SessionKickPath("alpha"), req, tc.status, tc.code) + } +} + +func TestSupervisorRoutes(t *testing.T) { + f := newFamServer(t) + f.rt.Add("alpha", "a1", crux.StateRunning) + var sup api.Supervision + f.expect(http.MethodGet, api.SupervisionPath("alpha"), nil, http.StatusOK, &sup) + if sup.Capabilities != nil || sup.Backend != nil || sup.Invocation != nil || sup.Restarts != nil || sup.StartFailure != nil || sup.ProviderBackoff != nil { + t.Fatalf("unknowns should be absent: %+v", sup) + } + f.rt.MayStart, f.rt.Valid, f.rt.OOM = true, true, true + f.rt.Refreshable, f.rt.RefreshKnown = true, true + f.rt.AuthAvail, f.rt.AuthKnown = false, true + f.rt.Caps = [4]bool{true, false, true, true} + f.rt.Backend, f.rt.BackendOK = "copilot", true + f.rt.Invocation, f.rt.InvocationOK = [3]string{"copilot", "gpt", "high"}, true + f.rt.Restarts.Total, f.rt.Restarts.Last24h, f.rt.Restarts.LastReason, f.rt.Restarts.OK = 3, 1, "crash", true + code := 137 + f.rt.StartFailure, f.rt.StartFailureOK = crux.StartFailureSnapshot{Reason: "oom", Count: 2, LastExitCode: &code, Blocked: true}, true + f.rt.Backoff.Remaining, f.rt.Backoff.Provider, f.rt.Backoff.OK = 1500*time.Millisecond, "anthropic", true + f.expect(http.MethodGet, api.SupervisionPath("alpha"), nil, http.StatusOK, &sup) + switch { + case !sup.MayStart || !sup.HasValidCredential || !sup.CrashOOMSuspected: + t.Fatalf("bools: %+v", sup) + case !sup.CredentialRefreshable || !sup.CredentialKnown || sup.AuthAvailable || !sup.AuthKnown: + t.Fatalf("credential: %+v", sup) + case sup.Capabilities == nil || !sup.Capabilities.CanOpenIssue || sup.Capabilities.CanOpenPR || !sup.Capabilities.CanMerge: + t.Fatalf("caps: %+v", sup.Capabilities) + case sup.Backend == nil || *sup.Backend != "copilot": + t.Fatalf("backend: %+v", sup.Backend) + case sup.Invocation == nil || sup.Invocation.Effort != "high": + t.Fatalf("invocation: %+v", sup.Invocation) + case sup.Restarts == nil || sup.Restarts.Total != 3 || sup.Restarts.LastReason != "crash": + t.Fatalf("restarts: %+v", sup.Restarts) + case sup.StartFailure == nil || sup.StartFailure.LastExitCode == nil || *sup.StartFailure.LastExitCode != 137 || !sup.StartFailure.Blocked: + t.Fatalf("start failure: %+v", sup.StartFailure) + case sup.ProviderBackoff == nil || sup.ProviderBackoff.RemainingMS != 1500 || sup.ProviderBackoff.Provider != "anthropic": + t.Fatalf("backoff: %+v", sup.ProviderBackoff) + } + f.expectCode(http.MethodGet, api.SupervisionPath("nope"), nil, http.StatusNotFound, api.CodeNotFound) + + f.expect(http.MethodPost, api.AgentTokenRefreshPath("alpha"), nil, http.StatusNoContent, nil) + f.called("RefreshAgentTokenFor") + f.rt.Err = cruxtest.ErrRecorder + f.expectCode(http.MethodPost, api.AgentTokenRefreshPath("alpha"), nil, http.StatusInternalServerError, api.CodeRuntime) + f.rt.Err = nil + + f.rt.BackendAuth["claude"] = [2]bool{true, true} + var auth api.BackendAuth + f.expect(http.MethodGet, api.BackendAuthPath("claude"), nil, http.StatusOK, &auth) + if auth.Backend != "claude" || !auth.Available || !auth.Known { + t.Fatalf("backend auth: %+v", auth) + } + f.expect(http.MethodGet, api.BackendAuthPath("codex"), nil, http.StatusOK, &auth) + if auth.Known { + t.Fatal("unknown backend must not be known") + } + + f.expect(http.MethodPost, api.NudgePath("alpha"), api.NudgeRequest{Message: "slow down"}, http.StatusNoContent, nil) + if c := f.called("SendGitHubAPIBudgetNudge"); c.Args[1] != "slow down" { + t.Fatalf("nudge: %v", c) + } + f.expectCode(http.MethodPost, api.NudgePath("alpha"), api.NudgeRequest{}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.NudgePath("alpha"), "x", http.StatusBadRequest, api.CodeBadRequest) + + f.expect(http.MethodPost, api.PathAudit, api.AuditRequest{Action: "pause", Agent: "alpha", Fields: map[string]any{"by": "me"}}, http.StatusNoContent, nil) + if c := f.called("RecordAudit"); c.Args[0] != "pause" || c.Args[2].(map[string]any)["by"] != "me" { + t.Fatalf("audit: %v", c) + } + f.expectCode(http.MethodPost, api.PathAudit, api.AuditRequest{}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathAudit, "x", http.StatusBadRequest, api.CodeBadRequest) + + f.rt.Views = []trajectory.AgentView{{Name: "alpha", Running: true, Intent: "fix", Idle: true, LastOutputAt: time.Unix(9, 0)}} + var tl api.TrajectoryList + f.expect(http.MethodGet, api.PathTrajectory, nil, http.StatusOK, &tl) + if len(tl.Agents) != 1 || tl.Agents[0].Intent != "fix" || !tl.Agents[0].Idle { + t.Fatalf("trajectory: %+v", tl) + } +} + +func TestSeedRoute(t *testing.T) { + f := newFamServer(t) + f.rt.Add("alpha", "a1", crux.StateStopped) + at := time.Unix(100, 0).UTC() + loss, _ := json.Marshal(crux.TurnLoss{Interruptions: 2, Producing: 1}) + req := api.SeedRequest{ + Pause: &api.PauseSeed{PausedAt: at, Trigger: "op", Reason: "r", By: "me"}, + Restarts: &api.RestartSeed{Count: 2, Events: []api.RestartEvent{{At: at, Reason: "crash"}}, LastReason: "crash"}, + TurnLoss: loss, + LastKick: &at, + KickHistory: &[]api.KickRecord{{Timestamp: at, Agent: "alpha", Snippet: "go"}}, + } + f.expect(http.MethodPost, api.SeedPath("alpha"), req, http.StatusNoContent, nil) + for _, m := range []string{"SeedPauseState", "SeedRestartTelemetry", "SeedTurnLoss", "SeedLastKick", "SeedKickHistory"} { + f.called(m) + } + if c := f.called("SeedRestartTelemetry"); c.Args[1] != 2 || len(c.Args[2].([]crux.RestartEvent)) != 1 { + t.Fatalf("restarts: %v", c) + } + if c := f.called("SeedTurnLoss"); c.Args[1].(crux.TurnLoss).Interruptions != 2 { + t.Fatalf("turn loss: %v", c) + } + if c := f.called("SeedKickHistory"); c.Args[1].([]crux.KickRecord)[0].Snippet != "go" { + t.Fatalf("kick history: %v", c) + } + f.rt.Reset() + f.expect(http.MethodPost, api.SeedPath("alpha"), api.SeedRequest{}, http.StatusNoContent, nil) + if len(f.rt.Methods()) != 2 { // ResolveAgent + GetStatusFast only + t.Fatalf("empty seed called %v", f.rt.Methods()) + } + f.expectCode(http.MethodPost, api.SeedPath("alpha"), api.SeedRequest{TurnLoss: json.RawMessage(`[]`)}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.SeedPath("alpha"), "x", http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.SeedPath("nope"), api.SeedRequest{}, http.StatusNotFound, api.CodeNotFound) +} + +func TestAuthorizeRoute(t *testing.T) { + f := newFamServer(t) + for action, method := range map[string]string{ + api.AuthorizePROpen: "AuthorizePROpen", + api.AuthorizePushBranch: "AuthorizePushBranch", + api.AuthorizeIssueOpen: "AuthorizeIssueOpen", + api.AuthorizeMerge: "AuthorizeMerge", + api.AuthorizeReviewRequest: "AuthorizeReviewRequest", + } { + f.expect(http.MethodPost, api.PathAuthorize, api.AuthorizeRequest{Action: action, Agent: "alpha", FileUID: 501, Kind: "bug", Repo: "o/r"}, http.StatusNoContent, nil) + if c := f.called(method); c.Args[0] != "alpha" || c.Args[1] != 501 { + t.Fatalf("%s: %v", action, c) + } + } + if c := f.called("AuthorizeIssueOpen"); c.Args[2] != "bug" { + t.Fatalf("kind: %v", c) + } + if c := f.called("AuthorizeMerge"); c.Args[2] != "o/r" { + t.Fatalf("repo: %v", c) + } + f.rt.AuthorizeErr = errors.New("agent alpha may not merge in o/r") + var e api.Error + f.expect(http.MethodPost, api.PathAuthorize, api.AuthorizeRequest{Action: api.AuthorizeMerge, Agent: "alpha"}, http.StatusForbidden, &e) + if e.Code != api.CodeForbidden || !strings.Contains(e.Error, "may not merge") { + t.Fatalf("deny: %+v", e) + } + f.expectCode(http.MethodPost, api.PathAuthorize, api.AuthorizeRequest{Action: "fly", Agent: "alpha"}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathAuthorize, api.AuthorizeRequest{Action: api.AuthorizePROpen}, http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPost, api.PathAuthorize, "x", http.StatusBadRequest, api.CodeBadRequest) +} + +func TestWatchdogRoutes(t *testing.T) { + f := newFamServer(t) + f.rt.Add("alpha", "a1", crux.StateRunning) + f.rt.WatchdogNames = []string{"alpha"} + f.rt.Observations["alpha"] = watchdog.Observation{Backend: "claude", SessionExists: true, Pane: "❯", HasCLIMarker: true, AuthKnown: true, ProviderErrorClass: "quota"} + var names api.Names + f.expect(http.MethodGet, api.PathWatchdogAgents, nil, http.StatusOK, &names) + if len(names.Names) != 1 { + t.Fatalf("names: %+v", names) + } + var obs api.WatchdogObservation + f.expect(http.MethodGet, api.WatchdogPath("a1"), nil, http.StatusOK, &obs) + if obs.Agent != "alpha" || !obs.SessionExists || !obs.HasCLIMarker || obs.ProviderErrorClass != "quota" || obs.LastProduction != nil { + t.Fatalf("observe: %+v", obs) + } + f.rt.Production["alpha"] = time.Unix(7, 0) + f.expect(http.MethodGet, api.WatchdogPath("alpha"), nil, http.StatusOK, &obs) + if obs.LastProduction == nil || !obs.LastProduction.Equal(time.Unix(7, 0)) { + t.Fatalf("production: %+v", obs) + } + f.expectCode(http.MethodGet, api.WatchdogPath("nope"), nil, http.StatusNotFound, api.CodeNotFound) + + conds := api.Conditions{Conditions: []api.Condition{{Type: "Ready", Status: "False", Reason: "dead"}}} + f.expect(http.MethodPut, api.WatchdogConditionsPath("alpha"), conds, http.StatusNoContent, nil) + if got := f.rt.SetConds["alpha"]; len(got) != 1 || got[0].Type != "Ready" || got[0].Reason != "dead" { + t.Fatalf("conditions: %+v", got) + } + f.expectCode(http.MethodPut, api.WatchdogConditionsPath("alpha"), "x", http.StatusBadRequest, api.CodeBadRequest) + f.expectCode(http.MethodPut, api.WatchdogConditionsPath("nope"), conds, http.StatusNotFound, api.CodeNotFound) + + // A runtime without the seam fails closed on every watchdog route. + plain := New(&noWatchdog{Runtime: f.rt}) + plain.Token = famToken + ts := httptest.NewServer(plain.Handler()) + defer ts.Close() + for _, r := range []struct{ method, path string }{ + {http.MethodGet, api.PathWatchdogAgents}, + {http.MethodGet, api.WatchdogPath("alpha")}, + {http.MethodPut, api.WatchdogConditionsPath("alpha")}, + } { + req, _ := http.NewRequest(r.method, ts.URL+r.path, strings.NewReader("{}")) + req.Header.Set("Authorization", "Bearer "+famToken) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + var e api.Error + _ = json.NewDecoder(resp.Body).Decode(&e) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotImplemented || e.Code != api.CodeNotSupported { + t.Fatalf("%s %s without seam: %d %+v", r.method, r.path, resp.StatusCode, e) + } + } + // So does one whose seam answers nil. + nilSeam := New(&nilWatchdog{Runtime: f.rt}) + nilSeam.Token = famToken + ts2 := httptest.NewServer(nilSeam.Handler()) + defer ts2.Close() + req, _ := http.NewRequest(http.MethodGet, ts2.URL+api.PathWatchdogAgents, nil) + req.Header.Set("Authorization", "Bearer "+famToken) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotImplemented { + t.Fatalf("nil seam: %d", resp.StatusCode) + } +} + +// noWatchdog is a Runtime without WatchdogSource. +type noWatchdog struct{ crux.Runtime } + +// nilWatchdog has the seam but no source behind it. +type nilWatchdog struct{ crux.Runtime } + +func (nilWatchdog) WatchdogSource() watchdog.Source { return nil } + +// TestHostObserversReachTheEventStream runs Serve over a Recorder, fires +// every crux.Host observer, and checks each arrives on /v1/events with its +// typed payload; the audit and condition tees also reach the Server's own +// sinks. Background loops start and are reported by the health route. +func TestHostObserversReachTheEventStream(t *testing.T) { + rt := cruxtest.NewRecorder() + rt.Add("alpha", "a1", crux.StateRunning) + srv := New(rt) + srv.PollInterval = 10 * time.Millisecond + srv.Background = true + audit := &recSink{} + srv.AuditSink = audit + srv.ConditionReporter = audit + socket := shortSocketDir(t) + "/crux.sock" + ls, err := Listen(config.ServeConfig{Socket: socket}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- srv.Serve(ctx, ls) }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + t.Error(err) + } + }) + client := unixClient(socket) + get := func(path string) *http.Response { + t.Helper() + resp, err := client.Get("http://crux" + path) + if err != nil { + t.Fatal(err) + } + return resp + } + + waitUntil(t, "background loops started", func() bool { return len(rt.Background()) == 4 }) + var h api.Health + resp := get(api.PathHealth) + if err := json.NewDecoder(resp.Body).Decode(&h); err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if !reflect.DeepEqual(h.Background, backgroundLoops) { + t.Fatalf("health background: %v", h.Background) + } + + stream := get(api.PathEvents) + waitUntil(t, "subscriber registered", func() bool { + srv.mu.Lock() + defer srv.mu.Unlock() + return len(srv.subscribers) == 1 + }) + ko, po, ro, pp, rp, sa, as, cr := rt.Observers() + if ko == nil || po == nil || ro == nil || pp == nil || rp == nil || sa == nil || as == nil || cr == nil { + t.Fatal("host observers not installed") + } + ko("alpha", crux.KickOutcome{Kind: "ended", At: time.Unix(1, 0)}) + po(crux.PauseTransitionEvent{Agent: "alpha", Paused: true, Trigger: "op", Causation: crux.PauseCausation{Depth: 1}}) + ro("alpha") + pp("alpha", false) + rp("alpha", "cadence", "do the thing") + sa("alpha", "push", "refs/heads/x") + as.Record("system", "pause", "alpha", map[string]any{"k": "v"}) + cr.Observe("agent_runtime", "x", "msg", "diag") + cr.Clear("agent_runtime", "x") + + got := readStream(t, stream, 9) + wantTypes := []string{api.EventKickOutcome, api.EventPauseTransition, api.EventRelaunch, api.EventPausePersist, api.EventPrompt, api.EventSandboxAudit, api.EventAudit, api.EventCondition, api.EventCondition} + for i, w := range wantTypes { + if !strings.HasPrefix(got[i], w+"|") { + t.Fatalf("event %d: want %s got %s", i, w, got[i]) + } + } + var ev api.Event + decode := func(i int) { + t.Helper() + ev = api.Event{} + if err := json.Unmarshal([]byte(got[i][strings.Index(got[i], "|")+1:]), &ev); err != nil { + t.Fatal(err) + } + } + decode(0) + var outcome api.KickOutcome + if ev.Agent != "alpha" || json.Unmarshal(ev.Data, &outcome) != nil || outcome.Kind != "ended" { + t.Fatalf("kick outcome: %+v", ev) + } + decode(1) + var pt api.PauseTransition + if json.Unmarshal(ev.Data, &pt) != nil || !pt.Paused || pt.Causation.Depth != 1 { + t.Fatalf("pause transition: %s", ev.Data) + } + decode(2) + if ev.Agent != "alpha" || len(ev.Data) != 0 { + t.Fatalf("relaunch: %+v", ev) + } + decode(4) + var prompt api.Prompt + if json.Unmarshal(ev.Data, &prompt) != nil || prompt.Prompt != "do the thing" { + t.Fatalf("prompt: %s", ev.Data) + } + decode(6) + var au api.Audit + if json.Unmarshal(ev.Data, &au) != nil || au.Actor != "system" || au.Fields["k"] != "v" { + t.Fatalf("audit: %s", ev.Data) + } + decode(8) + var ce api.ConditionEvent + if json.Unmarshal(ev.Data, &ce) != nil || ce.Action != api.ConditionClear || ce.Key != "x" { + t.Fatalf("condition: %s", ev.Data) + } + if audit.records != 1 || audit.observed != 1 || audit.cleared != 1 { + t.Fatalf("tee: %+v", audit) + } + + cancel() + if err := <-done; err != nil { + t.Fatal(err) + } + done <- nil + // Uninstalled on shutdown; the tees stay (replace-only setters). + ko, po, ro, pp, rp, sa, _, _ = rt.Observers() + if ko != nil || po != nil || ro != nil || pp != nil || rp != nil || sa != nil { + t.Fatal("host observers not uninstalled") + } +} + +type recSink struct{ records, observed, cleared int } + +func (r *recSink) Record(string, string, string, map[string]any) { r.records++ } +func (r *recSink) Observe(string, string, string, string) { r.observed++ } +func (r *recSink) Clear(string, string) { r.cleared++ } + +// TestServeWithoutHostOrBackground covers a Runtime that is not a Host and +// a Server not running the loops: nothing is installed, health lists none. +func TestServeWithoutHostOrBackground(t *testing.T) { + rt := cruxtest.NewRecorder() + srv := New(&noWatchdog{Runtime: rt}) + srv.installHostObservers()() + if _, ok := rt.Last("SetKickOutcomeObserver"); ok { + t.Fatal("observers installed on a non-Host") + } + srv.startBackground(context.Background()) + if len(rt.Background()) != 0 { + t.Fatal("background loops started while disabled") + } + var h api.Health + newFamServer(t).expect(http.MethodGet, api.PathHealth, nil, http.StatusOK, &h) + if len(h.Background) != 0 { + t.Fatalf("health: %+v", h) + } +} + +func TestPublishDataSkipsUnencodable(t *testing.T) { + srv := New(cruxtest.NewRecorder()) + ch := srv.subscribe() + srv.publishData(api.EventAudit, "a", map[string]any{"bad": make(chan int)}) + select { + case ev := <-ch: + t.Fatalf("unexpected event %+v", ev) + default: + } +} + +// TestServerConditionsAndHiveCallbackHealth covers the server's own +// condition reporter (event stream plus the tee) and the hive callback +// status on the health route. +func TestServerConditionsAndHiveCallbackHealth(t *testing.T) { + srv := New(cruxtest.NewRecorder()) + sink := &recSink{} + srv.ConditionReporter = sink + since := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + srv.HiveCallback = func() api.HiveCallback { + return api.HiveCallback{Address: "unix:///run/hive.sock", LastError: "refused", Since: &since, FailedCalls: 2} + } + ch := srv.subscribe() + srv.Conditions().Observe("hive_callback_unavailable", "endpoint", "down", "d") + srv.Conditions().Clear("hive_callback_unavailable", "endpoint") + for i, want := range []string{api.ConditionObserve, api.ConditionClear} { + var ce api.ConditionEvent + select { + case ev := <-ch: + if ev.Type != api.EventCondition || json.Unmarshal(ev.Data, &ce) != nil || ce.Action != want || ce.Class != "hive_callback_unavailable" { + t.Fatalf("event %d: %+v", i, ev) + } + case <-time.After(time.Second): + t.Fatalf("event %d not published", i) + } + } + if sink.observed != 1 || sink.cleared != 1 { + t.Fatalf("tee: %+v", sink) + } + + srv.Token = "t" + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(ts.Close) + req, _ := http.NewRequest(http.MethodGet, ts.URL+api.PathHealth, nil) + req.Header.Set("Authorization", "Bearer t") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var h api.Health + if err := json.NewDecoder(resp.Body).Decode(&h); err != nil { + t.Fatal(err) + } + if h.HiveCallback == nil || h.HiveCallback.Reachable || h.HiveCallback.LastError != "refused" || h.HiveCallback.FailedCalls != 2 || h.HiveCallback.Since == nil || !h.HiveCallback.Since.Equal(since) { + t.Fatalf("health: %+v", h.HiveCallback) + } +} diff --git a/src/crux/serve/serve.go b/src/crux/serve/serve.go new file mode 100644 index 0000000000..072f89f246 --- /dev/null +++ b/src/crux/serve/serve.go @@ -0,0 +1,839 @@ +// Package serve is the server side of the Crux API (Crux Stage 3, +// hivecommons/hive#13536; src/docs/design/crux.md "Stage 3"): an +// http.Handler that exposes a crux.Runtime over the routes crux/api defines, +// and the listener setup `crux serve` uses. It is the process boundary a +// harness other than Hive drives agents through; Hive itself still holds the +// runtime in-process. +// +// Fail-closed rules, enforced here and not left to the caller: a TCP +// listener is refused without a bearer token; on a server with no token, +// Handler refuses every request except those Serve tagged as having arrived +// on the unix socket, so mounting Handler on some other http.Server over TCP +// gets 401s, not an open runtime; the unix socket is created 0600 in a 0700 +// directory the server owns; a token, when set, is required on every route +// including the socket. +package serve + +import ( + "context" + "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" +) + +const ( + // DefaultPollInterval is how often followed logs and the state watcher + // re-read the runtime. + DefaultPollInterval = time.Second + // followWindow is how many ring-buffer lines each follow poll asks for. + followWindow = 200 + // maxRequestBody bounds a JSON request body; a definition is a few KB. + maxRequestBody = 1 << 20 + // subscriberBuffer is how many events a slow subscriber can lag before + // it is dropped rather than block the publisher. + subscriberBuffer = 64 + // shutdownGrace is how long Serve waits for in-flight requests. + shutdownGrace = 5 * time.Second + // maxAgentNameLen matches the CLI's definition-name gate. + maxAgentNameLen = 64 + // networkUnix is the networkKey value Serve tags unix-socket requests + // with; anything else, including no tag, is treated as TCP. + networkUnix = "unix" + // bearerPrefix is the Authorization scheme. + bearerPrefix = "Bearer " +) + +// ctxKey is the context key type for per-listener facts. +type ctxKey int + +// networkKey carries the listener's network ("unix" or "tcp") into each +// request's context; see Listen. +const networkKey ctxKey = iota + +// Server exposes one Runtime. Zero values are usable after Runtime is set; +// construct with New for the defaults. +type Server struct { + // Runtime is what every route acts on. + Runtime crux.Runtime + // Token, when non-empty, is required as a bearer token on every request. + Token string + // Logger receives request errors; nil discards. + Logger *slog.Logger + // PollInterval is the follow/state-watch cadence; zero means + // DefaultPollInterval. + PollInterval time.Duration + // LaunchContext is the context agents are started under; it must outlive + // the request that starts them, since the runtime ties the agent's + // lifetime to it. Serve sets it; a caller using Handler directly sets it + // or gets context.Background. + LaunchContext context.Context + // Background, when set, makes Serve run the crux.Background loops + // (token refresh, credential watchdog, Copilot session refresh, + // transcript activity) in this process; Health lists them. A sidecar + // host's own Background methods are then no-ops. + Background bool + // AuditSink and ConditionReporter, when set, receive the runtime's + // audit records and conditions in addition to the event stream, which + // always carries them (EventAudit, EventCondition). + AuditSink crux.AuditSink + ConditionReporter crux.ConditionReporter + // HiveCallback, when set, is reported on Health: the hive callback + // endpoint's reachability as `crux serve --hive-callback` sees it. + HiveCallback func() api.HiveCallback + + now func() time.Time + started time.Time + + mu sync.Mutex + subscribers map[chan api.Event]struct{} + lastStates map[string]string + background []string +} + +// New returns a Server for rt with the defaults filled in. +func New(rt crux.Runtime) *Server { + s := &Server{Runtime: rt} + s.defaults() + return s +} + +func (s *Server) defaults() { + if s.Logger == nil { + s.Logger = slog.New(slog.DiscardHandler) + } + if s.PollInterval <= 0 { + s.PollInterval = DefaultPollInterval + } + if s.LaunchContext == nil { + s.LaunchContext = context.Background() + } + if s.now == nil { + s.now = time.Now + } + if s.started.IsZero() { + s.started = s.now() + } + if s.subscribers == nil { + s.subscribers = map[chan api.Event]struct{}{} + } + if s.lastStates == nil { + s.lastStates = map[string]string{} + } +} + +// Handler returns the API routes behind the auth check. +func (s *Server) Handler() http.Handler { + s.defaults() + mux := http.NewServeMux() + mux.HandleFunc("GET "+api.PathHealth, s.handleHealth) + mux.HandleFunc("GET "+api.PathAgents, s.handleList) + mux.HandleFunc("POST "+api.PathAgents, s.handleRun) + mux.HandleFunc("GET "+api.PathAgents+"/{name}", s.handleGet) + mux.HandleFunc("GET "+api.PathAgents+"/{name}/logs", s.handleLogs) + mux.HandleFunc("POST "+api.PathAgents+"/{name}/kick", s.handleKick) + mux.HandleFunc("POST "+api.PathAgents+"/{name}/stop", s.handleStop) + mux.HandleFunc("GET "+api.PathEvents, s.handleEvents) + s.registerFamilies(mux) + return s.authenticate(mux) +} + +// agentNamePattern is the CLI's definition-name gate (cmd/crux +// validateDefinitionName): the name becomes a tmux socket and per-agent +// paths, so it must be a plain identifier. +var agentNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) + +// authenticate is the fail-closed gate: when the server has no token, only +// a request Serve tagged as arriving on the unix socket is allowed, so a +// Handler mounted elsewhere (any TCP listener, tagged or not) is refused; +// when the server has a token, every request must present it. +func (s *Server) authenticate(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + network, _ := r.Context().Value(networkKey).(string) + if s.Token == "" { + if network != networkUnix { + s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "this server has no bearer token; only unix-socket requests are accepted") + return + } + next.ServeHTTP(w, r) + return + } + got, ok := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) + if !ok || subtle.ConstantTimeCompare([]byte(got), []byte(s.Token)) != 1 { + w.Header().Set("WWW-Authenticate", `Bearer realm="crux"`) + s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "missing or invalid bearer token") + return + } + next.ServeHTTP(w, r) + }) +} + +// ownedPrivateDir refuses a socket directory this user does not own +// outright: MkdirAll is silent about a directory that already existed, and +// on a shared temp dir another user could have created it first and could +// then swap the socket. The directory must be a real directory (not a +// symlink), owned by this uid, with no group or other permission bits. +func ownedPrivateDir(dir string) error { + fi, err := os.Lstat(dir) + if err != nil { + return err + } + if fi.Mode()&os.ModeSymlink != 0 { + return errors.New("is a symlink") + } + if !fi.IsDir() { + return errors.New("is not a directory") + } + if perm := fi.Mode().Perm(); perm&0o077 != 0 { + return fmt.Errorf("mode %04o is readable by others; want %04o", perm, config.ServeSocketDirMode) + } + if st, ok := fi.Sys().(*syscall.Stat_t); ok && int(st.Uid) != os.Getuid() { + return fmt.Errorf("owned by uid %d, not this user (uid %d)", st.Uid, os.Getuid()) + } + return nil +} + +// Listeners is what Listen opened: the unix socket and, when configured, the +// TCP listener. Close releases both and removes the socket file. +type Listeners struct { + Unix net.Listener + TCP net.Listener + path string +} + +// Addrs lists the bound addresses for logging. +func (l *Listeners) Addrs() []string { + var out []string + if l.Unix != nil { + out = append(out, "unix://"+l.path) + } + if l.TCP != nil { + out = append(out, "http://"+l.TCP.Addr().String()) + } + return out +} + +// Close closes every listener and removes the socket file. +func (l *Listeners) Close() { + if l.Unix != nil { + _ = l.Unix.Close() + _ = os.Remove(l.path) + } + if l.TCP != nil { + _ = l.TCP.Close() + } +} + +// Listen validates cfg and binds: the unix socket (its directory created +// 0700 if missing, a stale socket file replaced, the socket chmod 0600) and +// the optional TCP address. A socket something is still listening on is an +// error so two servers never fight over one path. +func Listen(cfg config.ServeConfig) (*Listeners, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + dir := filepath.Dir(cfg.Socket) + if err := os.MkdirAll(dir, config.ServeSocketDirMode); err != nil { + return nil, fmt.Errorf("socket directory: %w", err) + } + if err := ownedPrivateDir(dir); err != nil { + return nil, fmt.Errorf("socket directory %s: %w", dir, err) + } + if conn, err := net.DialTimeout("unix", cfg.Socket, time.Second); err == nil { + _ = conn.Close() + return nil, fmt.Errorf("socket %s: another crux serve is already listening", cfg.Socket) + } + // Bind under a temporary name, set the mode, then rename into place, so + // the socket is never reachable at its advertised path with the default + // (umask-derived) permissions. + tmp := fmt.Sprintf("%s.%d.tmp", cfg.Socket, os.Getpid()) + _ = os.Remove(tmp) + ul, err := net.Listen("unix", tmp) + if err != nil { + return nil, fmt.Errorf("socket %s: %w", cfg.Socket, err) + } + if err := os.Chmod(tmp, config.ServeSocketMode); err != nil { + _ = ul.Close() + _ = os.Remove(tmp) + return nil, fmt.Errorf("socket %s: chmod: %w", cfg.Socket, err) + } + if err := os.Rename(tmp, cfg.Socket); err != nil { + _ = ul.Close() + _ = os.Remove(tmp) + return nil, fmt.Errorf("socket %s: %w", cfg.Socket, err) + } + ls := &Listeners{Unix: ul, path: cfg.Socket} + if cfg.Listen != "" { + tl, err := net.Listen("tcp", cfg.Listen) + if err != nil { + ls.Close() + return nil, fmt.Errorf("listen %s: %w", cfg.Listen, err) + } + ls.TCP = tl + } + return ls, nil +} + +// Serve runs the API on ls until ctx ends, then shuts down and closes ls. +// It starts the state watcher and installs the kick relay for the lifetime +// of the call. The Token is taken from s, not cfg; callers set both from +// the same source (cli.cmdServe does). +func (s *Server) Serve(ctx context.Context, ls *Listeners) error { + s.defaults() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + s.LaunchContext = ctx + s.Runtime.SetKickObserver(func(agentName, event, detail string) { + s.publish(api.Event{Type: api.EventKick, Agent: agentName, State: event, Detail: detail}) + }) + defer s.Runtime.SetKickObserver(nil) + defer s.installHostObservers()() + s.startBackground(ctx) + go s.watchStates(ctx) + + handler := s.Handler() + errc := make(chan error, 2) + var servers []*http.Server + serveOn := func(network string, l net.Listener) { + srv := &http.Server{ + Handler: handler, + ReadHeaderTimeout: 10 * time.Second, + BaseContext: func(net.Listener) context.Context { + return context.WithValue(ctx, networkKey, network) + }, + } + servers = append(servers, srv) + go func() { + if err := srv.Serve(l); err != nil && !errors.Is(err, http.ErrServerClosed) { + errc <- fmt.Errorf("%s: %w", network, err) + return + } + errc <- nil + }() + } + if ls.Unix != nil { + serveOn("unix", ls.Unix) + } + if ls.TCP != nil { + serveOn("tcp", ls.TCP) + } + var err error + select { + case <-ctx.Done(): + case err = <-errc: + cancel() + } + for _, srv := range servers { + sctx, scancel := context.WithTimeout(context.Background(), shutdownGrace) + _ = srv.Shutdown(sctx) + scancel() + } + ls.Close() + return err +} + +// ---- handlers ---- + +func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { + s.mu.Lock() + background := append([]string(nil), s.background...) + s.mu.Unlock() + now := s.now() + h := api.Health{Version: api.Version, Agents: len(s.Runtime.AllStatuses()), Background: background, StartedAt: s.started, UptimeSeconds: now.Sub(s.started).Seconds()} + if s.HiveCallback != nil { + hc := s.HiveCallback() + h.HiveCallback = &hc + } + s.writeJSON(w, http.StatusOK, h) +} + +func (s *Server) handleList(w http.ResponseWriter, _ *http.Request) { + statuses := s.Runtime.AllStatuses() + out := api.ProcessList{Processes: make([]api.Process, 0, len(statuses))} + for _, p := range statuses { + if p == nil { + continue + } + out.Processes = append(out.Processes, toWire(p)) + } + sort.Slice(out.Processes, func(i, j int) bool { return out.Processes[i].Name < out.Processes[j].Name }) + s.writeJSON(w, http.StatusOK, out) +} + +func (s *Server) handleRun(w http.ResponseWriter, r *http.Request) { + var req api.RunRequest + if !s.readJSON(w, r, &req) { + return + } + if strings.TrimSpace(req.Name) == "" { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "name is required") + return + } + name := req.Name + if len(req.Definition) > 0 { + if len(name) > maxAgentNameLen || !agentNamePattern.MatchString(name) { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, + fmt.Sprintf("name must match %s and be at most %d characters", agentNamePattern, maxAgentNameLen)) + return + } + cfg, err := decodeDefinition(name, req.Definition) + if err != nil { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "definition: "+err.Error()) + return + } + // The runtime's AddAgent is a no-op for a name it already holds, so a + // re-post with a changed definition would silently start the old one. + if _, err := s.Runtime.GetStatus(name); err == nil { + s.writeError(w, http.StatusConflict, api.CodeConflict, + fmt.Sprintf("agent %q is already defined on this server; post without a definition to start it", name)) + return + } + s.Runtime.AddAgent(name, cfg) + if _, err := s.Runtime.GetStatus(name); err != nil { + s.writeError(w, http.StatusUnprocessableEntity, api.CodeBadRequest, + fmt.Sprintf("agent %q was not accepted by the runtime: %v", name, err)) + return + } + } else { + name = s.Runtime.ResolveAgent(name) + if _, err := s.Runtime.GetStatus(name); err != nil { + s.writeError(w, http.StatusNotFound, api.CodeNotFound, fmt.Sprintf("agent %q is not defined on this server; send a definition to run one", req.Name)) + return + } + } + if err := s.Runtime.Start(s.LaunchContext, name); err != nil { + s.writeError(w, http.StatusInternalServerError, api.CodeRuntime, fmt.Sprintf("start %s: %v", name, err)) + return + } + s.publish(api.Event{Type: api.EventStarted, Agent: name}) + s.writeJSON(w, http.StatusCreated, s.agentResponse(name)) +} + +// decodeDefinition parses a RunRequest definition and fills the per-agent +// defaults a start depends on, the way the CLI's loader does with +// Config.ApplyAgentDefaults (which crux/config does not have): ID and Role +// default to the name, Replicas to 1, and Enabled to true unless the +// document set it. Everything else is the caller's. +func decodeDefinition(name string, raw json.RawMessage) (config.AgentConfig, error) { + var cfg config.AgentConfig + if err := json.Unmarshal(raw, &cfg); err != nil { + return cfg, err + } + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + return cfg, err + } + if cfg.ID == "" { + cfg.ID = name + } + if cfg.Role == "" { + cfg.Role = name + } + if cfg.Replicas == 0 { + cfg.Replicas = 1 + } + if _, set := keys["enabled"]; !set { + cfg.Enabled = true + } + return cfg, nil +} + +func (s *Server) handleGet(w http.ResponseWriter, r *http.Request) { + name := s.Runtime.ResolveAgent(r.PathValue("name")) + p, err := s.Runtime.GetStatus(name) + if err != nil || p == nil { + s.writeError(w, http.StatusNotFound, api.CodeNotFound, fmt.Sprintf("agent %q not found", name)) + return + } + s.writeJSON(w, http.StatusOK, toWire(p)) +} + +func (s *Server) handleLogs(w http.ResponseWriter, r *http.Request) { + name := s.Runtime.ResolveAgent(r.PathValue("name")) + q := r.URL.Query() + lines := api.DefaultLogLines + if v := q.Get(api.QueryLines); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, api.QueryLines+" must be a non-negative integer") + return + } + lines = n + } + if isTrue(q.Get(api.QueryFollow)) { + s.followLogs(w, r, name) + return + } + source := q.Get(api.QuerySource) + if source == "" { + source = api.LogSourcePane + } + out := api.Logs{Agent: name, Source: source} + var err error + switch source { + case api.LogSourcePane: + out.Lines, err = s.Runtime.GetOutput(name, lines) + case api.LogSourceBuffer: + out.Lines, err = s.Runtime.GetBufferOutput(name, lines) + case api.LogSourceFull: + out.Header = s.Runtime.FullLogHeader(name) + out.Content, err = s.Runtime.CaptureFullLog(name) + default: + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, fmt.Sprintf("%s must be %s, %s or %s", api.QuerySource, api.LogSourcePane, api.LogSourceBuffer, api.LogSourceFull)) + return + } + if err != nil { + s.writeError(w, http.StatusInternalServerError, api.CodeRuntime, fmt.Sprintf("logs %s: %v", name, err)) + return + } + s.writeJSON(w, http.StatusOK, out) +} + +// followLogs streams new ring-buffer lines as api.EventLog records until the +// client goes away or the runtime stops answering. +func (s *Server) followLogs(w http.ResponseWriter, r *http.Request, name string) { + flusher, ok := w.(http.Flusher) + if !ok { + s.writeError(w, http.StatusInternalServerError, api.CodeRuntime, "streaming is not supported on this connection") + return + } + prev, err := s.Runtime.GetBufferOutput(name, followWindow) + if err != nil { + s.writeError(w, http.StatusInternalServerError, api.CodeRuntime, fmt.Sprintf("logs %s: %v", name, err)) + return + } + s.startStream(w) + flusher.Flush() + if len(prev) > 0 { + if api.WriteEvent(w, api.EventLog, api.LogChunk{Agent: name, Lines: prev}) != nil { + return + } + flusher.Flush() + } + ticker := time.NewTicker(s.PollInterval) + defer ticker.Stop() + for { + select { + case <-r.Context().Done(): + return + case <-ticker.C: + cur, err := s.Runtime.GetBufferOutput(name, followWindow) + if err != nil { + _ = api.WriteEvent(w, api.EventState, api.Event{Time: s.now(), Type: api.EventState, Agent: name, Detail: err.Error()}) + flusher.Flush() + return + } + if fresh := newTail(prev, cur); len(fresh) > 0 { + if api.WriteEvent(w, api.EventLog, api.LogChunk{Agent: name, Lines: fresh}) != nil { + return + } + flusher.Flush() + } + prev = cur + } + } +} + +func (s *Server) handleKick(w http.ResponseWriter, r *http.Request) { + name := s.Runtime.ResolveAgent(r.PathValue("name")) + var req api.KickRequest + if !s.readJSON(w, r, &req) { + return + } + if strings.TrimSpace(req.Message) == "" { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "message is empty") + return + } + if req.Async { + started, err := s.Runtime.SendKickAsync(name, req.Message) + if err != nil { + s.writeRuntimeError(w, name, "kick", err) + return + } + resp := s.agentResponse(name) + resp.Started = &started + s.writeJSON(w, http.StatusAccepted, resp) + return + } + var err error + if req.Source != "" { + err = s.Runtime.SendKickWithSource(name, req.Message, req.Source) + } else { + err = s.Runtime.SendKick(name, req.Message) + } + if err != nil { + s.writeRuntimeError(w, name, "kick", err) + return + } + s.publish(api.Event{Type: api.EventKicked, Agent: name}) + s.writeJSON(w, http.StatusOK, s.agentResponse(name)) +} + +func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { + name := s.Runtime.ResolveAgent(r.PathValue("name")) + if err := s.Runtime.Stop(name); err != nil { + s.writeRuntimeError(w, name, "stop", err) + return + } + s.publish(api.Event{Type: api.EventStopped, Agent: name}) + s.writeJSON(w, http.StatusOK, s.agentResponse(name)) +} + +// handleEvents streams runtime events until the client goes away. +func (s *Server) handleEvents(w http.ResponseWriter, r *http.Request) { + flusher, ok := w.(http.Flusher) + if !ok { + s.writeError(w, http.StatusInternalServerError, api.CodeRuntime, "streaming is not supported on this connection") + return + } + ch := s.subscribe() + defer s.unsubscribe(ch) + s.startStream(w) + flusher.Flush() + keepalive := time.NewTicker(s.PollInterval * 15) + defer keepalive.Stop() + for { + select { + case <-r.Context().Done(): + return + case ev, ok := <-ch: + if !ok { + return + } + if api.WriteEvent(w, ev.Type, ev) != nil { + return + } + flusher.Flush() + case <-keepalive.C: + if _, err := fmt.Fprint(w, ": keep-alive\n\n"); err != nil { + return + } + flusher.Flush() + } + } +} + +// ---- events ---- + +func (s *Server) subscribe() chan api.Event { + ch := make(chan api.Event, subscriberBuffer) + s.mu.Lock() + s.subscribers[ch] = struct{}{} + s.mu.Unlock() + return ch +} + +func (s *Server) unsubscribe(ch chan api.Event) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.subscribers[ch]; ok { + delete(s.subscribers, ch) + close(ch) + } +} + +// publish stamps ev and fans it out. A subscriber whose buffer is full is +// dropped (its channel closed, which ends its stream) rather than allowed to +// stall the runtime's callbacks. +func (s *Server) publish(ev api.Event) { + if ev.Time.IsZero() { + ev.Time = s.now() + } + s.mu.Lock() + defer s.mu.Unlock() + for ch := range s.subscribers { + select { + case ch <- ev: + default: + delete(s.subscribers, ch) + close(ch) + } + } +} + +// watchStates polls AllStatuses and publishes an EventState for every agent +// whose state differs from the last poll, which is how crashes, restarts +// and pauses the API did not cause reach subscribers. +func (s *Server) watchStates(ctx context.Context) { + ticker := time.NewTicker(s.PollInterval) + defer ticker.Stop() + s.diffStates(false) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.diffStates(true) + } + } +} + +// diffStates records the current states and, when publishChanges is set, +// publishes the transitions since the previous call. The first call seeds +// the table silently so a fresh subscriber is not told about history. +func (s *Server) diffStates(publishChanges bool) { + statuses := s.Runtime.AllStatuses() + var changes []api.Event + s.mu.Lock() + seen := make(map[string]bool, len(statuses)) + for name, p := range statuses { + if p == nil { + continue + } + seen[name] = true + state := string(p.State) + if p.Paused { + state = string(crux.StatePaused) + } + if prev, ok := s.lastStates[name]; publishChanges && (!ok || prev != state) { + changes = append(changes, api.Event{Type: api.EventState, Agent: name, State: state, Detail: p.LastRestartReason}) + } + s.lastStates[name] = state + } + for name := range s.lastStates { + if !seen[name] { + delete(s.lastStates, name) + } + } + s.mu.Unlock() + sort.Slice(changes, func(i, j int) bool { return changes[i].Agent < changes[j].Agent }) + for _, ev := range changes { + s.publish(ev) + } +} + +// ---- helpers ---- + +func (s *Server) agentResponse(name string) api.AgentResponse { + resp := api.AgentResponse{Agent: name} + if p, err := s.Runtime.GetStatus(name); err == nil && p != nil { + w := toWire(p) + resp.Process = &w + } + return resp +} + +// toWire projects a runtime snapshot onto the API's Process. +func toWire(p *crux.Process) api.Process { + backend := p.Config.Backend + if p.BackendOverride != "" { + backend = p.BackendOverride + } + model := p.Config.Model + if p.ModelOverride != "" { + model = p.ModelOverride + } + return api.Process{ + Name: p.Name, + ID: p.ID, + State: string(p.State), + Backend: backend, + Model: model, + PID: p.PID, + Paused: p.Paused, + PausedReason: p.PausedReason, + PausedTrigger: p.PausedTrigger, + PausedBy: p.PausedBy, + StartedAt: p.StartedAt, + LastKick: p.LastKick, + RestartCount: p.RestartCount, + LastRestartReason: p.LastRestartReason, + KickRefused: p.KickRefused, + KickRefusalReason: p.KickRefusalReason, + KicksUndeliverable: p.KicksUndeliverable, + BusyCondition: p.BusyCondition, + StartFailureClass: p.StartFailureClass, + } +} + +// writeRuntimeError maps a runtime error onto a status: "not found" is a +// 404, everything else a 500 with the runtime's message. +func (s *Server) writeRuntimeError(w http.ResponseWriter, name, verb string, err error) { + status, code := http.StatusInternalServerError, api.CodeRuntime + if strings.Contains(err.Error(), "not found") { + status, code = http.StatusNotFound, api.CodeNotFound + } + s.writeError(w, status, code, fmt.Sprintf("%s %s: %v", verb, name, err)) +} + +func (s *Server) readJSON(w http.ResponseWriter, r *http.Request, v any) bool { + dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxRequestBody)) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "request body: "+err.Error()) + return false + } + return true +} + +func (s *Server) writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", api.ContentTypeJSON) + w.WriteHeader(status) + if err := json.NewEncoder(w).Encode(v); err != nil { + s.Logger.Warn("crux serve: writing response", "err", err) + } +} + +func (s *Server) writeError(w http.ResponseWriter, status int, code, msg string) { + s.writeJSON(w, status, api.Error{Error: msg, Code: code}) +} + +func (s *Server) startStream(w http.ResponseWriter) { + w.Header().Set("Content-Type", api.ContentTypeEvent) + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Accel-Buffering", "no") + w.WriteHeader(http.StatusOK) +} + +func isTrue(v string) bool { + switch strings.ToLower(v) { + case "1", "true", "yes", "on": + return true + } + return false +} + +// newTail returns the lines of cur that were not already seen in prev, +// treating both as windows onto the same append-only buffer: the longest +// suffix of prev that is a prefix of cur is the overlap. It is the CLI's +// stream rule (crux/cli.newTail) applied server side. +func newTail(prev, cur []string) []string { + max := len(prev) + if len(cur) < max { + max = len(cur) + } + for k := max; k > 0; k-- { + if equalLines(prev[len(prev)-k:], cur[:k]) { + return cur[k:] + } + } + return cur +} + +func equalLines(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/src/crux/serve/serve_test.go b/src/crux/serve/serve_test.go new file mode 100644 index 0000000000..e4c70f4446 --- /dev/null +++ b/src/crux/serve/serve_test.go @@ -0,0 +1,1096 @@ +package serve + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/config" +) + +// fakeRuntime answers the Runtime methods the server uses from canned state +// and records what it was asked. Anything else panics through the embedded +// nil interface, so a handler reaching past its documented methods fails +// loudly. +type fakeRuntime struct { + crux.Runtime + + mu sync.Mutex + procs map[string]*crux.Process + buffer map[string][][]string + pane map[string][]string + fullLog map[string]string + header map[string]string + kicks []string + startErr error + outputErr error + observer func(string, string, string) + startCtx context.Context +} + +func newFakeRuntime() *fakeRuntime { + return &fakeRuntime{ + procs: map[string]*crux.Process{}, + buffer: map[string][][]string{}, + pane: map[string][]string{}, + fullLog: map[string]string{}, + header: map[string]string{}, + } +} + +func (f *fakeRuntime) add(name, id string, state crux.ProcessState) *crux.Process { + f.mu.Lock() + defer f.mu.Unlock() + p := &crux.Process{Name: name, ID: id, State: state, Config: config.AgentConfig{ID: id, Backend: "claude", Model: "opus"}} + f.procs[name] = p + return p +} + +// AddAgent matches the manager: a no-op for a name it already holds (and, +// standing in for the ACMM gate, for the name "refused"). +func (f *fakeRuntime) AddAgent(name string, cfg config.AgentConfig) { + f.mu.Lock() + defer f.mu.Unlock() + if _, exists := f.procs[name]; exists || name == "refused" { + return + } + f.procs[name] = &crux.Process{Name: name, ID: cfg.ID, Config: cfg, State: crux.StateStopped} +} + +func (f *fakeRuntime) Start(ctx context.Context, name string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.startCtx = ctx + if f.startErr != nil { + return f.startErr + } + p, ok := f.procs[name] + if !ok { + return fmt.Errorf("agent %s not found", name) + } + p.State = crux.StateRunning + now := time.Now() + p.StartedAt = &now + return nil +} + +func (f *fakeRuntime) Stop(name string) error { + f.mu.Lock() + defer f.mu.Unlock() + p, ok := f.procs[name] + if !ok { + return fmt.Errorf("agent %s not found", name) + } + if p.State == crux.StateFailed { + return errors.New("tmux kill-session: exit status 1") + } + p.State = crux.StateStopped + return nil +} + +func (f *fakeRuntime) AllStatuses() map[string]*crux.Process { + f.mu.Lock() + defer f.mu.Unlock() + out := make(map[string]*crux.Process, len(f.procs)) + for k, v := range f.procs { + out[k] = clone(v) + } + out["nil-entry"] = nil + return out +} + +func (f *fakeRuntime) GetStatus(name string) (*crux.Process, error) { + f.mu.Lock() + defer f.mu.Unlock() + p, ok := f.procs[name] + if !ok { + return nil, fmt.Errorf("agent %s not found", name) + } + return clone(p), nil +} + +// clone copies the fields the server projects; crux.Process carries a mutex +// and cannot be copied whole. +func clone(p *crux.Process) *crux.Process { + return &crux.Process{ + Name: p.Name, ID: p.ID, Config: p.Config, State: p.State, PID: p.PID, + StartedAt: p.StartedAt, LastKick: p.LastKick, Paused: p.Paused, + PausedReason: p.PausedReason, PausedTrigger: p.PausedTrigger, PausedBy: p.PausedBy, + ModelOverride: p.ModelOverride, BackendOverride: p.BackendOverride, + RestartCount: p.RestartCount, LastRestartReason: p.LastRestartReason, + KickRefused: p.KickRefused, KickRefusalReason: p.KickRefusalReason, + KicksUndeliverable: p.KicksUndeliverable, BusyCondition: p.BusyCondition, + StartFailureClass: p.StartFailureClass, + } +} + +func (f *fakeRuntime) ResolveAgent(nameOrID string) string { + f.mu.Lock() + defer f.mu.Unlock() + for name, p := range f.procs { + if p.ID == nameOrID { + return name + } + } + return nameOrID +} + +func (f *fakeRuntime) GetOutput(name string, lines int) ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.outputErr != nil { + return nil, f.outputErr + } + out := f.pane[name] + if lines < len(out) { + out = out[len(out)-lines:] + } + return out, nil +} + +func (f *fakeRuntime) GetBufferOutput(name string, _ int) ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.outputErr != nil { + return nil, f.outputErr + } + answers := f.buffer[name] + if len(answers) == 0 { + return nil, nil + } + cur := answers[0] + if len(answers) > 1 { + f.buffer[name] = answers[1:] + } + return cur, nil +} + +func (f *fakeRuntime) CaptureFullLog(name string) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.outputErr != nil { + return "", f.outputErr + } + return f.fullLog[name], nil +} + +func (f *fakeRuntime) FullLogHeader(name string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.header[name] +} + +func (f *fakeRuntime) SendKick(name, message string) error { + f.mu.Lock() + defer f.mu.Unlock() + p, ok := f.procs[name] + if !ok { + return fmt.Errorf("agent %s not found", name) + } + if p.State != crux.StateRunning { + return errors.New("agent is not running") + } + f.kicks = append(f.kicks, message) + return nil +} + +func (f *fakeRuntime) SetKickObserver(fn func(agentName, event, detail string)) { + f.mu.Lock() + defer f.mu.Unlock() + f.observer = fn +} + +func (f *fakeRuntime) Kicks() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.kicks...) +} + +// shortSocketDir returns a directory short enough for a unix socket on every +// platform (macOS caps sun_path at 104 bytes and t.TempDir can exceed it). +func shortSocketDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("", "crux") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return dir +} + +// unixClient is an http.Client that dials the socket at path. +func unixClient(path string) *http.Client { + return &http.Client{Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", path) + }, + }} +} + +// served starts a Server over a real unix socket (and TCP when listen is +// set) and returns it with its runtime, socket path and TCP address. +type served struct { + t *testing.T + rt *fakeRuntime + srv *Server + socket string + tcp string + client *http.Client + cancel context.CancelFunc + done chan error +} + +func startServer(t *testing.T, token string, tcp bool) *served { + return startServerPolling(t, token, tcp, 10*time.Millisecond) +} + +func startServerPolling(t *testing.T, token string, tcp bool, poll time.Duration) *served { + t.Helper() + rt := newFakeRuntime() + srv := New(rt) + srv.Token = token + srv.PollInterval = poll + cfg := config.ServeConfig{Socket: filepath.Join(shortSocketDir(t), "crux.sock"), Token: token} + if tcp { + cfg.Listen = "127.0.0.1:0" + } + ls, err := Listen(cfg) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + s := &served{t: t, rt: rt, srv: srv, socket: cfg.Socket, client: unixClient(cfg.Socket), cancel: cancel, done: make(chan error, 1)} + if ls.TCP != nil { + s.tcp = "http://" + ls.TCP.Addr().String() + } + go func() { s.done <- srv.Serve(ctx, ls) }() + t.Cleanup(func() { + cancel() + select { + case err := <-s.done: + if err != nil { + t.Errorf("Serve: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("Serve did not return") + } + }) + return s +} + +func (s *served) do(method, path string, body any) (*http.Response, []byte) { + s.t.Helper() + var rd io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + s.t.Fatal(err) + } + rd = bytes.NewReader(b) + } + req, err := http.NewRequest(method, "http://crux"+path, rd) + if err != nil { + s.t.Fatal(err) + } + if s.srv.Token != "" { + req.Header.Set("Authorization", "Bearer "+s.srv.Token) + } + resp, err := s.client.Do(req) + if err != nil { + s.t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + data, err := io.ReadAll(resp.Body) + if err != nil { + s.t.Fatal(err) + } + return resp, data +} + +func (s *served) expect(method, path string, body any, status int, into any) { + s.t.Helper() + resp, data := s.do(method, path, body) + if resp.StatusCode != status { + s.t.Fatalf("%s %s: status %d body %s", method, path, resp.StatusCode, data) + } + if into != nil { + if err := json.Unmarshal(data, into); err != nil { + s.t.Fatalf("%s %s: %v in %s", method, path, err, data) + } + } +} + +func TestListenCreatesAnOwnerOnlySocketAndRefusesADouble(t *testing.T) { + dir := shortSocketDir(t) + cfg := config.ServeConfig{Socket: filepath.Join(dir, "nested", "crux.sock")} + ls, err := Listen(cfg) + if err != nil { + t.Fatal(err) + } + defer ls.Close() + fi, err := os.Stat(cfg.Socket) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != config.ServeSocketMode { + t.Fatalf("socket mode %o, want %o", fi.Mode().Perm(), config.ServeSocketMode) + } + di, _ := os.Stat(filepath.Dir(cfg.Socket)) + if di.Mode().Perm() != config.ServeSocketDirMode { + t.Fatalf("socket dir mode %o, want %o", di.Mode().Perm(), config.ServeSocketDirMode) + } + if ls.TCP != nil || len(ls.Addrs()) != 1 || ls.Addrs()[0] != "unix://"+cfg.Socket { + t.Fatalf("Addrs = %v", ls.Addrs()) + } + + // Something is listening: a second Listen must refuse, not steal. + go func() { + conn, err := ls.Unix.Accept() + if err == nil { + _ = conn.Close() + } + }() + if _, err := Listen(cfg); err == nil || !strings.Contains(err.Error(), "already listening") { + t.Fatalf("second Listen: %v", err) + } + + // A stale socket file (nobody answering) is replaced. + ls.Close() + if err := os.WriteFile(cfg.Socket, nil, 0o600); err != nil { + t.Fatal(err) + } + ls2, err := Listen(cfg) + if err != nil { + t.Fatalf("stale socket should be replaced: %v", err) + } + ls2.Close() + if _, err := os.Stat(cfg.Socket); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("Close should remove the socket, stat err = %v", err) + } +} + +func TestListenFailsClosed(t *testing.T) { + // TCP without a token is refused before anything binds. + if _, err := Listen(config.ServeConfig{Socket: filepath.Join(shortSocketDir(t), "a.sock"), Listen: "127.0.0.1:0"}); err == nil || !strings.Contains(err.Error(), "bearer token") { + t.Fatalf("TCP without token: %v", err) + } + // An unbindable TCP address releases the socket it already took. + dir := shortSocketDir(t) + sock := filepath.Join(dir, "b.sock") + if _, err := Listen(config.ServeConfig{Socket: sock, Listen: "256.256.256.256:1", Token: "t"}); err == nil || !strings.Contains(err.Error(), "listen") { + t.Fatalf("bad TCP address: %v", err) + } + if _, err := os.Stat(sock); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("socket should be released after a TCP bind failure, stat err = %v", err) + } + // A socket path whose directory cannot be created. + file := filepath.Join(dir, "file") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := Listen(config.ServeConfig{Socket: filepath.Join(file, "c.sock")}); err == nil || !strings.Contains(err.Error(), "socket") { + t.Fatalf("unmakeable dir: %v", err) + } + // A path that cannot be listened on (too long for sun_path). + if _, err := Listen(config.ServeConfig{Socket: filepath.Join(dir, strings.Repeat("x", 120)+".sock")}); err == nil { + t.Fatal("over-long socket path should fail") + } + // A socket directory this user does not own outright is refused: one + // readable by others (a squatter could have created it)... + loose := filepath.Join(dir, "loose") + if err := os.Mkdir(loose, 0o755); err != nil { + t.Fatal(err) + } + if _, err := Listen(config.ServeConfig{Socket: filepath.Join(loose, "d.sock")}); err == nil || !strings.Contains(err.Error(), "readable by others") { + t.Fatalf("world-readable dir: %v", err) + } + // ...or reached through a symlink. + private := filepath.Join(dir, "private") + if err := os.Mkdir(private, 0o700); err != nil { + t.Fatal(err) + } + link := filepath.Join(dir, "link") + if err := os.Symlink(private, link); err != nil { + t.Fatal(err) + } + if _, err := Listen(config.ServeConfig{Socket: filepath.Join(link, "e.sock")}); err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("symlinked dir: %v", err) + } + if err := ownedPrivateDir(file); err == nil || !strings.Contains(err.Error(), "not a directory") { + t.Fatalf("regular file: %v", err) + } + if err := ownedPrivateDir(filepath.Join(dir, "missing")); err == nil { + t.Fatal("missing dir should fail") + } + // The dir Listen creates itself passes. + if err := ownedPrivateDir(private); err != nil { + t.Fatal(err) + } +} + +func TestAuthRefusalOnTCPAndWithToken(t *testing.T) { + s := startServer(t, "s3cret", true) + + // The socket requires the token because the server has one. + bare := unixClient(s.socket) + resp, err := bare.Get("http://crux" + api.PathHealth) + if err != nil { + t.Fatal(err) + } + var e api.Error + _ = json.NewDecoder(resp.Body).Decode(&e) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized || e.Code != api.CodeUnauthorized || resp.Header.Get("WWW-Authenticate") == "" { + t.Fatalf("no token over socket: %d %+v %q", resp.StatusCode, e, resp.Header.Get("WWW-Authenticate")) + } + + // A wrong token is refused over TCP; the right one is accepted. + for _, c := range []struct { + token string + status int + }{{"wrong", http.StatusUnauthorized}, {"", http.StatusUnauthorized}, {"s3cret", http.StatusOK}} { + req, _ := http.NewRequest(http.MethodGet, s.tcp+api.PathHealth, nil) + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != c.status { + t.Fatalf("token %q over TCP: %d, want %d", c.token, resp.StatusCode, c.status) + } + } + // The health body names the version. + var h api.Health + s.expect(http.MethodGet, api.PathHealth, nil, http.StatusOK, &h) + if h.Version != api.Version || h.StartedAt.IsZero() || h.UptimeSeconds < 0 || h.StartedAt.After(time.Now()) { + t.Fatalf("health = %+v", h) + } +} + +func TestHandlerRefusesTCPWithoutTokenEvenIfBound(t *testing.T) { + // The second layer of fail-closed: a handler tagged as serving TCP with + // no token refuses every request, independent of Listen's check. + srv := New(newFakeRuntime()) + h := srv.Handler() + req := httptest.NewRequest(http.MethodGet, api.PathHealth, nil) + req = req.WithContext(context.WithValue(req.Context(), networkKey, "tcp")) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("TCP without token: %d %s", rec.Code, rec.Body) + } + // An untagged request (Handler mounted on someone else's http.Server) + // is refused the same way: only Serve's unix tag opens the door. + rec = httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, api.PathHealth, nil)) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("untagged without token: %d %s", rec.Code, rec.Body) + } + foreign := httptest.NewServer(h) + defer foreign.Close() + resp, err := http.Get(foreign.URL + api.PathHealth) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("foreign TCP server without token: %d", resp.StatusCode) + } + // With a token the foreign mount works like any other listener. + srv.Token = "t" + freq, _ := http.NewRequest(http.MethodGet, foreign.URL+api.PathHealth, nil) + freq.Header.Set("Authorization", "Bearer t") + resp, err = http.DefaultClient.Do(freq) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("foreign TCP server with token: %d", resp.StatusCode) + } + srv.Token = "" + // The same request on the unix network is served. + req = httptest.NewRequest(http.MethodGet, api.PathHealth, nil) + req = req.WithContext(context.WithValue(req.Context(), networkKey, "unix")) + rec = httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("unix without token: %d %s", rec.Code, rec.Body) + } +} + +func TestRunListGetOverTheSocket(t *testing.T) { + s := startServer(t, "", false) + s.rt.add("scout", "s1", crux.StateRunning) + + // run with a definition registers and starts. + def, _ := json.Marshal(config.AgentConfig{ID: "f1", Backend: "copilot", Model: "gpt-5"}) + var ar api.AgentResponse + s.expect(http.MethodPost, api.PathAgents, api.RunRequest{Name: "fixer", Definition: def}, http.StatusCreated, &ar) + if ar.Agent != "fixer" || ar.Process == nil || ar.Process.State != string(crux.StateRunning) || ar.Process.Backend != "copilot" || ar.Process.Model != "gpt-5" || ar.Process.StartedAt == nil { + t.Fatalf("run = %+v / %+v", ar, ar.Process) + } + // The agent was launched under the server's context, not the request's. + s.rt.mu.Lock() + launchCtx := s.rt.startCtx + gotCfg := s.rt.procs["fixer"].Config + s.rt.mu.Unlock() + if launchCtx == nil || launchCtx.Err() != nil { + t.Fatalf("launch context should be the live server context, got %v", launchCtx) + } + // The defaults a start depends on were filled; the given ID was kept. + if gotCfg.ID != "f1" || gotCfg.Role != "fixer" || gotCfg.Replicas != 1 || !gotCfg.Enabled { + t.Fatalf("defaults not applied: %+v", gotCfg) + } + // A definition that sets enabled or role keeps them. + def2 := json.RawMessage(`{"backend":"copilot","role":"scout-role","enabled":false,"replicas":3}`) + s.expect(http.MethodPost, api.PathAgents, api.RunRequest{Name: "quiet", Definition: def2}, http.StatusCreated, &ar) + s.rt.mu.Lock() + gotCfg = s.rt.procs["quiet"].Config + s.rt.mu.Unlock() + if gotCfg.ID != "quiet" || gotCfg.Role != "scout-role" || gotCfg.Replicas != 3 || gotCfg.Enabled { + t.Fatalf("explicit fields overridden: %+v", gotCfg) + } + // Re-posting a definition for a known name is a conflict, not a silent + // start of the old definition. + var e api.Error + s.expect(http.MethodPost, api.PathAgents, api.RunRequest{Name: "fixer", Definition: def}, http.StatusConflict, &e) + if e.Code != api.CodeConflict || !strings.Contains(e.Error, "already defined") { + t.Fatalf("conflict: %+v", e) + } + // A bare name may be an agent ID, as for every other verb. + s.expect(http.MethodPost, api.PathAgents, api.RunRequest{Name: "s1"}, http.StatusCreated, &ar) + if ar.Agent != "scout" { + t.Fatalf("run by id = %+v", ar) + } + + // run without a definition starts an agent the server already knows. + s.rt.add("idle", "i1", crux.StateStopped) + s.expect(http.MethodPost, api.PathAgents, api.RunRequest{Name: "idle"}, http.StatusCreated, &ar) + if ar.Process.State != string(crux.StateRunning) { + t.Fatalf("run known = %+v", ar.Process) + } + + // list is sorted by name and skips nil entries. + var list api.ProcessList + s.expect(http.MethodGet, api.PathAgents, nil, http.StatusOK, &list) + var names []string + for _, p := range list.Processes { + names = append(names, p.Name) + } + if strings.Join(names, ",") != "fixer,idle,quiet,scout" { + t.Fatalf("list = %v", names) + } + + // get by ID resolves to the name and carries the full projection. + s.rt.mu.Lock() + p := s.rt.procs["scout"] + p.ModelOverride = "sonnet" + p.Paused, p.PausedReason, p.PausedBy = true, "quiesced", "andy" + p.KickRefused, p.KickRefusalReason = true, "busy" + p.StartFailureClass = string(crux.StartFailureNoOutput) + s.rt.mu.Unlock() + var got api.Process + s.expect(http.MethodGet, api.AgentPath("s1"), nil, http.StatusOK, &got) + if got.Name != "scout" || got.Model != "sonnet" || !got.Paused || got.PausedBy != "andy" || !got.KickRefused || got.StartFailureClass != string(crux.StartFailureNoOutput) { + t.Fatalf("get = %+v", got) + } + + // health counts the definitions. + var h api.Health + s.expect(http.MethodGet, api.PathHealth, nil, http.StatusOK, &h) + if h.Agents != 5 { // four agents plus the fake's nil entry + t.Fatalf("health agents = %d", h.Agents) + } +} + +func TestRunErrors(t *testing.T) { + s := startServer(t, "", false) + var e api.Error + s.expect(http.MethodPost, api.PathAgents, api.RunRequest{}, http.StatusBadRequest, &e) + if e.Code != api.CodeBadRequest || !strings.Contains(e.Error, "name") { + t.Fatalf("empty name: %+v", e) + } + s.expect(http.MethodPost, api.PathAgents, api.RunRequest{Name: "ghost"}, http.StatusNotFound, &e) + if e.Code != api.CodeNotFound { + t.Fatalf("unknown without definition: %+v", e) + } + s.expect(http.MethodPost, api.PathAgents, api.RunRequest{Name: "x", Definition: json.RawMessage(`"not an object"`)}, http.StatusBadRequest, &e) + if !strings.Contains(e.Error, "definition") { + t.Fatalf("bad definition: %+v", e) + } + s.expect(http.MethodPost, api.PathAgents, map[string]any{"name": "x", "bogus": 1}, http.StatusBadRequest, &e) + if !strings.Contains(e.Error, "request body") { + t.Fatalf("unknown field: %+v", e) + } + for _, bad := range []string{"Fixer", "a b", "../x", "-lead", strings.Repeat("a", 65)} { + s.expect(http.MethodPost, api.PathAgents, api.RunRequest{Name: bad, Definition: json.RawMessage(`{}`)}, http.StatusBadRequest, &e) + if e.Code != api.CodeBadRequest || !strings.Contains(e.Error, "name must match") { + t.Fatalf("name %q: %+v", bad, e) + } + } + // The runtime declining the definition (the fake's stand-in for the + // ACMM gate) is reported, not followed by a start that cannot work. + s.expect(http.MethodPost, api.PathAgents, api.RunRequest{Name: "refused", Definition: json.RawMessage(`{}`)}, http.StatusUnprocessableEntity, &e) + if e.Code != api.CodeBadRequest || !strings.Contains(e.Error, "not accepted") { + t.Fatalf("refused: %+v", e) + } + s.rt.add("broken", "b1", crux.StateStopped) + s.rt.startErr = errors.New("tmux: no server") + s.expect(http.MethodPost, api.PathAgents, api.RunRequest{Name: "broken"}, http.StatusInternalServerError, &e) + if e.Code != api.CodeRuntime || !strings.Contains(e.Error, "no server") { + t.Fatalf("start error: %+v", e) + } + s.expect(http.MethodGet, api.AgentPath("ghost"), nil, http.StatusNotFound, &e) + if e.Code != api.CodeNotFound { + t.Fatalf("get ghost: %+v", e) + } + // Unknown routes and wrong methods are the mux's 404/405. + if resp, _ := s.do(http.MethodDelete, api.PathAgents, nil); resp.StatusCode != http.StatusMethodNotAllowed { + t.Fatalf("DELETE agents: %d", resp.StatusCode) + } + if resp, _ := s.do(http.MethodGet, "/v1/nope", nil); resp.StatusCode != http.StatusNotFound { + t.Fatalf("unknown route: %d", resp.StatusCode) + } +} + +func TestLogsSources(t *testing.T) { + s := startServer(t, "", false) + s.rt.add("fixer", "f1", crux.StateRunning) + s.rt.pane["fixer"] = []string{"p1", "p2", "p3"} + s.rt.buffer["fixer"] = [][]string{{"b1", "b2"}} + s.rt.fullLog["fixer"] = "everything" + s.rt.header["fixer"] = "# header" + + var logs api.Logs + s.expect(http.MethodGet, api.LogsPath("fixer"), nil, http.StatusOK, &logs) + if logs.Source != api.LogSourcePane || strings.Join(logs.Lines, ",") != "p1,p2,p3" { + t.Fatalf("pane default = %+v", logs) + } + s.expect(http.MethodGet, api.LogsPath("f1")+"?n=2", nil, http.StatusOK, &logs) + if logs.Agent != "fixer" || strings.Join(logs.Lines, ",") != "p2,p3" { + t.Fatalf("pane n=2 by id = %+v", logs) + } + s.expect(http.MethodGet, api.LogsPath("fixer")+"?source=buffer", nil, http.StatusOK, &logs) + if logs.Source != api.LogSourceBuffer || strings.Join(logs.Lines, ",") != "b1,b2" { + t.Fatalf("buffer = %+v", logs) + } + s.expect(http.MethodGet, api.LogsPath("fixer")+"?source=full", nil, http.StatusOK, &logs) + if logs.Source != api.LogSourceFull || logs.Header != "# header" || logs.Content != "everything" { + t.Fatalf("full = %+v", logs) + } + + var e api.Error + s.expect(http.MethodGet, api.LogsPath("fixer")+"?source=tape", nil, http.StatusBadRequest, &e) + s.expect(http.MethodGet, api.LogsPath("fixer")+"?n=-1", nil, http.StatusBadRequest, &e) + s.expect(http.MethodGet, api.LogsPath("fixer")+"?n=abc", nil, http.StatusBadRequest, &e) + s.rt.outputErr = errors.New("pane gone") + s.expect(http.MethodGet, api.LogsPath("fixer"), nil, http.StatusInternalServerError, &e) + if e.Code != api.CodeRuntime || !strings.Contains(e.Error, "pane gone") { + t.Fatalf("output error: %+v", e) + } + s.expect(http.MethodGet, api.LogsPath("fixer")+"?follow=1", nil, http.StatusInternalServerError, &e) + if !strings.Contains(e.Error, "pane gone") { + t.Fatalf("follow with output error: %+v", e) + } +} + +// readStream reads SSE records from resp until want have arrived or the +// deadline passes, then closes the body. +func readStream(t *testing.T, resp *http.Response, want int) []string { + t.Helper() + var mu sync.Mutex + var got []string + done := make(chan struct{}) + go func() { + defer close(done) + _ = api.ReadEvents(resp.Body, func(typ string, data []byte) error { + mu.Lock() + defer mu.Unlock() + got = append(got, typ+"|"+string(data)) + if len(got) >= want { + return io.EOF + } + return nil + }) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + _ = resp.Body.Close() + <-done + mu.Lock() + defer mu.Unlock() + t.Fatalf("stream delivered %d of %d records: %v", len(got), want, got) + } + _ = resp.Body.Close() + mu.Lock() + defer mu.Unlock() + return got +} + +func TestFollowLogsStreamsNewLines(t *testing.T) { + s := startServer(t, "", false) + s.rt.add("fixer", "f1", crux.StateRunning) + // Successive buffer windows: the initial tail, then overlap, then an + // unchanged window (no record), then a reset. + s.rt.buffer["fixer"] = [][]string{{"a", "b"}, {"b", "c"}, {"b", "c"}, {"z"}} + + req, _ := http.NewRequest(http.MethodGet, "http://crux"+api.LogsPath("fixer")+"?follow=true", nil) + resp, err := s.client.Do(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != api.ContentTypeEvent { + t.Fatalf("follow: %d %s", resp.StatusCode, resp.Header.Get("Content-Type")) + } + got := readStream(t, resp, 3) + want := []string{ + api.EventLog + `|{"agent":"fixer","lines":["a","b"]}`, + api.EventLog + `|{"agent":"fixer","lines":["c"]}`, + api.EventLog + `|{"agent":"fixer","lines":["z"]}`, + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("follow records:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +func TestFollowEndsWithAStateRecordWhenOutputFails(t *testing.T) { + s := startServer(t, "", false) + s.rt.add("fixer", "f1", crux.StateRunning) + req, _ := http.NewRequest(http.MethodGet, "http://crux"+api.LogsPath("fixer")+"?follow=1", nil) + resp, err := s.client.Do(req) + if err != nil { + t.Fatal(err) + } + s.rt.mu.Lock() + s.rt.outputErr = errors.New("session ended") + s.rt.mu.Unlock() + got := readStream(t, resp, 1) + if len(got) != 1 || !strings.HasPrefix(got[0], api.EventState+"|") || !strings.Contains(got[0], "session ended") { + t.Fatalf("records = %v", got) + } +} + +func TestKickAndStop(t *testing.T) { + s := startServer(t, "", false) + s.rt.add("fixer", "f1", crux.StateRunning) + s.rt.add("idle", "i1", crux.StateStopped) + s.rt.add("wedged", "w1", crux.StateFailed) + + var ar api.AgentResponse + s.expect(http.MethodPost, api.KickPath("f1"), api.KickRequest{Message: "look at main"}, http.StatusOK, &ar) + if ar.Agent != "fixer" || ar.Process == nil || strings.Join(s.rt.Kicks(), ",") != "look at main" { + t.Fatalf("kick = %+v kicks %v", ar, s.rt.Kicks()) + } + var e api.Error + s.expect(http.MethodPost, api.KickPath("fixer"), api.KickRequest{Message: " "}, http.StatusBadRequest, &e) + s.expect(http.MethodPost, api.KickPath("fixer"), map[string]int{"nope": 1}, http.StatusBadRequest, &e) + s.expect(http.MethodPost, api.KickPath("ghost"), api.KickRequest{Message: "x"}, http.StatusNotFound, &e) + if e.Code != api.CodeNotFound { + t.Fatalf("kick ghost: %+v", e) + } + s.expect(http.MethodPost, api.KickPath("idle"), api.KickRequest{Message: "x"}, http.StatusInternalServerError, &e) + if e.Code != api.CodeRuntime || !strings.Contains(e.Error, "not running") { + t.Fatalf("kick idle: %+v", e) + } + + s.expect(http.MethodPost, api.StopPath("fixer"), nil, http.StatusOK, &ar) + if ar.Process == nil || ar.Process.State != string(crux.StateStopped) { + t.Fatalf("stop = %+v", ar.Process) + } + s.expect(http.MethodPost, api.StopPath("ghost"), nil, http.StatusNotFound, &e) + s.expect(http.MethodPost, api.StopPath("wedged"), nil, http.StatusInternalServerError, &e) + if !strings.Contains(e.Error, "kill-session") { + t.Fatalf("stop wedged: %+v", e) + } +} + +// watched is the state the server's watcher last recorded for name. +func (s *served) watched(name string) string { + s.srv.mu.Lock() + defer s.srv.mu.Unlock() + return s.srv.lastStates[name] +} + +// waitUntil polls cond on a short ticker until it holds or the deadline +// passes; it is the package's stand-in for testutil.Eventually, which the +// nested module cannot import. +func waitUntil(t *testing.T, what string, cond func() bool) { + t.Helper() + tick := time.NewTicker(2 * time.Millisecond) + defer tick.Stop() + deadline := time.After(5 * time.Second) + for { + if cond() { + return + } + select { + case <-tick.C: + case <-deadline: + t.Fatalf("timed out waiting for %s", what) + } + } +} + +func TestEventStream(t *testing.T) { + s := startServer(t, "", false) + s.rt.add("fixer", "f1", crux.StateStopped) + // Let the watcher seed its table before subscribing so the stream + // carries transitions only. + waitUntil(t, "watcher seeded", func() bool { return s.watched("fixer") == string(crux.StateStopped) }) + + req, _ := http.NewRequest(http.MethodGet, "http://crux"+api.PathEvents, nil) + resp, err := s.client.Do(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != api.ContentTypeEvent { + t.Fatalf("events: %d %s", resp.StatusCode, resp.Header.Get("Content-Type")) + } + // Make sure the subscription is registered before acting. + waitUntil(t, "subscriber registered", func() bool { + s.srv.mu.Lock() + defer s.srv.mu.Unlock() + return len(s.srv.subscribers) == 1 + }) + + // An API-driven start, a kick, a stop; then a relayed kick-observer + // event; then a state change the API did not cause. After each state + // change wait for the watcher to record it so it sees every transition. + settle := func(state crux.ProcessState) { + waitUntil(t, "watcher saw "+string(state), func() bool { return s.watched("fixer") == string(state) }) + } + s.expect(http.MethodPost, api.PathAgents, api.RunRequest{Name: "fixer"}, http.StatusCreated, nil) + settle(crux.StateRunning) + s.expect(http.MethodPost, api.KickPath("fixer"), api.KickRequest{Message: "go"}, http.StatusOK, nil) + s.rt.mu.Lock() + observer := s.rt.observer + s.rt.mu.Unlock() + if observer == nil { + t.Fatal("Serve should install a kick observer") + } + observer("fixer", "delivered", "typed 2 lines") + s.expect(http.MethodPost, api.StopPath("fixer"), nil, http.StatusOK, nil) + settle(crux.StateStopped) + s.rt.mu.Lock() + s.rt.procs["fixer"].State = crux.StateFailed + s.rt.procs["fixer"].LastRestartReason = "crash loop" + s.rt.mu.Unlock() + + got := readStream(t, resp, 7) + var types []string + var events []api.Event + for _, rec := range got { + typ, data, _ := strings.Cut(rec, "|") + types = append(types, typ) + var ev api.Event + if err := json.Unmarshal([]byte(data), &ev); err != nil { + t.Fatalf("%s: %v", rec, err) + } + if ev.Time.IsZero() || ev.Type != typ { + t.Fatalf("event %+v is not stamped or typed", ev) + } + events = append(events, ev) + } + // started, state(running), kicked, kick relay, stopped, state(stopped), + // state(failed). The watcher's state records interleave with the + // lifecycle ones depending on timing, so check the multiset and the + // relative order of the lifecycle events. + count := map[string]int{} + for _, ty := range types { + count[ty]++ + } + if count[api.EventStarted] != 1 || count[api.EventKicked] != 1 || count[api.EventKick] != 1 || count[api.EventStopped] != 1 || count[api.EventState] != 3 { + t.Fatalf("event types = %v", types) + } + if idx(types, api.EventStarted) > idx(types, api.EventKicked) || idx(types, api.EventKicked) > idx(types, api.EventKick) || idx(types, api.EventKick) > idx(types, api.EventStopped) { + t.Fatalf("lifecycle order = %v", types) + } + var states []string + for _, ev := range events { + if ev.Type == api.EventState { + states = append(states, ev.State) + if ev.State == string(crux.StateFailed) && ev.Detail != "crash loop" { + t.Fatalf("failed state should carry the restart reason: %+v", ev) + } + } + if ev.Type == api.EventKick && (ev.State != "delivered" || ev.Detail != "typed 2 lines") { + t.Fatalf("kick relay = %+v", ev) + } + } + if strings.Join(states, ",") != "running,stopped,failed" { + t.Fatalf("state transitions = %v", states) + } +} + +func idx(list []string, v string) int { + for i, x := range list { + if x == v { + return i + } + } + return -1 +} + +func TestStateWatcherReportsPausedAndForgetsRemovedAgents(t *testing.T) { + rt := newFakeRuntime() + srv := New(rt) + rt.add("a", "a1", crux.StateRunning) + srv.diffStates(false) // seed + ch := srv.subscribe() + rt.mu.Lock() + rt.procs["a"].Paused = true + rt.mu.Unlock() + srv.diffStates(true) + select { + case ev := <-ch: + if ev.Type != api.EventState || ev.State != string(crux.StatePaused) { + t.Fatalf("paused event = %+v", ev) + } + default: + t.Fatal("no event for the pause") + } + rt.mu.Lock() + delete(rt.procs, "a") + rt.mu.Unlock() + srv.diffStates(true) + if _, ok := srv.lastStates["a"]; ok { + t.Fatal("removed agent should leave the state table") + } + select { + case ev := <-ch: + t.Fatalf("removal should publish nothing, got %+v", ev) + default: + } + srv.unsubscribe(ch) + srv.unsubscribe(ch) // idempotent + if _, ok := <-ch; ok { + t.Fatal("unsubscribe should close the channel") + } +} + +func TestSlowSubscriberIsDropped(t *testing.T) { + srv := New(newFakeRuntime()) + ch := srv.subscribe() + for i := 0; i < subscriberBuffer+1; i++ { + srv.publish(api.Event{Type: api.EventKicked, Agent: "a"}) + } + n := 0 + for range ch { + n++ + } + if n != subscriberBuffer { + t.Fatalf("drained %d events, want %d then a closed channel", n, subscriberBuffer) + } + srv.mu.Lock() + defer srv.mu.Unlock() + if len(srv.subscribers) != 0 { + t.Fatal("dropped subscriber should be removed") + } +} + +// noFlush is a ResponseWriter that cannot stream. +type noFlush struct{ http.ResponseWriter } + +func TestStreamsNeedAFlusher(t *testing.T) { + srv := New(newFakeRuntime()) + srv.Runtime.(*fakeRuntime).add("a", "a1", crux.StateRunning) + for _, path := range []string{api.PathEvents, api.LogsPath("a") + "?follow=1"} { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, path, nil) + req = req.WithContext(context.WithValue(req.Context(), networkKey, networkUnix)) + srv.Handler().ServeHTTP(noFlush{rec}, req) + if rec.Code != http.StatusInternalServerError || !strings.Contains(rec.Body.String(), "streaming") { + t.Fatalf("%s without flusher: %d %s", path, rec.Code, rec.Body) + } + } +} + +func TestEventStreamKeepAlive(t *testing.T) { + s := startServerPolling(t, "", false, time.Millisecond) // keep-alive every 15ms + req, _ := http.NewRequest(http.MethodGet, "http://crux"+api.PathEvents, nil) + resp, err := s.client.Do(req) + if err != nil { + t.Fatal(err) + } + buf := make([]byte, 64) + n, err := io.ReadAtLeast(resp.Body, buf, len(": keep-alive\n\n")) + _ = resp.Body.Close() + if err != nil || !strings.Contains(string(buf[:n]), "keep-alive") { + t.Fatalf("keep-alive: %q %v", buf[:n], err) + } +} + +func TestServeReturnsListenerErrors(t *testing.T) { + // A listener that fails on Accept makes Serve return its error. + srv := New(newFakeRuntime()) + ls := &Listeners{Unix: failingListener{}} + err := srv.Serve(context.Background(), ls) + if err == nil || !strings.Contains(err.Error(), "accept boom") { + t.Fatalf("Serve = %v", err) + } +} + +type failingListener struct{} + +func (failingListener) Accept() (net.Conn, error) { return nil, errors.New("accept boom") } +func (failingListener) Close() error { return nil } +func (failingListener) Addr() net.Addr { return &net.UnixAddr{Name: "fail", Net: "unix"} } + +func TestWriteJSONLogsEncodeFailures(t *testing.T) { + var logged bytes.Buffer + srv := New(newFakeRuntime()) + srv.Logger = slogTo(&logged) + rec := httptest.NewRecorder() + srv.writeJSON(rec, http.StatusOK, make(chan int)) + if !strings.Contains(logged.String(), "writing response") { + t.Fatalf("encode failure should be logged, got %q", logged.String()) + } +} + +func slogTo(w io.Writer) *slog.Logger { return slog.New(slog.NewTextHandler(w, nil)) } + +func TestIsTrueAndNewTail(t *testing.T) { + for _, v := range []string{"1", "true", "YES", "on"} { + if !isTrue(v) { + t.Errorf("%q should be true", v) + } + } + for _, v := range []string{"", "0", "false", "maybe"} { + if isTrue(v) { + t.Errorf("%q should be false", v) + } + } + cases := []struct{ prev, cur, want []string }{ + {nil, []string{"a"}, []string{"a"}}, + {[]string{"a", "b"}, []string{"a", "b"}, nil}, + {[]string{"a", "b"}, []string{"b", "c"}, []string{"c"}}, + {[]string{"a", "b"}, []string{"x", "y"}, []string{"x", "y"}}, + {[]string{"a", "b", "c"}, []string{"c"}, nil}, + } + for _, c := range cases { + if got := newTail(c.prev, c.cur); strings.Join(got, ",") != strings.Join(c.want, ",") { + t.Errorf("newTail(%v,%v) = %v, want %v", c.prev, c.cur, got, c.want) + } + } +} diff --git a/src/pkg/skillreg/agentrepos.go b/src/crux/skillreg/agentrepos.go similarity index 100% rename from src/pkg/skillreg/agentrepos.go rename to src/crux/skillreg/agentrepos.go diff --git a/src/pkg/skillreg/agentrepos_test.go b/src/crux/skillreg/agentrepos_test.go similarity index 100% rename from src/pkg/skillreg/agentrepos_test.go rename to src/crux/skillreg/agentrepos_test.go diff --git a/src/pkg/skillreg/agentspec.go b/src/crux/skillreg/agentspec.go similarity index 100% rename from src/pkg/skillreg/agentspec.go rename to src/crux/skillreg/agentspec.go diff --git a/src/pkg/skillreg/agentspec_test.go b/src/crux/skillreg/agentspec_test.go similarity index 100% rename from src/pkg/skillreg/agentspec_test.go rename to src/crux/skillreg/agentspec_test.go diff --git a/src/pkg/skillreg/skillreg.go b/src/crux/skillreg/skillreg.go similarity index 99% rename from src/pkg/skillreg/skillreg.go rename to src/crux/skillreg/skillreg.go index b5c42b18e5..af3e0a8e64 100644 --- a/src/pkg/skillreg/skillreg.go +++ b/src/crux/skillreg/skillreg.go @@ -65,7 +65,7 @@ import ( "gopkg.in/yaml.v3" - "github.com/hivecommons/hive/pkg/agentsmd" + "github.com/hivecommons/hive/crux/agentsmd" ) const ( diff --git a/src/pkg/skillreg/skillreg_test.go b/src/crux/skillreg/skillreg_test.go similarity index 99% rename from src/pkg/skillreg/skillreg_test.go rename to src/crux/skillreg/skillreg_test.go index f368411c0e..d138e66d9e 100644 --- a/src/pkg/skillreg/skillreg_test.go +++ b/src/crux/skillreg/skillreg_test.go @@ -6,7 +6,7 @@ import ( "sync" "testing" - "github.com/hivecommons/hive/pkg/agentsmd" + "github.com/hivecommons/hive/crux/agentsmd" ) // writeSkill writes content to / creating the dir as needed. diff --git a/src/pkg/taskmcp/ci.go b/src/crux/taskmcp/ci.go similarity index 100% rename from src/pkg/taskmcp/ci.go rename to src/crux/taskmcp/ci.go diff --git a/src/pkg/taskmcp/ci_test.go b/src/crux/taskmcp/ci_test.go similarity index 100% rename from src/pkg/taskmcp/ci_test.go rename to src/crux/taskmcp/ci_test.go diff --git a/src/pkg/taskmcp/lease_token.go b/src/crux/taskmcp/lease_token.go similarity index 100% rename from src/pkg/taskmcp/lease_token.go rename to src/crux/taskmcp/lease_token.go diff --git a/src/pkg/taskmcp/lease_token_test.go b/src/crux/taskmcp/lease_token_test.go similarity index 100% rename from src/pkg/taskmcp/lease_token_test.go rename to src/crux/taskmcp/lease_token_test.go diff --git a/src/pkg/taskmcp/related.go b/src/crux/taskmcp/related.go similarity index 100% rename from src/pkg/taskmcp/related.go rename to src/crux/taskmcp/related.go diff --git a/src/pkg/taskmcp/related_test.go b/src/crux/taskmcp/related_test.go similarity index 100% rename from src/pkg/taskmcp/related_test.go rename to src/crux/taskmcp/related_test.go diff --git a/src/pkg/taskmcp/taskmcp.go b/src/crux/taskmcp/taskmcp.go similarity index 100% rename from src/pkg/taskmcp/taskmcp.go rename to src/crux/taskmcp/taskmcp.go diff --git a/src/pkg/taskmcp/taskmcp_test.go b/src/crux/taskmcp/taskmcp_test.go similarity index 100% rename from src/pkg/taskmcp/taskmcp_test.go rename to src/crux/taskmcp/taskmcp_test.go diff --git a/src/pkg/toolapprove/bulk_parity_test.go b/src/crux/toolapprove/bulk_parity_test.go similarity index 100% rename from src/pkg/toolapprove/bulk_parity_test.go rename to src/crux/toolapprove/bulk_parity_test.go diff --git a/src/pkg/toolapprove/classify.go b/src/crux/toolapprove/classify.go similarity index 100% rename from src/pkg/toolapprove/classify.go rename to src/crux/toolapprove/classify.go diff --git a/src/pkg/toolapprove/coverage_test.go b/src/crux/toolapprove/coverage_test.go similarity index 100% rename from src/pkg/toolapprove/coverage_test.go rename to src/crux/toolapprove/coverage_test.go diff --git a/src/pkg/toolapprove/decide.go b/src/crux/toolapprove/decide.go similarity index 93% rename from src/pkg/toolapprove/decide.go rename to src/crux/toolapprove/decide.go index b4d8219927..0b8b35c3c6 100644 --- a/src/pkg/toolapprove/decide.go +++ b/src/crux/toolapprove/decide.go @@ -5,8 +5,8 @@ import ( "fmt" "strings" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/crux/types" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" ) // Decide evaluates a tool request against ACMM gates, agent identity, and tool policy, @@ -165,11 +165,11 @@ func Decide(ctx context.Context, req ToolRequest, acmmLevel int, agentId AgentId Agent: agentName, } - default: // L6 Full Autonomy - // L6 Full: side-effectful tools route through security scan, auto-approve on green + default: // L6 Autonomous + // L6 Autonomous: side-effectful tools route through security scan, auto-approve on green return Verdict{ Decision: DecisionSecurityScan, - Rationale: "ACMM L6 (full autonomy): side-effectful tool routes through pre-execution security scan; auto-approves on green", + Rationale: "ACMM L6 (Autonomous): side-effectful tool routes through pre-execution security scan; auto-approves on green", Tool: toolName, ACMMLevel: acmmLevel, Agent: agentName, @@ -213,7 +213,7 @@ func Resolve(ctx context.Context, req ToolRequest, acmmLevel int, agentId AgentI switch { case acmmLevel >= 6: v.Decision = DecisionAutoApprove - v.Rationale = "ACMM L6 full autonomy: auto-approved on green security scan" + v.Rationale = "ACMM L6 (Autonomous): auto-approved on green security scan" case acmmLevel == 5: v.Decision = DecisionAutoApprove v.Rationale = "ACMM L5 hold-gated: auto-approved on green security scan" @@ -232,13 +232,13 @@ func Resolve(ctx context.Context, req ToolRequest, acmmLevel int, agentId AgentI } // EvaluateAndAudit runs Resolve and emits the verdict to the provided AuditSink if configured. -func EvaluateAndAudit(ctx context.Context, req ToolRequest, acmmLevel int, agentId AgentIdentity, scanner SecurityScanner, sink types.AuditSink, actor string) Verdict { +func EvaluateAndAudit(ctx context.Context, req ToolRequest, acmmLevel int, agentId AgentIdentity, scanner SecurityScanner, sink crux.AuditSink, actor string) Verdict { v := Resolve(ctx, req, acmmLevel, agentId, scanner) if sink != nil { if actor == "" { actor = "system" } - sink.Record(actor, types.AuditToolApproval, agentId.Name, v.AuditFields()) + sink.Record(actor, crux.AuditToolApproval, agentId.Name, v.AuditFields()) } return v } diff --git a/src/pkg/toolapprove/desk.go b/src/crux/toolapprove/desk.go similarity index 99% rename from src/pkg/toolapprove/desk.go rename to src/crux/toolapprove/desk.go index af6105bdb1..7e7edde542 100644 --- a/src/pkg/toolapprove/desk.go +++ b/src/crux/toolapprove/desk.go @@ -115,7 +115,7 @@ type RuleMatch struct { // // L0-L2 operator-approve — nothing side-effectful resolves without a human // L3-L5 security-scan — may reach auto-approve only via a green scan -// L6+ auto-approve — full autonomy; the desk is an audit record, not a gate +// L6+ auto-approve — autonomous; the desk is an audit record, not a gate // // An unknown/negative level clamps to the most restrictive lane, so a // mis-parsed or absent level can never widen authority. diff --git a/src/pkg/toolapprove/desk_ceiling_test.go b/src/crux/toolapprove/desk_ceiling_test.go similarity index 100% rename from src/pkg/toolapprove/desk_ceiling_test.go rename to src/crux/toolapprove/desk_ceiling_test.go diff --git a/src/pkg/toolapprove/inbox.go b/src/crux/toolapprove/inbox.go similarity index 98% rename from src/pkg/toolapprove/inbox.go rename to src/crux/toolapprove/inbox.go index 2012a11e9e..e533f9fa04 100644 --- a/src/pkg/toolapprove/inbox.go +++ b/src/crux/toolapprove/inbox.go @@ -452,12 +452,12 @@ func (i *Inbox) ResolveMany(ids []string, approved bool, operator, rationale str return out } -// PendingAtFullAutonomy returns the pending items queued at an ACMM level that -// claims full autonomy. Per requirement 4 of the throughput contract, these are +// PendingAtAutonomousLevel returns the pending items queued at an ACMM level that +// is Autonomous (L6). Per requirement 4 of the throughput contract, these are // surfaced as PROBABLE POLICY BUGS rather than silently waiting: an inbox that -// accumulates on a hive whose level says "full autonomy" is a misconfiguration +// accumulates on a hive whose level says "Autonomous" is a misconfiguration // signal. The API marks them and the dashboard renders them distinctly. -func (i *Inbox) PendingAtFullAutonomy() []PendingItem { +func (i *Inbox) PendingAtAutonomousLevel() []PendingItem { var out []PendingItem for _, p := range i.List() { if p.ACMMLevel >= 6 { diff --git a/src/pkg/toolapprove/inbox_l6_test.go b/src/crux/toolapprove/inbox_l6_test.go similarity index 94% rename from src/pkg/toolapprove/inbox_l6_test.go rename to src/crux/toolapprove/inbox_l6_test.go index 6c8fb9c510..13544cd585 100644 --- a/src/pkg/toolapprove/inbox_l6_test.go +++ b/src/crux/toolapprove/inbox_l6_test.go @@ -146,9 +146,9 @@ func TestL6RuleAutoApproveNeverQueues(t *testing.T) { } } -// TestPendingAtFullAutonomyIsFlagged pins requirement 4: an item queued on a +// TestPendingAtAutonomousLevelIsFlagged pins requirement 4: an item queued on a // hive at L6 is surfaced as a probable policy bug rather than silently waiting. -func TestPendingAtFullAutonomyIsFlagged(t *testing.T) { +func TestPendingAtAutonomousLevelIsFlagged(t *testing.T) { inbox := NewInbox(filepath.Join(t.TempDir(), "inbox.json")) if _, err := inbox.Enqueue(sideEffectfulRequest(), Verdict{ @@ -156,9 +156,9 @@ func TestPendingAtFullAutonomyIsFlagged(t *testing.T) { }); err != nil { t.Fatalf("Enqueue: %v", err) } - bugs := inbox.PendingAtFullAutonomy() + bugs := inbox.PendingAtAutonomousLevel() if len(bugs) != 1 { - t.Fatalf("PendingAtFullAutonomy returned %d items, want 1 — an inbox accumulating at L6 "+ + t.Fatalf("PendingAtAutonomousLevel returned %d items, want 1 — an inbox accumulating at L6 "+ "is a misconfiguration signal that must be surfaced, not queued politely", len(bugs)) } @@ -172,8 +172,8 @@ func TestPendingAtFullAutonomyIsFlagged(t *testing.T) { if _, err := inbox.Enqueue(other, Verdict{Decision: DecisionOperatorApprove, ACMMLevel: 3}); err != nil { t.Fatalf("Enqueue: %v", err) } - if got := len(inbox.PendingAtFullAutonomy()); got != 1 { - t.Errorf("an L3 pending item was flagged as a policy bug: PendingAtFullAutonomy=%d, want 1", got) + if got := len(inbox.PendingAtAutonomousLevel()); got != 1 { + t.Errorf("an L3 pending item was flagged as a policy bug: PendingAtAutonomousLevel=%d, want 1", got) } } diff --git a/src/pkg/toolapprove/inbox_test.go b/src/crux/toolapprove/inbox_test.go similarity index 100% rename from src/pkg/toolapprove/inbox_test.go rename to src/crux/toolapprove/inbox_test.go diff --git a/src/pkg/toolapprove/parity.go b/src/crux/toolapprove/parity.go similarity index 99% rename from src/pkg/toolapprove/parity.go rename to src/crux/toolapprove/parity.go index 02c5747915..0fe1e7c421 100644 --- a/src/pkg/toolapprove/parity.go +++ b/src/crux/toolapprove/parity.go @@ -26,7 +26,7 @@ package toolapprove import ( "fmt" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // LegacyGateQueuedMerge reports whether a queued merge may proceed. Unlike the diff --git a/src/pkg/toolapprove/parity_test.go b/src/crux/toolapprove/parity_test.go similarity index 99% rename from src/pkg/toolapprove/parity_test.go rename to src/crux/toolapprove/parity_test.go index 7fbd8ee064..5ff38a56f9 100644 --- a/src/pkg/toolapprove/parity_test.go +++ b/src/crux/toolapprove/parity_test.go @@ -23,7 +23,7 @@ import ( "context" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestSelfMergeParityAcrossLevels pins the desk against the legacy diff --git a/src/pkg/toolapprove/rules.go b/src/crux/toolapprove/rules.go similarity index 100% rename from src/pkg/toolapprove/rules.go rename to src/crux/toolapprove/rules.go diff --git a/src/pkg/toolapprove/rules_test.go b/src/crux/toolapprove/rules_test.go similarity index 100% rename from src/pkg/toolapprove/rules_test.go rename to src/crux/toolapprove/rules_test.go diff --git a/src/pkg/toolapprove/security.go b/src/crux/toolapprove/security.go similarity index 59% rename from src/pkg/toolapprove/security.go rename to src/crux/toolapprove/security.go index 7bcb8f61b8..6cb3e5b78c 100644 --- a/src/pkg/toolapprove/security.go +++ b/src/crux/toolapprove/security.go @@ -2,14 +2,74 @@ package toolapprove import ( "context" + "errors" "fmt" "path/filepath" "regexp" "strings" - - "github.com/hivecommons/hive/pkg/ioscan" + "sync/atomic" ) +// TextScanner is the payload scanner DefaultSecurityScanner runs over tool +// content, commands and string arguments: prompt-injection, secret and +// dangerous-directive rules. Hive's implementation is pkg/ioscan's +// ToolApproveScanner, installed by the host through SetDefaultTextScanner +// (cmd/hive and pkg/turn do so at init); this package does not import +// pkg/ioscan so it stays inside the Crux ring. +type TextScanner interface { + ScanInput(text string) TextVerdict + ScanOutput(text string) TextVerdict +} + +// TextVerdict is a TextScanner's answer for one payload. +type TextVerdict struct { + // Blocked is the scanner's policy decision for the text. + Blocked bool + // CriticalInjection reports a critical-severity injection finding, which + // blocks input even when the scanner's own policy would not. + CriticalInjection bool + // Findings lists every rule hit; only High ones become violations. + Findings []TextFinding +} + +// TextFinding is one rule hit, with the masked snippet and rule id the +// violation text names. +type TextFinding struct { + Kind string + Snippet string + Rule string + // High is true at high severity or above. + High bool +} + +// defaultTextScanner is the TextScanner NewDefaultSecurityScanner installs. +var defaultTextScanner atomic.Pointer[TextScanner] + +// SetDefaultTextScanner installs the TextScanner that NewDefaultSecurityScanner +// (and every nil-scanner fallback in Resolve and the Desk) uses. Without one +// a scan fails closed: DefaultSecurityScanner.Scan returns ErrNoTextScanner +// and the scan lane denies, so a host that forgets the wiring sees denials, +// not silently unscanned payloads. +func SetDefaultTextScanner(s TextScanner) { + if s == nil { + defaultTextScanner.Store(nil) + return + } + defaultTextScanner.Store(&s) +} + +// DefaultTextScanner returns the installed TextScanner, or nil. +func DefaultTextScanner() TextScanner { + if p := defaultTextScanner.Load(); p != nil { + return *p + } + return nil +} + +// ErrNoTextScanner is returned by DefaultSecurityScanner.Scan when no +// TextScanner is installed; the scan lane treats it as a failed scan. +var ErrNoTextScanner = errors.New("toolapprove: no text scanner installed (SetDefaultTextScanner)") + // SecurityScanner provides pre-execution security analysis of tool requests. type SecurityScanner interface { Scan(ctx context.Context, req ToolRequest) (ScanResult, error) @@ -18,10 +78,14 @@ type SecurityScanner interface { // DefaultSecurityScanner evaluates tool requests against prompt injections, // destructive directives, secret leaks, and guardrail path modifications. type DefaultSecurityScanner struct { + // Text scans payload text; nil falls back to DefaultTextScanner at scan + // time, and a scan with neither fails closed. + Text TextScanner guardrailPathPatterns []string } -// NewDefaultSecurityScanner returns a configured DefaultSecurityScanner. +// NewDefaultSecurityScanner returns a configured DefaultSecurityScanner that +// scans payload text through the installed DefaultTextScanner. func NewDefaultSecurityScanner() *DefaultSecurityScanner { return &DefaultSecurityScanner{ guardrailPathPatterns: []string{ @@ -59,22 +123,29 @@ func (s *DefaultSecurityScanner) Scan(ctx context.Context, req ToolRequest) (Sca } } - for _, text := range textsToScan { - if text == "" { + text := s.Text + if text == nil { + text = DefaultTextScanner() + } + if text == nil { + return ScanResult{}, ErrNoTextScanner + } + for _, t := range textsToScan { + if t == "" { continue } - inVerdict := ioscan.ScanInput(text) - if inVerdict.Blocked || inVerdict.HasCriticalInjection() { + inVerdict := text.ScanInput(t) + if inVerdict.Blocked || inVerdict.CriticalInjection { for _, f := range inVerdict.Findings { - if f.Severity >= ioscan.SeverityHigh { + if f.High { violations = append(violations, fmt.Sprintf("input scan [%s]: %s (rule: %s)", f.Kind, f.Snippet, f.Rule)) } } } - outVerdict := ioscan.ScanOutput(text) + outVerdict := text.ScanOutput(t) if outVerdict.Blocked { for _, f := range outVerdict.Findings { - if f.Severity >= ioscan.SeverityHigh { + if f.High { violations = append(violations, fmt.Sprintf("output scan [%s]: %s (rule: %s)", f.Kind, f.Snippet, f.Rule)) } } diff --git a/src/crux/toolapprove/text_scanner_test.go b/src/crux/toolapprove/text_scanner_test.go new file mode 100644 index 0000000000..2837d52650 --- /dev/null +++ b/src/crux/toolapprove/text_scanner_test.go @@ -0,0 +1,71 @@ +package toolapprove + +import ( + "context" + "errors" + "strings" + "testing" +) + +// fakeTextScanner stands in for pkg/ioscan in this package's tests (the +// desk does not import ioscan). It flags the two shapes the scanner tests +// rely on: an "ignore all previous instructions" injection, a github_token +// secret and an "rm -rf" dangerous directive. +type fakeTextScanner struct{} + +func (fakeTextScanner) ScanInput(text string) TextVerdict { + lower := strings.ToLower(text) + var v TextVerdict + if strings.Contains(lower, "ignore all previous instructions") { + v.Blocked, v.CriticalInjection = true, true + v.Findings = append(v.Findings, TextFinding{Kind: "injection", Snippet: "ignore all previous instructions", Rule: "fake-injection", High: true}) + } + if strings.Contains(lower, "github_token") { + v.Blocked = true + v.Findings = append(v.Findings, TextFinding{Kind: "secret", Snippet: "github_token: ***", Rule: "fake-secret", High: true}) + } + if strings.Contains(lower, "rm -rf") { + v.Blocked = true + v.Findings = append(v.Findings, TextFinding{Kind: "dangerous", Snippet: "rm -rf", Rule: "fake-dangerous", High: true}) + } + return v +} + +func (f fakeTextScanner) ScanOutput(text string) TextVerdict { return f.ScanInput(text) } + +func init() { SetDefaultTextScanner(fakeTextScanner{}) } + +func TestDefaultSecurityScannerFailsClosedWithoutTextScanner(t *testing.T) { + SetDefaultTextScanner(nil) + t.Cleanup(func() { SetDefaultTextScanner(fakeTextScanner{}) }) + if DefaultTextScanner() != nil { + t.Fatal("nil install must uninstall") + } + _, err := NewDefaultSecurityScanner().Scan(context.Background(), ToolRequest{Tool: "Write", Target: "x", Content: "hello"}) + if !errors.Is(err, ErrNoTextScanner) { + t.Fatalf("err = %v, want ErrNoTextScanner", err) + } + // An explicit per-scanner TextScanner wins over the (missing) default. + res, err := (&DefaultSecurityScanner{Text: fakeTextScanner{}}).Scan(context.Background(), ToolRequest{Tool: "Write", Target: "x", Content: "hello"}) + if err != nil || !res.Passed { + t.Fatalf("explicit text scanner: res=%+v err=%v", res, err) + } +} + +func TestDefaultSecurityScannerOnlyHighFindingsBecomeViolations(t *testing.T) { + low := textScannerFunc(func(string) TextVerdict { + return TextVerdict{Blocked: true, Findings: []TextFinding{{Kind: "secret", Snippet: "x", Rule: "low", High: false}}} + }) + res, err := (&DefaultSecurityScanner{Text: low}).Scan(context.Background(), ToolRequest{Tool: "Write", Target: "x", Content: "hello"}) + if err != nil { + t.Fatal(err) + } + if len(res.Violations) != 0 || !res.Passed { + t.Fatalf("low-severity finding became a violation: %+v", res) + } +} + +type textScannerFunc func(string) TextVerdict + +func (f textScannerFunc) ScanInput(text string) TextVerdict { return f(text) } +func (f textScannerFunc) ScanOutput(text string) TextVerdict { return f(text) } diff --git a/src/pkg/toolapprove/toolapprove_test.go b/src/crux/toolapprove/toolapprove_test.go similarity index 98% rename from src/pkg/toolapprove/toolapprove_test.go rename to src/crux/toolapprove/toolapprove_test.go index c137129634..c03907110c 100644 --- a/src/pkg/toolapprove/toolapprove_test.go +++ b/src/crux/toolapprove/toolapprove_test.go @@ -5,9 +5,9 @@ import ( "strings" "testing" - "github.com/hivecommons/hive/pkg/agentmode" - "github.com/hivecommons/hive/pkg/crux/config" - "github.com/hivecommons/hive/pkg/crux/types" + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/agentmode" + "github.com/hivecommons/hive/crux/config" ) type fakeAuditSink struct { @@ -330,8 +330,8 @@ func TestEvaluateAndAuditEmitsToAuditSink(t *testing.T) { if rec.actor != "operator-alice" { t.Errorf("actor = %q, want 'operator-alice'", rec.actor) } - if rec.action != types.AuditToolApproval { - t.Errorf("action = %q, want %q", rec.action, types.AuditToolApproval) + if rec.action != crux.AuditToolApproval { + t.Errorf("action = %q, want %q", rec.action, crux.AuditToolApproval) } if rec.agentName != "quality" { t.Errorf("agentName = %q, want 'quality'", rec.agentName) diff --git a/src/pkg/toolapprove/types.go b/src/crux/toolapprove/types.go similarity index 98% rename from src/pkg/toolapprove/types.go rename to src/crux/toolapprove/types.go index 432fad6d23..29b6bb4eb8 100644 --- a/src/pkg/toolapprove/types.go +++ b/src/crux/toolapprove/types.go @@ -3,8 +3,8 @@ package toolapprove import ( "strings" - "github.com/hivecommons/hive/pkg/agentmode" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/agentmode" + "github.com/hivecommons/hive/crux/config" ) // Decision represents the tool approval outcome. diff --git a/src/pkg/toolapprove/wire.go b/src/crux/toolapprove/wire.go similarity index 97% rename from src/pkg/toolapprove/wire.go rename to src/crux/toolapprove/wire.go index e91b3f920b..75d3b5071d 100644 --- a/src/pkg/toolapprove/wire.go +++ b/src/crux/toolapprove/wire.go @@ -7,7 +7,7 @@ package toolapprove import ( "log/slog" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // DeskFromConfig builds a Desk and its operator Inbox from the loaded diff --git a/src/pkg/toolapprove/wire_test.go b/src/crux/toolapprove/wire_test.go similarity index 99% rename from src/pkg/toolapprove/wire_test.go rename to src/crux/toolapprove/wire_test.go index 743aad2709..f4579eae9c 100644 --- a/src/pkg/toolapprove/wire_test.go +++ b/src/crux/toolapprove/wire_test.go @@ -9,7 +9,7 @@ import ( "path/filepath" "testing" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // TestDeskFromConfigDisabledByDefault is the additive guarantee: an absent or diff --git a/src/pkg/trajectory/lane.go b/src/crux/trajectory/lane.go similarity index 100% rename from src/pkg/trajectory/lane.go rename to src/crux/trajectory/lane.go diff --git a/src/pkg/trajectory/lane_test.go b/src/crux/trajectory/lane_test.go similarity index 100% rename from src/pkg/trajectory/lane_test.go rename to src/crux/trajectory/lane_test.go diff --git a/src/pkg/trajectory/skip_test.go b/src/crux/trajectory/skip_test.go similarity index 100% rename from src/pkg/trajectory/skip_test.go rename to src/crux/trajectory/skip_test.go diff --git a/src/pkg/trajectory/trajectory.go b/src/crux/trajectory/trajectory.go similarity index 99% rename from src/pkg/trajectory/trajectory.go rename to src/crux/trajectory/trajectory.go index 8540f58993..1fcaa139ce 100644 --- a/src/pkg/trajectory/trajectory.go +++ b/src/crux/trajectory/trajectory.go @@ -21,7 +21,7 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/jsonextract" + "github.com/hivecommons/hive/crux/jsonextract" ) const ( diff --git a/src/pkg/trajectory/trajectory_test.go b/src/crux/trajectory/trajectory_test.go similarity index 100% rename from src/pkg/trajectory/trajectory_test.go rename to src/crux/trajectory/trajectory_test.go diff --git a/src/pkg/watchdog/classify.go b/src/crux/watchdog/classify.go similarity index 100% rename from src/pkg/watchdog/classify.go rename to src/crux/watchdog/classify.go diff --git a/src/pkg/watchdog/classify_copilot_login_test.go b/src/crux/watchdog/classify_copilot_login_test.go similarity index 100% rename from src/pkg/watchdog/classify_copilot_login_test.go rename to src/crux/watchdog/classify_copilot_login_test.go diff --git a/src/pkg/watchdog/classify_login_grace_test.go b/src/crux/watchdog/classify_login_grace_test.go similarity index 100% rename from src/pkg/watchdog/classify_login_grace_test.go rename to src/crux/watchdog/classify_login_grace_test.go diff --git a/src/pkg/watchdog/classify_test.go b/src/crux/watchdog/classify_test.go similarity index 100% rename from src/pkg/watchdog/classify_test.go rename to src/crux/watchdog/classify_test.go diff --git a/src/pkg/watchdog/conditions.go b/src/crux/watchdog/conditions.go similarity index 53% rename from src/pkg/watchdog/conditions.go rename to src/crux/watchdog/conditions.go index e183d0e3ee..daaa226e48 100644 --- a/src/pkg/watchdog/conditions.go +++ b/src/crux/watchdog/conditions.go @@ -13,47 +13,37 @@ // uncoordinated control loop. package watchdog -import "time" +import ( + "time" -// ConditionType identifies one axis of an agent's observed health, mirroring -// Kubernetes' status.conditions vocabulary. -type ConditionType string + "github.com/hivecommons/hive/crux" +) + +// ConditionType identifies one axis of an agent's observed health. It is +// defined in the crux root so that crux.Process can carry watchdog +// conditions without the root importing this package. +type ConditionType = crux.ConditionType +// Condition axes; see crux.ConditionReady and friends. const ( - // ConditionReady reports the liveness verdict: the agent's CLI is at its - // ready signature and able to accept work. - ConditionReady ConditionType = "Ready" - // ConditionAuthenticated reports the credential verdict: the provider - // credential behind the agent's backend is alive, independent of whether - // the CLI currently looks healthy (a dead refresh token can hide behind a - // live-looking pane for hours — failure mode 1 of RFC #4665). - ConditionAuthenticated ConditionType = "Authenticated" - // ConditionProducing reports the readiness verdict: the agent shows - // evidence of recent output (state-file activity), catching the - // idle-but-alive class invisible to liveness alone. - ConditionProducing ConditionType = "Producing" + ConditionReady = crux.ConditionReady + ConditionAuthenticated = crux.ConditionAuthenticated + ConditionProducing = crux.ConditionProducing ) -// ConditionStatus is the tri-state verdict of a condition. Unknown is a -// first-class value, never collapsed into True: a probe that could not -// determine the answer says so honestly (unknown ≠ healthy). -type ConditionStatus string +// ConditionStatus is the tri-state verdict of a condition; see +// crux.ConditionStatus. +type ConditionStatus = crux.ConditionStatus +// Condition statuses. const ( - ConditionTrue ConditionStatus = "True" - ConditionFalse ConditionStatus = "False" - ConditionUnknown ConditionStatus = "Unknown" + ConditionTrue = crux.ConditionTrue + ConditionFalse = crux.ConditionFalse + ConditionUnknown = crux.ConditionUnknown ) -// Condition is one observed-health axis for one agent, k8s-shaped so the -// dashboard (and any operator tooling) can consume it without translation. -type Condition struct { - Type ConditionType `json:"type"` - Status ConditionStatus `json:"status"` - Reason string `json:"reason,omitempty"` - Message string `json:"message,omitempty"` - LastTransitionTime time.Time `json:"lastTransitionTime"` -} +// Condition is one observed-health axis for one agent; see crux.Condition. +type Condition = crux.Condition // setCondition updates (or inserts) the condition of the given type in conds, // preserving LastTransitionTime when the status did not change — the k8s diff --git a/src/pkg/watchdog/conditions_test.go b/src/crux/watchdog/conditions_test.go similarity index 100% rename from src/pkg/watchdog/conditions_test.go rename to src/crux/watchdog/conditions_test.go diff --git a/src/pkg/watchdog/reconciler.go b/src/crux/watchdog/reconciler.go similarity index 96% rename from src/pkg/watchdog/reconciler.go rename to src/crux/watchdog/reconciler.go index 0f63af699e..1d1c9515b9 100644 --- a/src/pkg/watchdog/reconciler.go +++ b/src/crux/watchdog/reconciler.go @@ -8,19 +8,37 @@ import ( "sync" "time" - "github.com/hivecommons/hive/pkg/fleetreport" + "github.com/hivecommons/hive/crux" ) -// Fleet is the reconciler's window onto the agent manager. Implemented by -// pkg/agent's watchdog adapter so every observation and action reuses the -// manager's existing pane-capture, restart, and pause machinery — the -// watchdog never grows a second uncoordinated control loop. -type Fleet interface { +// Source is the half of Fleet only the runtime can answer: which agents to +// reconcile, what each one's pane and session look like, when it last +// produced, and where to publish the verdict. crux/agent's WatchdogFleet +// implements it over the manager's tables; `crux serve` exposes it over the +// API (hivecommons/hive#14138, stage 3) so a sidecar fleet is Source over +// the wire plus crux.Lifecycle for the actions. +type Source interface { // AgentNames lists the agents to reconcile. AgentNames() []string // Observe gathers one agent's observed truth. An error means the agent // could not be observed at all (removed mid-tick). Observe(name string) (Observation, error) + // LastProduction returns the newest production evidence timestamp for + // the agent (state-file mtimes, pane activity). ok=false means no + // evidence source is readable — reported as Producing=Unknown, never + // silently as healthy or as failed. + LastProduction(name string) (time.Time, bool) + // SetConditions publishes the agent's observed conditions so the + // dashboard renders truth instead of the state echo. + SetConditions(name string, conds []Condition) +} + +// Fleet is the reconciler's window onto the agent manager. Implemented by +// pkg/agent's watchdog adapter so every observation and action reuses the +// manager's existing pane-capture, restart, and pause machinery — the +// watchdog never grows a second uncoordinated control loop. +type Fleet interface { + Source // IsPaused reports the operator/escalation pause state; paused agents // are observed but never acted on. IsPaused(name string) bool @@ -31,11 +49,6 @@ type Fleet interface { // Pause pauses the agent, recording the trigger/reason (crash-loop // escalation). Pause(name, trigger, reason string) error - // LastProduction returns the newest production evidence timestamp for - // the agent (state-file mtimes, pane activity). ok=false means no - // evidence source is readable — reported as Producing=Unknown, never - // silently as healthy or as failed. - LastProduction(name string) (time.Time, bool) // QueuedWork reports how much actionable work is waiting for this agent, // and whether the count is known at all. An agent producing nothing while // nothing is queued is CORRECT, not unhealthy, so Producing=False requires @@ -44,9 +57,6 @@ type Fleet interface { // assumed empty (which would silence a real fault) or assumed full (which // would manufacture one). QueuedWork(name string) (count int, known bool) - // SetConditions publishes the agent's observed conditions so the - // dashboard renders truth instead of the state echo. - SetConditions(name string, conds []Condition) } // QueuedRef identifies one queued work item behind a QueuedWork count, so a @@ -314,6 +324,8 @@ type Reconciler struct { authProbes map[string]AuthProbe // backendProvider maps a backend name to its provider key. backendProvider func(backend string) string + // conditions receives the agent_crash_loop self-report; nil drops it. + conditions crux.ConditionReporter mu sync.Mutex agents map[string]*agentRecord @@ -359,6 +371,25 @@ func WithBackendProvider(f func(string) string) Option { return func(r *Reconciler) { r.backendProvider = f } } +// WithConditionReporter injects the host sink for the watchdog's +// agent_crash_loop self-report (hive wires pkg/fleetreport). Without one the +// crash loop still pauses and alerts; it just is not reported upstream. +func WithConditionReporter(rep crux.ConditionReporter) Option { + return func(r *Reconciler) { r.conditions = rep } +} + +func (r *Reconciler) observeCondition(class, key, message, diagnostics string) { + if r.conditions != nil { + r.conditions.Observe(class, key, message, diagnostics) + } +} + +func (r *Reconciler) clearCondition(class, key string) { + if r.conditions != nil { + r.conditions.Clear(class, key) + } +} + // New builds a Reconciler. alerter may be nil (alerts go to the journal via // logger only); logger must not be nil. func New(settings Settings, fleet Fleet, alerter Alerter, logger *slog.Logger, opts ...Option) *Reconciler { @@ -625,7 +656,7 @@ func (r *Reconciler) updateHealthyWindowLocked(name string, rec *agentRecord, cl fmt.Sprintf("continuously ready for %v; failure counter cleared (was %d, crash_looping=%v)", r.settings.HealthyReset, rec.Failures, rec.CrashLooping), name) if rec.CrashLooping { - fleetreport.Clear(fleetreport.ClassAgentCrashLoop, name) + r.clearCondition(crux.ClassAgentCrashLoop, name) } rec.Failures = 0 rec.CrashLooping = false @@ -686,7 +717,7 @@ func (r *Reconciler) planRestartLocked(name string, rec *agentRecord, cls Classi // Self-report (#13470, was the hive-code-defect "agent crash loop" // counter): the latch fires once per episode and the healthy-window // reset below clears it, so the fleet issue tracks real episodes. - fleetreport.Observe(fleetreport.ClassAgentCrashLoop, name, reason, + r.observeCondition(crux.ClassAgentCrashLoop, name, reason, fmt.Sprintf("agent=%s %s backoff_ladder=%v healthy_reset=%s", name, detail, r.settings.Backoff, r.settings.HealthyReset)) if r.alerter != nil { r.alerter.AddSystemAlert(crashLoopAlertID(name), severityError, diff --git a/src/pkg/watchdog/reconciler_options_test.go b/src/crux/watchdog/reconciler_options_test.go similarity index 100% rename from src/pkg/watchdog/reconciler_options_test.go rename to src/crux/watchdog/reconciler_options_test.go diff --git a/src/pkg/watchdog/reconciler_test.go b/src/crux/watchdog/reconciler_test.go similarity index 96% rename from src/pkg/watchdog/reconciler_test.go rename to src/crux/watchdog/reconciler_test.go index c7456a963a..2f5fda55e0 100644 --- a/src/pkg/watchdog/reconciler_test.go +++ b/src/crux/watchdog/reconciler_test.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - "github.com/hivecommons/hive/pkg/fleetreport" + "github.com/hivecommons/hive/crux" "io" "log/slog" "strings" @@ -429,8 +429,7 @@ func TestDeadAgentRestartWithExponentialBackoff(t *testing.T) { } func TestCrashLoopEscalatesToPauseAndAlertOnce(t *testing.T) { - tr := fleetreport.NewRuntimeTracker(nil) - fleetreport.SetDefaultTrackerForTest(t, tr) + tr := &fakeConditions{} fleet := newFakeFleet("a1") fleet.setObs("a1", deadObs()) clock := newFakeClock() @@ -438,7 +437,7 @@ func TestCrashLoopEscalatesToPauseAndAlertOnce(t *testing.T) { s := fastSettings() s.CrashLoopAfter = 2 s.Backoff = []time.Duration{time.Millisecond} - r := newTestReconciler(t, s, fleet, alerter, clock) + r := newTestReconciler(t, s, fleet, alerter, clock, WithConditionReporter(tr)) for i := 0; i < 2; i++ { r.Tick(context.Background()) @@ -469,7 +468,7 @@ func TestCrashLoopEscalatesToPauseAndAlertOnce(t *testing.T) { // The latch is the agent_crash_loop self-report (#13470): one evidence // entry naming the agent with the classifier detail as diagnostics. ev := crashLoopEvidence(tr) - if len(ev) != 1 || ev[0].Agent != "a1" || ev[0].Count != 1 || !strings.Contains(ev[0].Diagnostics, "agent=a1") || !strings.Contains(ev[0].Diagnostics, "failures=") { + if len(ev) != 1 || ev[0].Key != "a1" || !strings.Contains(ev[0].Diagnostics, "agent=a1") || !strings.Contains(ev[0].Diagnostics, "failures=") { t.Fatalf("crash-loop self-report evidence = %#v", ev) } @@ -480,24 +479,61 @@ func TestCrashLoopEscalatesToPauseAndAlertOnce(t *testing.T) { if fleet.pauseCount() != 1 || fleet.restartCount() != 2 { t.Fatal("latched crash loop must not pause or restart again") } - if ev := crashLoopEvidence(tr); len(ev) != 1 || ev[0].Count != 1 { + if ev := crashLoopEvidence(tr); len(ev) != 1 { t.Fatalf("latched crash loop re-observed on later sweeps: %#v", ev) } } -func crashLoopEvidence(tr *fleetreport.RuntimeTracker) []fleetreport.Evidence { - var out []fleetreport.Evidence - for _, e := range tr.Evidence() { - if e.Class == fleetreport.ClassAgentCrashLoop { - out = append(out, e) +// fakeConditions is the test crux.ConditionReporter: it records every +// Observe and Clear so a test can assert exactly what the watchdog emitted. +// Streak and persistence semantics are the host's (pkg/fleetreport). +type fakeConditions struct { + mu sync.Mutex + observed []fakeCondition + cleared []fakeCondition +} + +type fakeCondition struct{ Class, Key, Message, Diagnostics string } + +func (f *fakeConditions) Observe(class, key, message, diagnostics string) { + f.mu.Lock() + defer f.mu.Unlock() + f.observed = append(f.observed, fakeCondition{class, key, message, diagnostics}) +} + +func (f *fakeConditions) Clear(class, key string) { + f.mu.Lock() + defer f.mu.Unlock() + f.cleared = append(f.cleared, fakeCondition{Class: class, Key: key}) +} + +// crashLoopEvidence returns the agent_crash_loop observations not cleared +// since. +func crashLoopEvidence(f *fakeConditions) []fakeCondition { + f.mu.Lock() + defer f.mu.Unlock() + open := map[string]bool{} + for _, o := range f.observed { + if o.Class == crux.ClassAgentCrashLoop { + open[o.Key] = true + } + } + for _, c := range f.cleared { + if c.Class == crux.ClassAgentCrashLoop { + open[c.Key] = false + } + } + var out []fakeCondition + for _, o := range f.observed { + if o.Class == crux.ClassAgentCrashLoop && open[o.Key] { + out = append(out, o) } } return out } func TestHealthyWindowResetsFailuresAndClearsAlert(t *testing.T) { - tr := fleetreport.NewRuntimeTracker(nil) - fleetreport.SetDefaultTrackerForTest(t, tr) + tr := &fakeConditions{} fleet := newFakeFleet("a1") fleet.setObs("a1", deadObs()) clock := newFakeClock() @@ -506,7 +542,7 @@ func TestHealthyWindowResetsFailuresAndClearsAlert(t *testing.T) { s.CrashLoopAfter = 2 s.Backoff = []time.Duration{time.Millisecond} s.HealthyReset = 10 * time.Minute - r := newTestReconciler(t, s, fleet, alerter, clock) + r := newTestReconciler(t, s, fleet, alerter, clock, WithConditionReporter(tr)) // Drive into crash loop. for i := 0; i < 2; i++ { @@ -1137,8 +1173,7 @@ func TestObserveModeTakesNoAction(t *testing.T) { // TestObserveModeRecordsCrashLoopWithoutPausing covers the terminal decision: // the entry an operator reads to decide whether to enable heal. func TestObserveModeRecordsCrashLoopWithoutPausing(t *testing.T) { - tr := fleetreport.NewRuntimeTracker(nil) - fleetreport.SetDefaultTrackerForTest(t, tr) + tr := &fakeConditions{} fleet := newFakeFleet("a1") fleet.setObs("a1", deadObs()) clock := newFakeClock() @@ -1146,7 +1181,7 @@ func TestObserveModeRecordsCrashLoopWithoutPausing(t *testing.T) { s := fastSettings() s.Mode = ModeObserve s.CrashLoopAfter = 2 - r := newTestReconciler(t, s, fleet, alerter, clock) + r := newTestReconciler(t, s, fleet, alerter, clock, WithConditionReporter(tr)) // Seed a record already at the cap, as a heal-mode run would have left it. r.mu.Lock() diff --git a/src/pkg/watchdog/refreshable_credential_test.go b/src/crux/watchdog/refreshable_credential_test.go similarity index 100% rename from src/pkg/watchdog/refreshable_credential_test.go rename to src/crux/watchdog/refreshable_credential_test.go diff --git a/src/pkg/watchdog/rotation_probe.go b/src/crux/watchdog/rotation_probe.go similarity index 75% rename from src/pkg/watchdog/rotation_probe.go rename to src/crux/watchdog/rotation_probe.go index 84673b989a..fa80fa5470 100644 --- a/src/pkg/watchdog/rotation_probe.go +++ b/src/crux/watchdog/rotation_probe.go @@ -3,8 +3,6 @@ package watchdog import ( "context" "strings" - - "github.com/hivecommons/hive/pkg/rotation" ) // RotationAuthProbe adapts a rotation-package provider prober (#4608) into a @@ -14,7 +12,8 @@ import ( // answer proves the credential is alive, and a failure whose output shows // auth chrome proves it is dead. When PR #4645 rewrites these probers over // OAuth/JSON-RPC, this adapter picks the rewrite up unchanged: it depends -// only on the rotation.Prober contract. +// only on the UsageProber slice of the rotation.Prober contract, which +// cmd/hive adapts (the watchdog does not import pkg/rotation). // // Verdict mapping (deny silent failures — unknown ≠ healthy AND ≠ dead): // - probe answered → AuthOK (the credential produced an answer) @@ -22,7 +21,16 @@ import ( // login/auth chrome → AuthFailed // - any other probe error → AuthUnknown (couldn't probe is not a verdict) type RotationAuthProbe struct { - Prober rotation.Prober + Prober UsageProber +} + +// UsageProber is what the auth probe reads from a provider usage prober: +// its provider key and the sanitised text of the probe's error, "" when the +// probe answered. It is the two-method slice of rotation.HeadroomSource the +// adapter needs; hive wraps each rotation prober in cmd/hive. +type UsageProber interface { + Provider() string + ProbeError(ctx context.Context) string } // authErrorMarkers are substrings in a probe error that prove the failure is @@ -42,8 +50,7 @@ var authErrorMarkers = []string{ func (p RotationAuthProbe) Provider() string { return p.Prober.Provider() } func (p RotationAuthProbe) ProbeAuth(ctx context.Context) (AuthStatus, string) { - h := p.Prober.Probe(ctx) - errText := h.ProbeError() + errText := p.Prober.ProbeError(ctx) if errText == "" { return AuthOK, "provider usage probe answered" } diff --git a/src/crux/watchdog/rotation_probe_test.go b/src/crux/watchdog/rotation_probe_test.go new file mode 100644 index 0000000000..f7fa7f5534 --- /dev/null +++ b/src/crux/watchdog/rotation_probe_test.go @@ -0,0 +1,45 @@ +package watchdog + +import ( + "context" + "testing" +) + +type scriptedProber struct { + provider string + probeErr string +} + +func (p scriptedProber) Provider() string { return p.provider } +func (p scriptedProber) ProbeError(context.Context) string { return p.probeErr } + +func TestRotationAuthProbeVerdicts(t *testing.T) { + cases := []struct { + name string + probeErr string + want AuthStatus + }{ + {"clean answer proves the credential", "", AuthOK}, + {"401 in the error is an auth failure", "claude probe failed: HTTP 401", AuthFailed}, + {"login-expired chrome is an auth failure", "output: ● Login Expired · please run /login", AuthFailed}, + {"unauthorized is an auth failure", "Unauthorized: token rejected", AuthFailed}, + {"invalid api key is an auth failure", "Invalid API key provided", AuthFailed}, + {"transport error is honestly unknown", "dial tcp: connection refused", AuthUnknown}, + {"parse error is honestly unknown", "claude usage output did not match", AuthUnknown}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := RotationAuthProbe{Prober: scriptedProber{provider: "anthropic", probeErr: tc.probeErr}} + if p.Provider() != "anthropic" { + t.Fatal("provider must pass through") + } + got, detail := p.ProbeAuth(context.Background()) + if got != tc.want { + t.Fatalf("ProbeAuth = %s (%s), want %s", got, detail, tc.want) + } + if detail == "" { + t.Fatal("every verdict must carry its evidence") + } + }) + } +} diff --git a/src/pkg/watchdog/settings.go b/src/crux/watchdog/settings.go similarity index 99% rename from src/pkg/watchdog/settings.go rename to src/crux/watchdog/settings.go index 1b3118b363..df9f6a3cca 100644 --- a/src/pkg/watchdog/settings.go +++ b/src/crux/watchdog/settings.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) // RFC #4665 defaults. The zero value of every governor.watchdog config field diff --git a/src/pkg/watchdog/settings_test.go b/src/crux/watchdog/settings_test.go similarity index 99% rename from src/pkg/watchdog/settings_test.go rename to src/crux/watchdog/settings_test.go index 75f4c4e6a0..44fa32f327 100644 --- a/src/pkg/watchdog/settings_test.go +++ b/src/crux/watchdog/settings_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/hivecommons/hive/pkg/crux/config" + "github.com/hivecommons/hive/crux/config" ) func TestSettingsFromZeroValueIsRFCDefaults(t *testing.T) { diff --git a/src/crux/watchdog_conditions.go b/src/crux/watchdog_conditions.go new file mode 100644 index 0000000000..6513d9230f --- /dev/null +++ b/src/crux/watchdog_conditions.go @@ -0,0 +1,43 @@ +package crux + +import "time" + +// ConditionType identifies one axis of an agent's observed health, mirroring +// Kubernetes' status.conditions vocabulary. +type ConditionType string + +const ( + // ConditionReady reports the liveness verdict: the agent's CLI is at its + // ready signature and able to accept work. + ConditionReady ConditionType = "Ready" + // ConditionAuthenticated reports the credential verdict: the provider + // credential behind the agent's backend is alive, independent of whether + // the CLI currently looks healthy (a dead refresh token can hide behind a + // live-looking pane for hours — failure mode 1 of RFC #4665). + ConditionAuthenticated ConditionType = "Authenticated" + // ConditionProducing reports the readiness verdict: the agent shows + // evidence of recent output (state-file activity), catching the + // idle-but-alive class invisible to liveness alone. + ConditionProducing ConditionType = "Producing" +) + +// ConditionStatus is the tri-state verdict of a condition. Unknown is a +// first-class value, never collapsed into True: a probe that could not +// determine the answer says so honestly (unknown ≠ healthy). +type ConditionStatus string + +const ( + ConditionTrue ConditionStatus = "True" + ConditionFalse ConditionStatus = "False" + ConditionUnknown ConditionStatus = "Unknown" +) + +// Condition is one observed-health axis for one agent, k8s-shaped so the +// dashboard (and any operator tooling) can consume it without translation. +type Condition struct { + Type ConditionType `json:"type"` + Status ConditionStatus `json:"status"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` + LastTransitionTime time.Time `json:"lastTransitionTime"` +} diff --git a/src/pkg/watsonx/endpoint.go b/src/crux/watsonx/endpoint.go similarity index 100% rename from src/pkg/watsonx/endpoint.go rename to src/crux/watsonx/endpoint.go diff --git a/src/pkg/watsonx/endpoint_test.go b/src/crux/watsonx/endpoint_test.go similarity index 100% rename from src/pkg/watsonx/endpoint_test.go rename to src/crux/watsonx/endpoint_test.go diff --git a/src/pkg/watsonx/iam.go b/src/crux/watsonx/iam.go similarity index 100% rename from src/pkg/watsonx/iam.go rename to src/crux/watsonx/iam.go diff --git a/src/pkg/watsonx/iam_test.go b/src/crux/watsonx/iam_test.go similarity index 100% rename from src/pkg/watsonx/iam_test.go rename to src/crux/watsonx/iam_test.go diff --git a/src/deploy/README.md b/src/deploy/README.md index c1e4ea0b02..1dbfd34df9 100644 --- a/src/deploy/README.md +++ b/src/deploy/README.md @@ -6,7 +6,7 @@ This directory contains Kubernetes/LXC manifests plus runtime helper scripts use ### `ttyd-tmux.sh` -`ttyd` serves the browser terminal websocket, but agent tmux sessions may be owned by per-agent UIDs. Because tmux only lets the socket owner attach, `ttyd-tmux.sh ` finds the matching `/tmp/tmux-*/` socket, waits for it when an agent is still being launched (default `900` seconds, override with `HIVE_TTYD_WAIT_SEC`; poll interval `HIVE_TTYD_POLL_SEC`, default `2`), derives its numeric UID/GID, and uses `su-exec` to attach as that owner. While attached it enables tmux mouse mode (so the scroll wheel drives copy-mode scrollback instead of just reflowing the viewport — issue #3694) and raises the session `history-limit` (default `50000`, override with `HIVE_TTYD_HISTORY_LIMIT`) for panes created later in the session. Both options are restored to their previous values on detach. Note that tmux reads `history-limit` at pane creation, so the attach-time raise cannot deepen an already-created pane — the authoritative deep-scrollback depth is set when the agent manager creates the session (`newSessionCommands` in `src/pkg/agent/manager.go`, default `50000`, override with `HIVE_TMUX_HISTORY_LIMIT`). +`ttyd` serves the browser terminal websocket, but agent tmux sessions may be owned by per-agent UIDs. Because tmux only lets the socket owner attach, `ttyd-tmux.sh ` finds the matching `/tmp/tmux-*/` socket, waits for it when an agent is still being launched (default `900` seconds, override with `HIVE_TTYD_WAIT_SEC`; poll interval `HIVE_TTYD_POLL_SEC`, default `2`), derives its numeric UID/GID, and uses `su-exec` to attach as that owner. While attached it enables tmux mouse mode (so the scroll wheel drives copy-mode scrollback instead of just reflowing the viewport — issue #3694) and raises the session `history-limit` (default `50000`, override with `HIVE_TTYD_HISTORY_LIMIT`) for panes created later in the session. Both options are restored to their previous values on detach. Note that tmux reads `history-limit` at pane creation, so the attach-time raise cannot deepen an already-created pane — the authoritative deep-scrollback depth is set when the agent manager creates the session (`newSessionCommands` in `src/crux/agent/manager_tmux.go`, default `50000`, override with `HIVE_TMUX_HISTORY_LIMIT`). Use it indirectly through the dashboard terminal link. If a terminal pane says it is waiting for an agent to start, the dashboard reached ttyd before that agent's tmux session existed; it should attach automatically when the socket appears. If it gives up and says no tmux socket was found, check that the agent session name exists and that the socket is present under `/tmp/tmux-*` in the container. The dashboard opens ttyd's bundled terminal page rather than owning the terminal WebSocket client, so reconnect throttling for that page lives in ttyd; this wrapper prevents reconnect storms by keeping the ttyd child alive while the socket is legitimately pending. diff --git a/src/deploy/ci-runners/Dockerfile b/src/deploy/ci-runners/Dockerfile index 348d1dffe2..817b9fa0f6 100644 --- a/src/deploy/ci-runners/Dockerfile +++ b/src/deploy/ci-runners/Dockerfile @@ -48,7 +48,7 @@ ARG RUNNER_BASE_IMAGE=summerwind/actions-runner:v2.337.0-ubuntu-24.04 FROM ${RUNNER_BASE_IMAGE} -ARG GITHUB_CLI_VERSION=2.102.1 +ARG GITHUB_CLI_VERSION=2.103.3 ARG GITHUB_CLI_KEYRING_SHA256=6084d5d7bd8e288441e0e94fc6275570895da18e6751f70f057485dc2d1a811b # The base image runs as uid 1001 (`runner`), which does not own /etc/apt. diff --git a/src/deploy/entrypoint.sh b/src/deploy/entrypoint.sh index 0b4302e551..8263cad13a 100644 --- a/src/deploy/entrypoint.sh +++ b/src/deploy/entrypoint.sh @@ -119,7 +119,7 @@ hive_run_uid_isolation_migration() { if [ "${HIVE_HOME_MIGRATION_NEEDED:-false}" = "true" ]; then echo "[entrypoint] UID isolation: repairing shared home in background" - chown -R dev:node /data/home/.claude /data/home/.codex /data/home/.bob 2>/dev/null \ + chown -R dev:node /data/home/.claude /data/home/.codex /data/home/.gemini /data/home/.bob 2>/dev/null \ || echo "[entrypoint] WARN: shared-home chown was incomplete; continuing with the existing fail-open permission policy" if [ "${HIVE_HOME_MODE_REPAIR_NEEDED:-false}" = "true" ]; then chmod -R g+rwX /data/home 2>/dev/null \ @@ -1425,8 +1425,15 @@ if [ "$(id -u)" = "0" ]; then # pre-create it group-writable + setgid so every agent UID (node group) can # init it, matching .claude/.copilot. Without this, switching an agent to the # codex backend fails with "Permission denied" initializing state_N.sqlite. + # + # Only the directory itself is re-owned here. The recursive pass over the + # existing tree (tens of thousands of session/sqlite files on a mature spoke) + # runs in hive_run_uid_isolation_migration, off PID 1's startup path: on a + # shared NFS server a synchronous `chown -R` here held the fleet's daily + # rollout at ~3 min per boot, past the stale 160s startup budget, and 40 LKE + # spokes crash-looped in lock-step (2026-10-26). chmod 2775 /data/home/.codex 2>/dev/null || true - chown -R dev:node /data/home/.codex 2>/dev/null || true + chown dev:node /data/home/.codex 2>/dev/null || true # Antigravity (`agy`) persists its OAuth session to # $HOME/.gemini/antigravity-cli/antigravity-oauth-token. Pre-create the tree # group-writable + setgid for the same reason as .codex above. @@ -1450,7 +1457,9 @@ if [ "$(id -u)" = "0" ]; then # watch. Creating it here removes that dependency entirely: the directory is # on disk, with the right mode, before any watch is set up. chmod 2770 /data/home/.gemini /data/home/.gemini/antigravity-cli /data/home/.gemini/antigravity-cli/cache 2>/dev/null || true - chown -R dev:node /data/home/.gemini 2>/dev/null || true + # Non-recursive for the same reason as .codex above; the tree is re-owned in + # the background worker. + chown dev:node /data/home/.gemini /data/home/.gemini/antigravity-cli /data/home/.gemini/antigravity-cli/cache 2>/dev/null || true # bob writes installation_id, settings.json, trustedFolders.json and tmp/ under # $HOME/.bob, plus custom modes under $HOME/.bob/settings. Pre-create both # group-writable + setgid so the FIRST agent to launch (a 2001+ UID in group @@ -2581,7 +2590,7 @@ for path in (os.environ.get('HIVE_UID_MAP_PERSISTED', '/data/.hive/uid-map.json' # So do NOT promise it here. Report what is actually in the chain, which is # verifiable, instead of asserting a backstop that may not be loaded. # The Go proxy logs the unmarked dial once - # (src/pkg/proxy/github_proxy.go warnSockMarkOnce). + # (src/crux/proxy/github_proxy.go warnSockMarkOnce). if iptables -t nat -C HIVE_PROXY -m owner --uid-owner "$PROXY_UID" -j RETURN 2>/dev/null; then _owner_backstop="present (xt_owner loaded) — proxy dials still exempt via owner-UID" else diff --git a/src/deploy/k8s/deployment.yaml b/src/deploy/k8s/deployment.yaml index 3dcf454c98..dc6ccb3630 100644 --- a/src/deploy/k8s/deployment.yaml +++ b/src/deploy/k8s/deployment.yaml @@ -195,14 +195,15 @@ spec: # startupProbe holds the liveness clock off until the process has # finished booting, so a slow start can never race the liveness # timer into a restart loop. failureThreshold x periodSeconds - # (30 x 5s = 150s) is the boot budget. + # plus initialDelaySeconds (10s + 118 x 5s = 600s) matches + # spoke.HostedStartupBudget; keep in sync with its probe constants. startupProbe: httpGet: path: /api/health port: dashboard initialDelaySeconds: 10 periodSeconds: 5 - failureThreshold: 30 + failureThreshold: 118 livenessProbe: # /api/livez checks that the HTTP server responds AND that the # heartbeat loop is still *attempting* sends — i.e. it only fails diff --git a/src/deploy/probe_podman_rootful_netadmin.sh b/src/deploy/probe_podman_rootful_netadmin.sh index 1c34509a1b..d8f0f15eb8 100755 --- a/src/deploy/probe_podman_rootful_netadmin.sh +++ b/src/deploy/probe_podman_rootful_netadmin.sh @@ -238,7 +238,7 @@ if [[ "$SOMARK" == "true" ]]; then pod exec "${PROBE_PREFIX}-live" iptables-nft -t nat -S HIVE_PROXY 2>&1 | sed 's/^/ /' # CONTROL. curl exec'd as the proxy UID carries NO packet mark: SO_MARK is set - # by the proxy's own dialer (pkg/proxy/somark_linux.go), and an exec'd process + # by the proxy's own dialer (crux/proxy/somark_linux.go), and an exec'd process # gets neither that code path nor the ambient CAP_NET_ADMIN that setting a mark # requires. So this line measures the OWNER rule only, never the mark. printf ' control, unmarked uid-1001 dial, owner rules present: %s\n' \ diff --git a/src/deploy/probe_podman_rootless_netadmin.sh b/src/deploy/probe_podman_rootless_netadmin.sh index ee62c6ccc4..910736eec0 100755 --- a/src/deploy/probe_podman_rootless_netadmin.sh +++ b/src/deploy/probe_podman_rootless_netadmin.sh @@ -91,7 +91,9 @@ trap cleanup EXIT if [[ "$SHARED_STORE" != "true" ]]; then if [[ -z "$STORE" ]]; then - STORE="$(mktemp -d -t hive-rootless-probe-store-XXXXXX)" + # Keep multi-GB image layers off quota-limited /tmp (or tmpfs), as the + # rootful probe does. The private store is still removed by cleanup. + STORE="$(mktemp -d -p /var/tmp hive-rootless-probe-store-XXXXXX)" OWN_STORE="true" fi mkdir -p "${STORE}/graph" "${STORE}/run" "${STORE}/tmp" @@ -144,7 +146,9 @@ printf 'podman=%s rootless=%s cgroups=%s netbackend=%s rootlessnet=%s runtime=%s "$rootlessnet" "$(pinfo .Host.OCIRuntime.Name)" printf 'store=%s\nimage=%s\n\n' "${CONTAINERS_STORAGE_CONF:-}" "$IMAGE" -podman pull -q "$IMAGE" >/dev/null 2>&1 || fail_prereq "cannot pull ${IMAGE}" +# Keep registry and storage errors visible: exit 78 alone cannot distinguish +# an unavailable image from a transient registry failure or a full runner disk. +podman pull -q "$IMAGE" || fail_prereq "cannot pull ${IMAGE}" failures=0 note_fail() { printf ' RESULT: UNEXPECTED — %s\n' "$1"; failures=$((failures + 1)); } diff --git a/src/deploy/test_attach_hint_runtime.sh b/src/deploy/test_attach_hint_runtime.sh index adb620ea19..2b5d49d907 100755 --- a/src/deploy/test_attach_hint_runtime.sh +++ b/src/deploy/test_attach_hint_runtime.sh @@ -68,21 +68,22 @@ done # ── 3. The entrypoint's hint renders the runtime it is given ──────────────── HINT_LINE="$(grep -F 'echo " Tmux:' "$AGENT" | head -1)" +HINT_SETUP="$(awk '/^CONTAINER_NAME=/{inblock=1} /^echo "Contributor agent is running\."/{exit} inblock{print}' "$AGENT")" # render_hint — evaluates the shipped echo -# with the same defaulting the entrypoint applies around it. env -i so a +# and its runtime selection/defaults from the entrypoint. env -i so a # HIVE_CONTAINER_RUNTIME exported by whoever runs this suite cannot leak in. render_hint() { - env -i CONTAINER="$1" RT="${2-}" HINT="$HINT_LINE" bash -c ' - CONTAINER_NAME="${CONTAINER}" - CONTAINER_RUNTIME="${RT:-docker}" + env -i HIVE_CONTAINER_NAME="$1" HIVE_CONTAINER_RUNTIME="${2-}" \ + SETUP="$HINT_SETUP" HINT="$HINT_LINE" bash -c ' TMUX_SESSION="contributor" + eval "$SETUP" eval "$HINT" ' } -if [ -z "$HINT_LINE" ]; then - fail "extract the Tmux hint from bin/contributor-agent.sh" \ +if [ -z "$HINT_LINE" ] || [ -z "$HINT_SETUP" ]; then + fail "extract the Tmux hint and setup from bin/contributor-agent.sh" \ "the anchor moved — this test cannot verify the real hint" else pass "Tmux hint extracted from bin/contributor-agent.sh (not restated here)" @@ -96,6 +97,17 @@ else want: $WANT" fi + for runtime in apptainer singularity; do + GOT="$(render_hint hive-contributor-agy-5b4f "$runtime")" + WANT=" Tmux: $runtime exec instance://hive-contributor-agy-5b4f tmux attach -t contributor" + if [ "$GOT" = "$WANT" ]; then + pass "a $runtime launch attaches to its named instance without -it" + else + fail "a $runtime launch attaches to its named instance without -it" "got: $GOT + want: $WANT" + fi + done + # An older image, or a launch by anything that does not pass the variable, # must print exactly what it printed before the fix. GOT="$(render_hint hive-contributor '')" diff --git a/src/deploy/test_entrypoint_token_access_audit.sh b/src/deploy/test_entrypoint_token_access_audit.sh index 53e2945946..bb4c72adca 100755 --- a/src/deploy/test_entrypoint_token_access_audit.sh +++ b/src/deploy/test_entrypoint_token_access_audit.sh @@ -92,7 +92,7 @@ fi # 7. The Go permissions watcher never has /var/run/hive-metrics in its roots, # so nothing running as dev after boot can re-open the hole (#6238 / #6277 # moved umask handling around; the watcher's roots are the only mode-fixer). -if grep -q '"/var/run/hive-metrics' "${REPO_ROOT}/src/pkg/agent/permissions_watcher.go"; then +if grep -q '"/var/run/hive-metrics' "${REPO_ROOT}/src/crux/agent/permissions_watcher.go"; then bad "the permissions watcher walks /var/run/hive-metrics and could loosen the audit log" else ok "the permissions watcher does not walk /var/run/hive-metrics" diff --git a/src/deploy/test_entrypoint_uid_isolation.sh b/src/deploy/test_entrypoint_uid_isolation.sh index 9a3861da0f..8b6c0b924a 100644 --- a/src/deploy/test_entrypoint_uid_isolation.sh +++ b/src/deploy/test_entrypoint_uid_isolation.sh @@ -33,6 +33,29 @@ else "expected recursive repair in the worker and an '&' at its call site" fi +# The shared-home preparation (/data/home/.codex, .gemini, .bob, ...) is also +# on PID 1's startup path and runs on EVERY boot, not just a version flip. A +# recursive chown there over a mature spoke's tens of thousands of session +# files on shared NFS pushed boot past the startup budget and crash-looped 40 +# spokes in lock-step (2026-10-26). Only the directories themselves may be +# re-owned synchronously; the tree belongs to the background worker. +shared_home="$(sed -n '/Shared CLI auth\/cache lives in \/data\/home/,/ownership marker and perms OK/p' "$ENTRYPOINT")" +if [ -z "$shared_home" ]; then + bad "could not extract the shared-home preparation block from entrypoint.sh" +elif grep -qE '^[[:space:]]*(chown|chmod|chgrp)[[:space:]]+-[^[:space:]]*R[^[:space:]]*[[:space:]].*/data/home' <<<"$shared_home"; then + bad "the synchronous shared-home preparation contains a recursive walk under /data/home" \ + "$(grep -nE '^[[:space:]]*(chown|chmod|chgrp)[[:space:]]+-[^[:space:]]*R' <<<"$shared_home")" +else + ok "the synchronous shared-home preparation is O(number of directories)" +fi + +if grep -qE 'chown -R dev:node .*/data/home/\.codex' <<<"$worker" \ + && grep -qE 'chown -R dev:node .*/data/home/\.gemini' <<<"$worker"; then + ok "the background worker re-owns the .codex and .gemini trees" +else + bad "the background worker does not cover both /data/home/.codex and /data/home/.gemini" +fi + if grep -q "isolation_marker_dir" "$ENTRYPOINT" \ && grep -q "isolation_revision" "$ENTRYPOINT"; then ok "uid-map.json carries the marker contract to the Go manager" diff --git a/src/deploy/test_terminal_scrollback_status.sh b/src/deploy/test_terminal_scrollback_status.sh index 8c020093d6..70cb696503 100755 --- a/src/deploy/test_terminal_scrollback_status.sh +++ b/src/deploy/test_terminal_scrollback_status.sh @@ -43,7 +43,7 @@ fi # The format under test is read from the shipped sources, never restated here: # a copy would keep passing after the real one regressed. -MANAGER="${ROOT}/src/pkg/agent/manager.go" +MANAGER="${ROOT}/src/crux/agent/manager.go" ATTACH="${ROOT}/src/deploy/ttyd-tmux.sh" STATUS_FMT="$(sed -n 's/^\ttmuxStatusRight = "\(.*\)"$/\1/p' "$MANAGER" | head -1)" if [ -z "$STATUS_FMT" ]; then @@ -52,7 +52,7 @@ if [ -z "$STATUS_FMT" ]; then echo "=== Results: $PASS passed, $FAIL failed ===" exit 1 fi -pass "status format extracted from src/pkg/agent/manager.go (not restated here)" +pass "status format extracted from src/crux/agent/manager.go (not restated here)" # The attach path must carry the SAME format, or a session created before the # manager change gets a different terminal than one created after. @@ -300,7 +300,7 @@ if tmux -S "$SOCK" list-commands copy-mode 2>/dev/null | grep -qE '\[-[a-zA-Z]*H # The rebind itself must be carried by BOTH the session-creation path and # the attach path, or a session created one way scrolls differently. grep -q 'copy-mode -eH' "$MANAGER" \ - && pass "src/pkg/agent/manager.go rebinds the wheel to copy-mode -eH" \ + && pass "src/crux/agent/manager.go rebinds the wheel to copy-mode -eH" \ || fail "manager.go carries the wheel rebind" grep -q "copy-mode -eH" "$ATTACH" \ && pass "src/deploy/ttyd-tmux.sh applies the identical rebind on attach" \ diff --git a/src/deploy/test_ttyd_url_arg.sh b/src/deploy/test_ttyd_url_arg.sh index 291e9d4428..ea138647a7 100755 --- a/src/deploy/test_ttyd_url_arg.sh +++ b/src/deploy/test_ttyd_url_arg.sh @@ -25,7 +25,7 @@ # The failure chain --url-arg sits at the head of: # 1. ttyd without -a discards the ?arg= in the URL # 2. ttyd-tmux.sh gets no argument and falls back to its default session name -# 3. real sessions are hive- (src/pkg/agent/manager.go), so the glob +# 3. real sessions are hive- (src/crux/agent/manager.go), so the glob # /tmp/tmux-*/ matches nothing # 4. "error: no tmux socket found for session 'supervisor'" — a message that # names a session the operator never asked for and never mentions --url-arg diff --git a/src/deploy/ttyd-tmux.sh b/src/deploy/ttyd-tmux.sh index 9a71f872f4..f9df51f6d5 100644 --- a/src/deploy/ttyd-tmux.sh +++ b/src/deploy/ttyd-tmux.sh @@ -148,7 +148,7 @@ CURRENT_UID=$(id -u) # NOTE: tmux reads history-limit at PANE creation, so the raise below CANNOT # deepen the pane that already exists when the browser attaches. The # authoritative deep-scrollback setting is applied at session creation by the -# agent manager (newSessionCommands in src/pkg/agent/manager.go, default 50000, +# agent manager (newSessionCommands in src/crux/agent/manager.go, default 50000, # override HIVE_TMUX_HISTORY_LIMIT). The attach-time raise is only defense in # depth for sessions created outside the manager. # @@ -173,7 +173,7 @@ TTYD_HISTORY_LIMIT="${HIVE_TTYD_HISTORY_LIMIT:-50000}" # output" report in #4399. # # The authoritative setting is applied at SESSION creation by the agent manager -# (newSessionCommands in src/pkg/agent/manager.go). This attach-time set is the +# (newSessionCommands in src/crux/agent/manager.go). This attach-time set is the # same defense in depth as the history-limit raise above: it reaches sessions # created before that change, or outside the manager. Saved and restored around # the attach so an agent CLI that manages its own status line is not permanently diff --git a/src/docs/README.md b/src/docs/README.md index 2097ae9f12..0da883c4dd 100644 --- a/src/docs/README.md +++ b/src/docs/README.md @@ -115,6 +115,7 @@ Start with [Zero to Automation: Getting Started with Hive](getting-started.md). ## Where to go next - [Architecture](architecture.md) +- [Integrations](integrations.md) — backends, trackers, forges and sandboxes Hive works with, at a glance. - [Getting started](getting-started.md) - [Operator reference](operator-reference.md) - [GitHub webhooks: cadence, patch and reconcile](webhooks.md) — polling fallback, webhook patch and reconcile, and the state-accuracy guarantees. diff --git a/src/docs/acmm-advisor.md b/src/docs/acmm-advisor.md index 489c7524e4..1fc17f3008 100644 --- a/src/docs/acmm-advisor.md +++ b/src/docs/acmm-advisor.md @@ -18,7 +18,7 @@ The advisor evaluates a fixed set of six signals (`Signals` struct, `src/pkg/acm | `HoldCount` | Count of open PRs still carrying a `hold` label awaiting human review. | | `HasQualityAgent` | Whether an active quality/coverage agent is present in the config. | -Levels run L1 (Assisted) through L6 (Fully Autonomous), matching the level pack files and the dashboard's 1–6 bound (`src/pkg/acmmadvisor/acmmadvisor.go:34-42`). The advisor never proposes a target more than one level above the current level — advancement is always incremental (`levelStep = 1`, same file). +Levels run L1 (Assisted) through L6 (Autonomous), matching the level pack files and the dashboard's 1–6 bound (`src/pkg/acmmadvisor/acmmadvisor.go:34-42`). The advisor never proposes a target more than one level above the current level — advancement is always incremental (`levelStep = 1`, same file). ### Where the live signals come from @@ -42,7 +42,7 @@ The advisor computes the checklist for advancing to the level **one above** the | L3 (Measured) | required | ≥ 40% | ≥ 3 | — | — | — | | L4 (Adaptive) | required | ≥ 60% | ≥ 5 | ≥ 70% | — | — | | L5 (Semi-Automated) | required | ≥ 75% | ≥ 8 | ≥ 85% | ≤ 25 | ≤ 15 | -| L6 (Fully Autonomous) | required | ≥ 90% | ≥ 12 | ≥ 95% | ≤ 10 | ≤ 5 | +| L6 (Autonomous) | required | ≥ 90% | ≥ 12 | ≥ 95% | ≤ 10 | ≤ 5 | The L2 pack includes a COMMENT-only `reviewer` as the review-first entry point, and the reviewer ships in every pack from L2 up (L3/L4 review the `hold`-gated queue with no merge power; L5/L6 per [#8023](https://github.com/hivecommons/hive/issues/8023)); the advisor criteria above are unchanged by it. Decision taken by hive: review is part of L2 proper (COMMENT-only); no L2.5 level — a new level would ripple through level enums, advisor and dashboard; keeping L2 triage+review preserves the review-first entry point the vote asked for. diff --git a/src/docs/acmm-policy-matrix.md b/src/docs/acmm-policy-matrix.md index 69b5c2dea5..d73b2d6f44 100644 --- a/src/docs/acmm-policy-matrix.md +++ b/src/docs/acmm-policy-matrix.md @@ -103,7 +103,7 @@ Per-repo opt-in `auto_merge.queue_on_approve_and_unhold` (off by default) lets a | operations (paused) | holdgated | `operations-holdgated.md` | | brainstorm | advisory | `brainstorm-advisory.md` | -### L6 — Fully Autonomous (15 agents) +### L6 — Autonomous (15 agents) Existing autonomous lanes can open issues, create PRs, and auto-merge on green CI. Auto-merge is off below L6; switching to L6 turns it on for every active repository, after which owners can toggle repositories individually. L6 PRs do not get the level hold, except outreach PRs into a repository the hive does not manage, which are still held for human review; outreach PRs into the hive's own repos auto-merge like any other lane's. Contributor PRs can merge through the contributor lane when the author has an operator-trusted tier or the conservative `auto_merge.contributor_tier.auto_promote_after_merged` clean-history ratchet admits them; reviewer verdict, CI, diff fences, soak and branch protection still apply. Hive never needs App Actions write for `action_required` workflow runs: when trusted same-repo bot/App/registry-trusted clanker PRs hit the workflow-approval banner, Hive posts `/ok-to-test` once per head and the repository's self-contained `hive-ok-to-test` workflow approves those runs with its `GITHUB_TOKEN`; if the workflow is absent or a `sentinel-alert` is present, Hive surfaces `workflow-approval-required:` / stage 085 for a human decision. Outreach handles community engagement. Reviewer stays advisory unless the self-authored review gate is enforcing: `review.require_approval: true` (or `governor.review_gate.mode: enforce`) makes the App self-authored sweep require same-head aggregate approval, while `requires_human`/incomplete verdicts can be cleared by a trusted human's normal GitHub Approve review on that exact head. For Hive-authored PRs, `review.self_authored_decision_policy: resolve` (the default) lets the reviewer resolve every decision question itself except those in a named risk class (permissions/credentials, security, sensitive paths, data loss/irreversible, merge/trust policy, external side effects, licensing/legal, unverifiable) or marked `hive-decision-risk: high`; set `escalate` to park every decision question at 085. Swarm `changes_requested`/`reject` still requires a fix or direct human merge. Use actual holds or GitHub branch rules for a universal merge stop. Adjudicator is the reviewer lane: it repairs, de-escalates, or recommends closing (and may close) escalated `needs-human` hive PRs, and never merges. Dedupe posts the same daily consolidation checklist as at L5 and stays recommendation-only. Telemetry and operations remain paused and use `ISSUES_AND_PRS`, so they never merge their own PRs. diff --git a/src/docs/adr/0009-trajectory-review.md b/src/docs/adr/0009-trajectory-review.md index 1fdd05a233..5f7744ed67 100644 --- a/src/docs/adr/0009-trajectory-review.md +++ b/src/docs/adr/0009-trajectory-review.md @@ -17,8 +17,8 @@ Run a periodic trajectory-review lane from the governor tick. The lane snapshots running, non-exempt agents with their assigned intent and recent transcript, sends a bounded transcript tail to a second model through the existing OpenAI-compatible/LiteLLM endpoint, and asks for a compact JSON verdict -([reviewer](../../pkg/trajectory/trajectory.go), -[lane](../../pkg/trajectory/lane.go)). On divergence, the configured response is +([reviewer](../../crux/trajectory/trajectory.go), +[lane](../../crux/trajectory/lane.go)). On divergence, the configured response is to pause the agent and alert by default, or to alert only. The reviewer prompt instructs the model to judge direction, not single commands, and to answer non-divergent when unsure. @@ -49,7 +49,7 @@ still in progress can itself be the warning signal, so **"transcript bytes identical" is never a skip condition**. The lane now skips a review only when **all** of the following hold, as decided -by `skipDecision` in [lane.go](../../pkg/trajectory/lane.go): +by `skipDecision` in [lane.go](../../crux/trajectory/lane.go): 1. the agent has been reviewed at least once before; 2. the review is **not overdue** — fewer than `MaxSkipIntervals` (default 6) @@ -57,7 +57,7 @@ by `skipDecision` in [lane.go](../../pkg/trajectory/lane.go): 3. the agent manager reports the agent **idle** from its own state — the last kicked turn has ended (`KickOutcome` settled against `LastKick`), the agent is parked at the prompt awaiting CI, or it has never been kicked - (`trajectoryIdle` in [trajectory_view.go](../../pkg/agent/trajectory_view.go)); + (`trajectoryIdle` in [trajectory_view.go](../../crux/agent/trajectory_view.go)); 4. the manager has a known last-output time and it precedes the last review. If any condition fails the review runs. An unknown last-output time is treated diff --git a/src/docs/adr/0012-skill-registry.md b/src/docs/adr/0012-skill-registry.md index 5257c34598..c7cdd7f9f5 100644 --- a/src/docs/adr/0012-skill-registry.md +++ b/src/docs/adr/0012-skill-registry.md @@ -14,7 +14,7 @@ Hive a shareable catalog or version metadata. ## Decision Introduce `skillreg` as a concurrency-safe registry of named, versioned skill -files ([skill registry](../../pkg/skillreg/skillreg.go)). A skill is a Markdown +files ([skill registry](../../crux/skillreg/skillreg.go)). A skill is a Markdown body with optional YAML front matter for name, version, description, and tags. Missing skill directories, unreadable files, and malformed front matter are skipped with logs so one bad catalog entry does not prevent Hive from loading. diff --git a/src/docs/adr/0025-central-stage-engine.md b/src/docs/adr/0025-central-stage-engine.md index 66205e76aa..ad2849fa9b 100644 --- a/src/docs/adr/0025-central-stage-engine.md +++ b/src/docs/adr/0025-central-stage-engine.md @@ -21,13 +21,13 @@ re-read-then-write `Enforce` that heals duplicate stage pills records transitions, regressions and history hops (`src/pkg/stagelabels/store.go:397`, `IsRegression` at `src/pkg/stagelabels/store.go:540`). The dashboard activity tracker and the -hub read that store and nothing else (`src/pkg/dashboard/stage_tracker.go:584-588`). +hub read that store and nothing else (`src/pkg/dashboard/stage_tracker.go:957-962`). **What is not central — three gaps.** 1. *Two derivation engines.* Issues are staged by `IssueStageFor` / - `SyncIssueStageLabels` (`src/pkg/github/issue_stage_labels.go:145`, - `src/pkg/github/issue_stage_labels.go:233`) from the enumeration bucket. PRs + `SyncIssueStageLabels` (`src/pkg/github/issue_stage_labels.go:205`, + `src/pkg/github/issue_stage_labels.go:337`) from the enumeration bucket. PRs are staged by the automerge engine: each lane calls `recordStage` (`src/pkg/github/automerge/automerge_stage_labels.go:365`), observations are sorted by `stageLanePrecedence` (self 3 > trusted 2 > contributor 1, @@ -36,7 +36,7 @@ hub read that store and nothing else (`src/pkg/dashboard/stage_tracker.go:584-58 into a ranked opinion (`prstage.RankNone..prstage.RankOwned`, `src/pkg/stageengine/prstage/types.go:145-148`), and `ResolveStage` picks the winner - (`src/pkg/stageengine/derive_pr_classify.go:459`). These PR helpers now + (`src/pkg/stageengine/derive_pr_classify.go:454`). These PR helpers now live in the engine (#13779); automerge keeps only gathering-side previews. Shared concepts — parking (045/085, `needs-human`, hold labels), author kind (070/071/072), adoption (074), terminal states, `origin/*` — are modelled twice with @@ -52,7 +52,7 @@ hub read that store and nothing else (`src/pkg/dashboard/stage_tracker.go:584-58 the issue path uses `Client.StageLabeler` + `Enforce` (`src/pkg/github/stage_labeler.go:63`); the claim relay and ledger hooks write `stage/020` / `stage/030` directly as a side effect of a claim - (`src/cmd/hive/claimswire.go:196-201`, `src/cmd/hive/claimswire.go:158-171`, + (`src/cmd/hive/claimswire.go:236-238`, `src/cmd/hive/claimswire.go:176-178`, `src/pkg/github/issue_inprogress.go:126-141`, `src/pkg/github/issue_inprogress.go:186`, `src/pkg/github/stage_labeler.go:114`); the PR-claim sync writes 040 (`src/pkg/github/prclaims.go:1777`, `src/pkg/github/prclaims.go:2037`) and so diff --git a/src/docs/adr/0026-sandbox-runtime-selection.md b/src/docs/adr/0026-sandbox-runtime-selection.md new file mode 100644 index 0000000000..3ef7e1ba2c --- /dev/null +++ b/src/docs/adr/0026-sandbox-runtime-selection.md @@ -0,0 +1,212 @@ +# ADR-0026: Sandbox runtime selection + +Status: Proposed + +## Context + +By default Hive agents share one container, separated by Unix user. The opt-in +sandbox ([sandbox-isolation.md](../sandbox-isolation.md), #2804) runs a kick in a +rootless Podman container or, with `sandbox.runtime: job`, a Kubernetes Job +(#6311). Both are ordinary containers: the boundary is the runtime's default +seccomp profile and namespaces, and network confinement in `restricted` mode is +whatever policy the operator supplies. + +The landscape refresh (#13806) named two external runtimes built for that gap. +#14092 asked for a spike comparing them against the sandbox contract. This +record covers that comparison as a paper evaluation, read from each project's +own documentation on 2026-10-25: + +- **NVIDIA OpenShell** ([repo](https://github.com/NVIDIA/OpenShell), v0.1.5, + 2026-10-20). +- **OpenSandbox** ([repo](https://github.com/opensandbox-group/OpenSandbox), + 1.1.1, 2026-10-09; 1.2.0-rc.2 on 2026-10-21). It is a Linux Foundation project + under AAIF, contributed by Alibaba. + +**Not done in this record:** the prototype run on a hive spoke (#14092 step 2). +Startup latency, image and boot size, and proxy and attribution behaviour are +**not measured** here. [Prototype protocol](#prototype-protocol) lists what to +measure. This record stays `Proposed` until those numbers exist. + +### The contract a runtime must satisfy + +These are the requirements from `sandbox-isolation.md`. The seam is the +`sandbox.Launcher` interface (`src/crux/sandbox/sandbox.go`): +`Run(ctx, LaunchSpec) (Result, error)`. Podman and `crux/kubejob` already +implement it. A new runtime is one more `sandbox.runtime` value behind that +interface. + +| # | Requirement | Where hive enforces it today | +|---|---|---| +| C1 | Task in: a prepared workspace and `.hive/kick-prompt.txt`, a command, an env allowlist | `LaunchSpec` | +| C2 | The workspace is credential-free and no GitHub token enters | `SanitizedEnv`, credential-name filter | +| C3 | Pushes go through the push broker, outside the sandbox | `crux/pushbroker` reads commits back from the workspace | +| C4 | The proxy still mediates GitHub | ADR-0002 iptables redirect in the hive pod; agent identified by socket UID | +| C5 | Transcript and exit code come back; a timeout kills the run | `Result`, `.hive/sandbox-transcript.log` | +| C6 | Hosted spokes (OKE) need no privileged pods | namespace-scoped RBAC from the hub provisioner | + +## Evaluation + +### NVIDIA OpenShell + +**Model.** A gateway is the control plane. A compute driver (Docker, Podman, +Kubernetes or MicroVM) creates two parts per sandbox: + +- a **workload**, which holds the agent and has no network of its own; +- a trusted **supervisor**, which carries every TCP connection and DNS lookup + the workload makes. + +Inside the workload, the agent runs as one non-root user with no Linux +capabilities. Landlock confines files. Seccomp user notification hands each +`connect` and DNS query to the supervisor, which checks it against policy and +opens the real connection. Policy covers files, processes, network endpoints +and HTTP methods. A prover flags proposed changes that would widen +credentialed reach. + +| # | Fit | Notes | +|---|---|---| +| C1 | Yes | `sandbox create` from an OCI image, `sandbox upload` for the workspace, `sandbox exec`, which returns the command's exit code. The Go SDK is `github.com/NVIDIA/OpenShell/sdk/go`. Workspace transport changes: hive's shared-PVC `subPath` model becomes an upload/download, or a mount set in the sandbox template. | +| C2 | Yes, stronger | OpenShell's own premise is that the workload never holds credentials. Hive attaches no GitHub provider, so there is nothing to inject. | +| C3 | Yes | Unchanged. The broker reads commits from the returned workspace. | +| C4 | Changes shape | The supervisor terminates TLS and opens the upstream connection. In the Kubernetes driver the supervisor is **its own pod**, so the hive pod's iptables redirect never sees the traffic, and ADR-0002's socket-UID identification has no meaning across pods. See [Proxy and attribution](#proxy-and-attribution). | +| C5 | Yes | Exec exit code, gateway logs and OCSF JSON export. If the supervisor connection drops, the workload freezes, so the run fails closed. | +| C6 | Yes, with cluster prerequisites | No privileged pod. The workload has no added capabilities, and the gateway meets the Restricted Pod Security Standard. The cluster needs three things: (a) the [Agent Sandbox](https://github.com/kubernetes-sigs/agent-sandbox) controller and CRDs, a cluster-scoped install; (b) a CNI that **enforces** ingress and egress `NetworkPolicy`, because without one the workload can bypass the supervisor; (c) node kernels with Landlock ABI 3 (Linux ≥ 6.2) and the seccomp user-notification operations it probes for. A node that lacks them fails closed at sandbox start. | + +**Licensing and cadence.** Apache-2.0, DCO, public RFC board. There have been +six stable `0.1.x` releases since 0.1.0 on 2026-09-25: stable builds are +promoted from nightly pre-releases that pass qualification. It is pre-1.0, so +expect API movement. Anonymous telemetry is **on by default**. Hive must set +`OPENSHELL_TELEMETRY_ENABLED=false`, or `server.telemetryEnabled=false` in +Helm. + +### OpenSandbox + +**Model.** A lifecycle server (Python, `opensandbox-server`) creates sandboxes +on Docker or Kubernetes, the latter through an operator. Each sandbox runs +`execd` for commands and files. Isolation is chosen per server profile: + +- `""`: plain runc; +- `gvisor`; +- `kata`: a RuntimeClass; +- `firecracker`: Kata-FC, or the "Fast Sandbox" Fastlet path. + +Network policy is an **egress sidecar** in the sandbox's network namespace. It +proxies DNS, sets nftables allow sets and can run an optional transparent +mitmproxy, which the Credential Vault uses. + +| # | Fit | Notes | +|---|---|---| +| C1 | Yes | Lifecycle API with `commands.run` and `files.write_files`. Volumes, including Kubernetes PVCs, can be set at creation. The Go SDK's import path is still `github.com/alibaba/OpenSandbox/sdks/sandbox/go`. Hive would run and upgrade one more service, the Python lifecycle server. | +| C2 | Yes | The Credential Vault keeps real secrets in the sidecar. Hive would use none. | +| C3 | Yes | Unchanged. | +| C4 | Changes shape | The sidecar owns transparent interception in the pod and rejects a second interceptor in the same namespace. It can chain to an upstream proxy over HTTP/1 CONNECT, but only with its own transparent MITM enabled. Hive's proxy would then see a re-originated TLS session from the sidecar, not the agent. | +| C5 | Yes | Sandbox state, exec results and OpenTelemetry metrics. | +| C6 | **Only at the weakest isolation level** | On Kubernetes the egress sidecar holds the pod's `NET_ADMIN`. That is the same capability class the hive pod already needs, and it is not privileged. But with runc that buys FQDN egress policy and nothing more. The boundary is the same container boundary `sandbox.runtime: job` already has. gVisor is **rejected** with egress policy, because its netstack cannot take the nat redirect. Kata needs a cluster RuntimeClass and KVM on the nodes. Fast Sandbox's Fastlet container is **privileged with `/dev/kvm`** and needs bare metal or nested virtualization, plus XFS reflink storage. Its headline numbers (64 ms P50 create) come from that path. | + +**Licensing and cadence.** Apache-2.0, Linux Foundation, release images signed +with Cosign. A new line ships every two weeks. Only the latest line is +supported (no LTS): a line reaches end-of-life when the next one ships, apart +from a 72-hour emergency-CVE window. The project is post-1.0, but that pace is +an upgrade obligation on every spoke. + +### Proxy and attribution + +Both runtimes move the agent's outbound TCP to a component outside the hive +pod's network namespace. ADR-0002's two mechanisms rest on the hive pod: the +iptables redirect, and agent identification by socket owner UID. Neither +reaches a sandbox on these runtimes. + +For a sandboxed kick this matters less than it first appears. C2 already keeps +every GitHub token out of the sandbox, so nothing inside can authenticate a +GitHub write. The proxy's job for a sandboxed kick reduces to "unauthenticated +reads only". OpenShell can express that directly: a network policy that allows +`GET` on `api.github.com` and git fetch on `github.com`, and denies the rest. The +prover can check it in CI. The push broker's own traffic leaves from the hive +pod and stays under ADR-0002. + +Attribution moves from UID to sandbox: one OpenShell sandbox per kick, named +for the agent and kick. OpenShell's logs carry the sandbox name and the calling +executable. Hive's existing `audit: sandbox agent kicked` line already keys on +the agent. + +## Decision + +**Proposed:** adopt **OpenShell** as a third `sandbox.runtime` value +(`openshell`), behind the existing `sandbox.Launcher` interface. Use the +Kubernetes driver on hosted spokes and the Podman driver on standalone hosts. +The existing `podman` and `job` runtimes stay as they are, and the default +isolation mode does not change (out of scope for #14092). + +Reasons: + +1. **It meets C6 without privileged pods and still raises the boundary.** The + workload has no capabilities, and Landlock, seccomp mediation and a + network-less workload are confirmed before the agent starts. OpenSandbox + meets C6 only with runc, where it adds egress policy but no stronger + boundary than the current Job runtime. Its stronger modes need KVM, and + Fast Sandbox needs privileged pods. +2. **No new long-lived service per spoke beyond the gateway**, and the gateway + speaks gRPC with a Go SDK. OpenSandbox adds a Python lifecycle server plus a + per-pod sidecar image. +3. **Default-deny egress is the runtime's contract**, not a policy the operator + has to supply. That closes the "`restricted` depends on an operator-provided + network policy" gap listed in `sandbox-isolation.md`. + +**Do not adopt OpenSandbox now.** Revisit it for self-hosted spokes on +KVM-capable nodes, where Kata gives a VM boundary without privileged pods, if +the OpenShell prototype fails its gates. + +### Preconditions the implementation epic must verify first + +These are blocking. If any one fails, the recommendation falls back to "no +change". + +- The hosted OKE node pool's kernel passes OpenShell's Landlock ABI 3 and + seccomp qualification. Oracle UEK7 is 5.15 and fails the Landlock ABI 3 + floor; a 6.2+ kernel such as UEK8 is required. Check `uname -r` on the pool, + then run a real `sandbox create`. +- The hosted cluster's CNI **enforces** egress `NetworkPolicy`. Flannel does + not; Calico or Cilium do. Kubernetes accepts unenforced policies silently, so + test with a denied connection, not by reading the manifest. +- Someone with cluster-admin can install the Agent Sandbox controller and CRDs + once per cluster. Per-spoke isolation then maps to OpenShell workspaces in + `operator` namespace mode, one pre-provisioned namespace per spoke, so a spoke + still needs only namespace-scoped RBAC. + +### Prototype protocol + +Run one real kick for the same agent and task on hive's own spoke, under +`job`, `openshell` (Kubernetes driver), and OpenSandbox (runc + egress sidecar) +for comparison. Record these numbers and attach them to this ADR before it +moves to `Accepted`: + +| Measure | How | +|---|---| +| Cold start to agent's first command | Kick timestamp until the first transcript line, median of 5 | +| Warm start | Same, with images already on the node | +| Image and boot size | Pulled bytes for sandbox, supervisor and sidecar images | +| GitHub write refused | A `POST` to `api.github.com` from inside the sandbox fails with no token present | +| GitHub read allowed | `gh api repos/` read and `git fetch` succeed | +| Direct egress refused | A raw TCP connect to an arbitrary IP fails | +| Attribution | The refused write appears in runtime logs with the sandbox and kick name | +| Broker path intact | The kick's commits arrive as a PR through `pushbroker`, unchanged | + +## Consequences + +- Hive gains a sandbox boundary that is checked before the agent runs, without + privileged pods. The cost is three cluster prerequisites: the controller, an + enforcing CNI, and a 6.2+ kernel. Those turn into hub provisioning and + preflight work. +- For sandboxed kicks under `openshell`, ADR-0002's iptables and UID mechanism + is replaced by OpenShell network policy. ADR-0002 still governs the hive pod + and tmux agents. A change to the OpenShell policy file for hive is a policy + change and goes through the same maintainer-approval gate as proxy rules. +- OpenShell is pre-1.0 and ships often. Pin the gateway, supervisor and sandbox + images by digest, upgrade through Renovate, and run its prover in CI against + hive's policy. +- Telemetry must be disabled explicitly. A preflight check should refuse a + gateway that reports it enabled. +- If the prototype fails a gate, this record is rewritten to decide "no + change" with the measured reason, and the Podman and Job runtimes remain the + only ones. + +Refs #2804, #6311, #13806, #14092. diff --git a/src/docs/adr/README.md b/src/docs/adr/README.md index 2b869a85f3..7bcf095f41 100644 --- a/src/docs/adr/README.md +++ b/src/docs/adr/README.md @@ -64,6 +64,7 @@ What becomes easier, harder, safer, or riskier because of this decision? - [ADR-0023: Hive may merge PRs whose author could merge them](0023-trusted-author-automerge.md) - [ADR-0024: Provider credential lifecycle](0024-provider-credential-lifecycle.md) - [ADR-0025: One stage engine derives and writes every `stage/*` label](0025-central-stage-engine.md) +- [ADR-0026: Sandbox runtime selection](0026-sandbox-runtime-selection.md) (Proposed) ### v6-only ADRs (`0100`+) diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 3a5b1dc06b..896b5bea2a 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -106,7 +106,7 @@ older overlays, but new configs should use `bob.session_label`. ```yaml backend: copilot # the method: claude | copilot | goose | codex | pi | - # bob | aider | omp, or an inference backend: + # bob | aider | omp | openhands | omnigent, or an inference backend: # vllm | llm-d | litellm | watsonx | named gateway model: claude-sonnet-4-6 # model id for that method reasoning_effort: high # reasoning effort, for methods that have one: @@ -143,7 +143,7 @@ omp stores credentials in the SQLite store `~/.omp/agent/agent.db` (table `auth_ 2. Pick the provider. For a subscription, omp runs a browser or device OAuth flow and prints a URL in the pane: open it, approve, and paste the returned code into the dashboard login-code box (or into the pane). For an API key, choose the provider's API-key option and paste the key; omp stores it in `auth_credentials`. 3. Repeat for every provider the agent needs (for example `openai-codex` for the primary model and `anthropic` for an advisor). 4. Repeat for every omp agent: each hub agent has its own HOME under `/data/home/agents/`, so sign-ins are not shared. -5. Provider API keys that omp reads from the environment (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, ...) are not forwarded to agents: the manager's agent environment (`pkg/agent/manager_env.go`) exports only Hive's own variables, and `ANTHROPIC_API_KEY` there is a placeholder for inference-routed backends. Use the stored API-key sign-in above. +5. Provider API keys that omp reads from the environment (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, ...) are not forwarded to agents: the manager's agent environment (`crux/agent/manager_env.go`) exports only Hive's own variables, and `ANTHROPIC_API_KEY` there is a placeholder for inference-routed backends. Use the stored API-key sign-in above. 6. Sign-ins survive container restarts only because `/data` is the persistent volume (`Volume=hive-data.volume:/data` in `src/deploy/quadlet/hive.container`); `/home/dev` is not persistent. After signing in, the model picker lists the agent's real models on its next poll. @@ -151,6 +151,10 @@ After signing in, the model picker lists the agent's real models on its next pol Sign-in is per agent: each omp hub agent signs in to its providers in its own terminal, since its `~/.omp` lives in that agent's HOME. If omp opens its first-run setup wizard ("Setup step 1 of 5", "Sign in to your providers"), Hive detects the wizard and its browser/auth-code hand-off as a login-needed state rather than a ready prompt, so no task is typed into it; complete the sign-in (or skip the wizard and use `/login`) from the agent terminal. +#### Omnigent (`omnigent`) agents + +`backend: omnigent` launches the Omnigent CLI as an experimental T3 backend. Omnigent provides an open-source meta-harness layer orchestrating underlying agent engines (Claude Code, Codex, Cursor, Pi, and custom harnesses). On contributor local launches, Omnigent has no OS sandbox wired by Hive, so local mode refuses by default without `HIVE_OMNIGENT_DANGEROUSLY_RUN_UNCONFINED=1`. Container mode is the default and recommended boundary. Every launch is `omnigent run --server local [--model ]`: `run` opens the agent REPL (bare `omnigent` only starts a background server) and `--server local` pins a local Omnigent server, so remote-server mode is unsupported. Omnigent's internal policy engine and sandbox evaluate additively, while Hive's proxy remains authoritative on all repository writes. See [`backend-omnigent-cli-contract.md`](backend-omnigent-cli-contract.md). + > The dashboard also offers **gemini** as a live method (with live model discovery); as a persisted `backend:` value in `hive.yaml`, stick to the validated list above. ### Reviewer independence @@ -586,7 +590,9 @@ Turning the check off removes the script at each agent's next launch and drops a ## Methods: subscription CLIs vs self-hosted inference -`backend:` picks one of two families. They differ in how you authenticate and where model lists come from: +`backend:` picks one of the CLI or inference families below. Hosted execution +is a separate `runs.external.` binding and is never a `backend:` value: +it has no local binary, CLI login, or live model discovery. | Method | Family | Auth | Model discovery | |---|---|---|---| @@ -598,6 +604,7 @@ Turning the check off removes the script at each agent's next launch and drops a | `llm-d` | inference | endpoint + optional key — no login | **live** — `/v1/models` | | `litellm` | inference | endpoint + API key — no login | **live** — `/v1/models`, entitlement-filtered per key | | `openrouter` (gateway name) | inference | Model Gateway key or scan-to-fund flow — no CLI login | **live** — OpenRouter `/v1/models`, plus curated fallback | +| `claude-managed-agents` | hosted | API key reference (`api_key_env` or `/data/secrets` `api_key_file`) — no CLI login | pinned agent/model version; no discovery | Two rules of thumb: @@ -608,6 +615,50 @@ Two rules of thumb: Every Model Gateway (and the bob backend) also accepts an optional `key_name` — a human-chosen LABEL for the configured key, e.g. `key_name: openrouter-prod-key`. It is safe-to-show metadata, not a secret: the dashboard's gateway row displays it as "Using key: ``", or "(unnamed)" when no label is set, so operators can tell keys apart without ever seeing the value. For gateway keys, the row and settings/status APIs also expose `keySHA256`, the lowercase SHA-256 of the exact effective key string Hive will inject through the inference proxy; this lets operators compare Hive with LiteLLM/OpenRouter key-hash displays without revealing the secret. Saving a replacement key refreshes live inference proxy routes for running agents, so the next kick uses the new key without an agent restart. See [`inference-backends.md`](../../docs/inference-backends.md) for a full YAML example. +### Claude Managed Agents hosted execution + +Claude Managed Agents is an external hosted engine, not a `backend:` value. +The prototype is linked only with `extwork_claude_managed_agents` and is +selected under `runs.external.claude_managed_agents`. It requires a Claude +`self_hosted` environment in the contributor-private spoke with +`forced-egress` or `proxy-only` policy. The worker receives its environment key +and per-session work secret; it never receives `ANTHROPIC_API_KEY`. + +```yaml +runs: + external: + claude_managed_agents: + enabled: false + mode: shadow # report-only is the dispatch posture + endpoint: https://api.anthropic.com + workflow_version: claude-managed/2026-04-01 + agent_id: agt_private + agent_version: 1 + toolset_version: hive-tools/1 + environment_id: env_private + environment_type: self_hosted + network_policy: forced-egress + api_key_env: ANTHROPIC_API_KEY # reference only; never the value + model: claude-sonnet-4-6 + state_dir: /data/extwork/claude-managed-agents + daily_budget_cents: 2500 # hard USD 25/day maximum + github_write_path: none +``` + +The adapter performs an authenticated `GET /v1/environments/{id}` before +creating a session, pins the agent/toolset versions, sends the bounded prompt +and repository workspace, and records `span.model_request_end` token events in +`data.metrics_dir`. Event IDs are deduplicated across restarts. Session +`list_cost`, active seconds, and web-search counts are kept in the durable +managed-agents ledger; the session budget is the remaining daily cents. Set +`github_write_path` to `none` or `proxy` for a report-only spoke. Any other +value (a direct GitHub resource, vaulted secret, GitHub MCP path, or +`direct_github_write`) is rejected when the adapter is built. `endpoint` must +be `https` unless its host is loopback, because the vendor API key is sent on +every request. The hosted worker has no GitHub path in this prototype; the +`POST /pulls` and `PUT /pulls/{n}/merge` proxy denial tests from #14124 are not +yet covered. + Kubernetes manifests for deploying inference backends (vllm Deployment, EPP RBAC, kustomization) are in [`deploy/inference/`](../deploy/inference/). Agents can inspect peer panes with `hive-panes [lines]` when the deployment includes `src/deploy/hive-panes.sh`. It reads pluk JSONL logs from `/var/run/pluk/logs`, skips the calling agent named by `HIVE_PROXY_AGENT`, strips terminal escapes, and prints the last N raw-output lines for every other agent. Use it for situational awareness; it is read-only and does not attach to another agent's tmux session. See [Agent peer-awareness logging](agent-logging.md) for the pluk log format, when it's available, and its retention behavior. @@ -758,7 +809,7 @@ You don't have to design a roster. Hive ships six **ACMM packs** (`level-1.yaml` | L3 | Quality-Gated (Measured) | quality opens issues and test PRs gated by literal `hold`; the rest stay advisory | | L4 | Security-Aware (Adaptive) | scanner/guide file issues; quality, ci-maintainer, and sec-check can open PRs gated by literal `hold` | | L5 | Semi-Autonomous (Semi-Automated) | issues **and** PRs; PRs are gated by literal `hold`; humans batch-approve | -| L6 | Fully Autonomous | auto-merge on green CI; no level hold, except cross-repo outreach PRs (target repo not managed by this hive) remain held | +| L6 | Autonomous | auto-merge on green CI; no level hold, except cross-repo outreach PRs (target repo not managed by this hive) remain held | Applying a level **reconciles the whole roster**, not just the diff: missing agents are created (as overlay files in `/data/agent-configs/`), existing agents are merged — pack values fill blanks, but your explicit `backend:`, `model:`, and `enabled: false` always win — and the level's `kick_template`, `mode` and `on_demand` are updated so the agent's *policy* matches the level (an `on_demand` you toggled yourself in the agent's settings dialog is operator-owned and left alone; leaving on-demand starts the agent, entering it stops it). One caveat for `backend:` edited by hand in `hive.yaml` or `hive.yaml.runtime` after a pack has been applied: the edit is honoured only when that agent entry carries **no** `backend_owner:` line, so delete the owner line (`pack` or `operator`) alongside your `backend:` change, and the next apply stamps the kept backend `backend_owner: operator` (#13221). A failed agent doesn't abort the rest; the level is only recorded as cleanly applied when every agent reconciled. @@ -841,7 +892,7 @@ agents: ref: main # optional; branch/tag/SHA — omit for the default branch ``` -`definition_source` is `DefinitionSourceConfig` (`src/pkg/crux/config/agent_runtime.go:62`), a field on `AgentConfig` (`src/pkg/crux/config/agent_config.go:356`). `owner`, `repo`, and `path` are required for the source to be considered set (`IsSet()`, `src/pkg/crux/config/agent_runtime.go:88`); `ref` is optional and falls back to the repo's default branch. `url` is a fifth, informational-only field the dashboard import UI uses to round-trip the pasted `github.com` blob URL — it plays no part in fetching. +`definition_source` is `DefinitionSourceConfig` (`src/crux/config/agent_runtime.go:62`), a field on `AgentConfig` (`src/crux/config/agent_config.go:356`). `owner`, `repo`, and `path` are required for the source to be considered set (`IsSet()`, `src/crux/config/agent_runtime.go:88`); `ref` is optional and falls back to the repo's default branch. `url` is a fifth, informational-only field the dashboard import UI uses to round-trip the pasted `github.com` blob URL — it plays no part in fetching. ### What it does diff --git a/src/docs/agent-logging.md b/src/docs/agent-logging.md index 665886df5e..65cefbe389 100644 --- a/src/docs/agent-logging.md +++ b/src/docs/agent-logging.md @@ -18,7 +18,7 @@ Pluk is installed in the container image as an npm global package Hive-authored component; Hive only shells out to it. When the agent manager creates a tmux session for an agent -(`pkg/agent/manager.go`), it looks up `pluk` on `PATH` and, if found, attaches +(`crux/agent/manager.go`), it looks up `pluk` on `PATH` and, if found, attaches it to that session's pane with: ``` @@ -58,7 +58,7 @@ fails, because it writes through a named FIFO and confirms delivery via a - The run directory (`/var/run/pluk` — `logs/` and `commands/` subdirectories) is created ahead of time in the image (`Dockerfile`) and re-created defensively before each attach by `ensurePlukRunDirs` - (`pkg/agent/pluk_dirs.go`). Both subdirectories are `0770` plus setgid, + (`crux/agent/pluk_dirs.go`). Both subdirectories are `0770` plus setgid, owned by the `dev` user and the shared `node` group, so any agent UID in that group can read another agent's log without world-readable permissions. @@ -82,7 +82,7 @@ object, newline-delimited (JSONL), appended as the agent produces output. The file is written by a shell redirect, not by pluk. `pluk watch` classifies its stdin and prints events to stdout; it opens no file of its own. `tmux pipe-pane` feeds the pane into that command's stdin but does nothing with its -stdout, so the publisher invocation in `pkg/agent/manager.go` has to end in +stdout, so the publisher invocation in `crux/agent/manager.go` has to end in `>> /var/run/pluk/logs/.jsonl` for anything to be written at all. It did not between [#1759](https://github.com/hivecommons/hive/pull/1759) — which replaced the Go `pluk-publish` binary, a publisher that wrote the log itself, diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index e837dc76d9..59c3f0feba 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -10,20 +10,20 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)); also `hub_build` (the hub's short SHA) and, once a stdio `hive-admin-mcp` client has called `/api/version`, `/api/status` or `/api/status/summary` with its `hive-admin-mcp/+` User-Agent, `last_seen_stdio_build` and `last_seen_stdio_at` (RFC3339) so the dashboard can flag build skew ([#12318](https://github.com/hivecommons/hive/issues/12318)) | `pkg/dashboard/api.go:38` | -| `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:39` | -| `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:66` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1577` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1578` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1579` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1585` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); every your-turn issue and PR row carries the critical path (#13646): `waits_on[]` (`{kind,ref}`, kind one of `pr`, `issue`, `external`, `base-check` (ref = the check red on the base branch), `approval`, `merge`; ref `#N` in the same repository or `owner/repo#N`), `unblocks` (transitive count of your-turn items waiting on it, cycle-guarded), `may_conflict_with[]` (PR numbers of the same repository sharing a closing issue or a known changed file; never an edge), `action` (`merge`, `release-hold`, `rebase`, `rerun` or `approve`), `action_ref`, `root_cause` and `blocker` (the root cause it collapses under); `repos[].yourTurn` has `issues`, `prs` (your turn, `CLOSE` excluded), `root_issues`, `root_prs`, `root_cause_count`, `order[]` (`{kind,repo,number}` by unblocks then age), `groups[]` (`{blocker,blocker_title,dependents[]}`) and `main_red` (`{check,since,fix,fix_your_turn}`); the top-level `yourTurn` sums the counts | `pkg/dashboard/server.go:1590` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers, with the hive-wide `yourTurn` root-cause counts the phone widget reads (#13646); honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1591` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1594` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1595` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1596` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1597` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1598` | +| `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)); also `hub_build` (the hub's short SHA) and, once a stdio `hive-admin-mcp` client has called `/api/version`, `/api/status` or `/api/status/summary` with its `hive-admin-mcp/+` User-Agent, `last_seen_stdio_build` and `last_seen_stdio_at` (RFC3339) so the dashboard can flag build skew ([#12318](https://github.com/hivecommons/hive/issues/12318)) | `pkg/dashboard/api.go:43` | +| `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:44` | +| `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:71` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1622` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1623` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1624` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1630` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); every your-turn issue and PR row carries the critical path (#13646): `waits_on[]` (`{kind,ref}`, kind one of `pr`, `issue`, `external`, `base-check` (ref = the check red on the base branch), `approval`, `merge`; ref `#N` in the same repository or `owner/repo#N`), `unblocks` (transitive count of your-turn items waiting on it, cycle-guarded), `may_conflict_with[]` (PR numbers of the same repository sharing a closing issue or a known changed file; never an edge), `action` (`merge`, `release-hold`, `rebase`, `rerun` or `approve`), `action_ref`, `root_cause` and `blocker` (the root cause it collapses under); `repos[].yourTurn` has `issues`, `prs` (your turn, `CLOSE` excluded), `root_issues`, `root_prs`, `root_cause_count`, `order[]` (`{kind,repo,number}` by unblocks then age), `groups[]` (`{blocker,blocker_title,dependents[]}`) and `main_red` (`{check,since,fix,fix_your_turn}`); the top-level `yourTurn` sums the counts | `pkg/dashboard/server.go:1635` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers, with the hive-wide `yourTurn` root-cause counts the phone widget reads (#13646); honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1636` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1639` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1640` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1641` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1642` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1643` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | @@ -33,231 +33,234 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `PUT` | `/api/swarm/themes` | Owner only | Set a per-repo swarm theme (`event_name`, `call_to_arms`, `leaderboard_title`). | `pkg/dashboard/swarm.go:316` | | `PATCH` | `/api/swarm/objectives` | Owner only | Update the active swarm SDLC objective checklist completion state. | `pkg/dashboard/swarm.go:317` | | `GET` | `/api/leaderboard/swarm` | Public | Stable public JSON record of swarm history, leaderboard, top players, and achievements. | `pkg/dashboard/swarm.go:318` | -| `GET` | `/api/runs` | Dashboard auth/session | Active staged runs projected from task leases, plans, and lifecycle timeline. Returns an array of `Run` objects: canonical `key` (`owner/repo#number`), current-stage `lease_key`, `title`, `repo` (`owner/repo`), `stage`, `gen`, `stage_started_at`, `waiting_on` (`agent`, `remote`, `human`, `ci`, `none`), `waiting_since`, `assignee`, `last_receipt`, `plan_epic_id`, and `stages[]` (`name`, `status`, `gen`, `receipt`). | `pkg/dashboard/api.go:88` | -| `GET` | `/api/runs/audit` | Owner only | Read-only cross-repo audit query over existing retained artifacts. Query `repo`, canonical `run` (`owner/repo#number`), RFC3339 `since`/`until`, `kind`, `limit` (default 100, max 500), and `page_token`. Returns `{items,next_page_token,retention}`; each item has `source` (`audit_log`, `timeline`, `lease_receipt`, `plan_epic`), `kind`, `run`, `repo`, `at`, `actor`, `artifact_id`, `attrs`, or an explicit `kind=expired,status=expired,state=Unknown` marker when requested history may have aged out. | `pkg/dashboard/api.go:89` | -| `POST` | `/api/runs/audit` | Owner only | Activate the convergence audit campaign over a scope directory. Body `{"campaign_key":"audit-campaign","scope_dir":"/data/convergence/audit/scope","store":"audit","generation":1,"run_key":"","run_url":""}`; `scope_dir` defaults to `/data/convergence/audit/scope`, `store` defaults to `audit`, `auditor`, `scanner`, `supervisor`, then the first configured bead store. Runs inspection even when `publication.enabled` is false, but then skips the publisher and returns `publication_skipped:true` with reason `publication.disabled`. | `pkg/dashboard/api.go:90` | -| `GET` | `/api/runs/{key}/checkpoint` | Dashboard auth/session | Compact mobile/chat checkpoint review payload for a held run. Returns `run_key`, `stage`, lease `gen`, `repo`, `title`, capped plain-text `summary`, approve/reject `decisions`, dashboard deep link, `lease_gen` staleness fence, and verified-owner approver rule. | `pkg/dashboard/api.go:94` | -| `POST` | `/api/runs/{key}/checkpoint` | Owner only | Resolve a compact checkpoint decision. Body `{"action":"approve"|"reject","gen":}`; refuses stale generations with 409 and non-owner callers with 403 before mutating the plan. | `pkg/dashboard/api.go:95` | -| `GET` | `/api/runs/{key}` | Dashboard auth/session | One run by URL-escaped canonical run key (`owner/repo%23number`); early lease-shaped keys (`repo!run-key:stage`) and queued Wavefront run-stage keys (`owner/repo!graph:node`) are also accepted. Includes timeline-derived stage history in the same `Run` shape as `/api/runs` when a lease/timeline exists. History entries that came from an owner reset carry the owner's `reason`. When the key belongs to a wired convergence campaign, the detail response also includes an optional `burndown` block with `source`, `satisfied`, `remaining`, `unknown`, and `scope`; the list endpoint omits this block. | `pkg/dashboard/api.go:98` | -| `POST` | `/api/runs/{key}/reset` | Owner only | Move a run's lease back to an earlier stage (for example `implement` to `plan` after a rejected plan). Body `{"to": "", "reason": ""}`, both required. Mints a new generation so the relay holding the old one can no longer resume, persists before answering, and records the reason on the agent audit sink, the lifecycle timeline, and the `stage_completed` hook payload (`attrs.reset = "true"`). Returns `{ok, key, stage_from, stage, gen, reason}`. 400 for the same or a later stage or a missing reason, 404 when no live staged lease holds the key, 409 when the lease expired, 500 when the registry could not be written. | `pkg/dashboard/api.go:100` | -| `GET` | `/api/campaigns` | Dashboard auth/session | List resumable campaigns from archived Inception sessions and retained Spektacular (Spek) runs. Query `search`/`q`, `repo`, `stage`, `status`, and `owner`; each campaign is keyed by stable spec/session id and includes title, source, repos, current stage/step, artifact links, linked issues/PRs, contributors, last activity, status, engine, type, run link when applicable, and recheck status (`enabled`, `interval`, `last_at`, `next_at`, `in_flight`, `last_delta_count`, `external_count`, `sources_failed`). | `pkg/dashboard/api.go:101` | -| `GET` | `/api/campaigns/{id}` | Dashboard auth/session | Return one campaign by URL-escaped stable campaign id or run key, using the same campaign shape as `/api/campaigns`; recheck revisions may include a `drift` block with `codebase_changed`, `prior_revision`, `delta_count`, `recheck_reason`, `external[]` (`source`, `kind`, `title`, `url`, `published_at`, `summary`), `external_count`, and `sources_failed`. | `pkg/dashboard/api.go:102` | -| `POST` | `/api/campaigns/{id}/resume` | Owner only | Resume or pick up a campaign. Archived Inception campaigns take a durable campaign lease, restore their saved state and wiki working files; Spek campaigns return the retained run detail plus a CLI status command that reloads state from Spek artifacts. Body may include `{ "surface": "chat"|"contributor"|"cli"|"dashboard" }`; 409 means another user holds the lease. | `pkg/dashboard/api.go:103` | -| `POST` | `/api/campaigns/{id}/release` | Owner only | Release the caller's Inception campaign lease and write a `campaign_release` audit entry so another user can continue it. Run-backed Spektacular campaigns return `409 Conflict`; this endpoint never revokes their run-lifecycle stage leases. | `pkg/dashboard/api.go:104` | -| `POST` | `/api/campaigns/{id}/revise` | Owner only | Create a linked revision campaign (`revision_of`, `revision`) for revisiting shipped or parked Inception/Spek work without overwriting the previous campaign. The new revision is leased to the caller and audit-logged as `campaign_revise`. | `pkg/dashboard/api.go:105` | -| `POST` | `/api/campaigns/{id}/recheck` | Owner only | Create an immediate linked Spek recheck revision using the cadence path. Returns `409 Conflict` when a recheck revision is already in flight, `404 Not Found` when recheck is disabled unless `?force=true` is supplied, and records `campaign_recheck`. | `pkg/dashboard/api.go:107` | -| `GET` | `/api/campaigns/{id}/jam` | Dashboard auth/session | Return the per-campaign Jam workspace: section-anchored threads, comments, suggestions, polls, poll decisions, current spec revision and attributed revision history. | `pkg/dashboard/api.go:108` | -| `POST` | `/api/campaigns/{id}/jam` | Read-write role | Record an attributed spec revision for a campaign. Body includes `spec_content`, optional `reason`, and optional agent/model attribution fields. | `pkg/dashboard/api.go:109` | -| `GET` | `/api/campaigns/{id}/jam/threads` | Dashboard auth/session | List async Jam threads anchored to spec sections, including comments and author attribution. | `pkg/dashboard/api.go:110` | -| `POST` | `/api/campaigns/{id}/jam/threads` | Read-write role | Create a section-anchored Jam thread or add a comment to an existing thread. Body includes `section` and `body`, or `thread_id` and `body`. | `pkg/dashboard/api.go:111` | -| `POST` | `/api/campaigns/{id}/jam/agents` | Maintainer only | Invite Spektacular or a configured hive agent into a Jam thread. The agent reply is posted as an attributed thread comment and an open suggestion for human accept/reject; no spec revision is auto-applied. | `pkg/dashboard/api.go:112` | -| `GET` | `/api/campaigns/{id}/jam/suggestions` | Dashboard auth/session | List Jam suggestions with open/accepted/rejected status, proposed text, attribution and applied revision id when accepted. | `pkg/dashboard/api.go:113` | -| `POST` | `/api/campaigns/{id}/jam/suggestions` | Read-write role; maintainer for accept | Create a suggestion (`section`, `proposed_text`) or resolve it with `action: "accept"`/`"reject"` and `suggestion_id`. Accepting records a new attributed spec revision. | `pkg/dashboard/api.go:114` | -| `GET` | `/api/campaigns/{id}/jam/polls` | Dashboard auth/session | List Jam polls, advisory vote tallies and maintainer decisions stored against spec revisions. | `pkg/dashboard/api.go:115` | -| `POST` | `/api/campaigns/{id}/jam/polls` | Read-write role; maintainer for decide | Create polls (`section`, `question`, `options`), cast/update advisory votes (`action: "vote"`), or record a maintainer decision with rationale (`action: "decide"`). | `pkg/dashboard/api.go:116` | -| `GET` | `/api/campaigns/{id}/jam/project-sync` | Dashboard auth/session | Return the campaign's GitHub Projects sync configuration, last published items, last status, and retry guidance for failures. | `pkg/dashboard/api.go:117` | -| `POST` | `/api/campaigns/{id}/jam/project-sync` | Maintainer only | Enable/disable opt-in GitHub Projects sync, publish campaign specs/decisions/derived issues to the linked project, or record inbound project status updates without losing local Jam decisions. | `pkg/dashboard/api.go:118` | -| `GET` | `/api/campaigns/{id}/jam/ws` | Dashboard auth/session | Upgrade to a Jam WebSocket for live participant presence, section focus, and conflict-aware spec co-edit messages. Live edits require read-write role and stale `base_revision_id` values are rejected with a conflict message. | `pkg/dashboard/api.go:119` | -| `GET` | `/api/runs/{key}/trace` | Dashboard auth/session | Resolve `Hive-Run`, `Hive-Plan`, and `Hive-Spec` commit trailers for `?sha=...`, returning the linked plan section, spek clause, approval audit record, rationale audit entries, and typed no-linkage results when trailers are missing. | `pkg/dashboard/api.go:91` | -| `DELETE` | `/api/campaigns/{id}` | Owner only | Delete a campaign record: removes the archived Inception state and wiki snapshot for the stable campaign id and audit-logs `campaign_delete`. Knowledge-base facts, Jam threads, scaffolded repo files and other revisions in the same chain are kept; deleting the live L1 session clears it. Run-backed Spektacular campaigns return `409 Conflict`; unknown ids `404`. | `pkg/dashboard/api.go:106` | +| `GET` | `/api/runs` | Dashboard auth/session | Active staged runs projected from task leases, plans, and lifecycle timeline. Returns an array of `Run` objects: canonical `key` (`owner/repo#number`), current-stage `lease_key`, `title`, `repo` (`owner/repo`), `stage`, `gen`, `stage_started_at`, `waiting_on` (`agent`, `remote`, `human`, `ci`, `none`), `waiting_since`, `assignee`, `last_receipt`, `plan_epic_id`, and `stages[]` (`name`, `status`, `gen`, `receipt`). | `pkg/dashboard/api.go:93` | +| `GET` | `/api/runs/audit` | Owner only | Read-only cross-repo audit query over existing retained artifacts. Query `repo`, canonical `run` (`owner/repo#number`), RFC3339 `since`/`until`, `kind`, `limit` (default 100, max 500), and `page_token`. Returns `{items,next_page_token,retention}`; each item has `source` (`audit_log`, `timeline`, `lease_receipt`, `plan_epic`), `kind`, `run`, `repo`, `at`, `actor`, `artifact_id`, `attrs`, or an explicit `kind=expired,status=expired,state=Unknown` marker when requested history may have aged out. | `pkg/dashboard/api.go:94` | +| `POST` | `/api/runs/audit` | Owner only | Activate the convergence audit campaign over a scope directory. Body `{"campaign_key":"audit-campaign","scope_dir":"/data/convergence/audit/scope","store":"audit","generation":1,"run_key":"","run_url":""}`; `scope_dir` defaults to `/data/convergence/audit/scope`, `store` defaults to `audit`, `auditor`, `scanner`, `supervisor`, then the first configured bead store. Runs inspection even when `publication.enabled` is false, but then skips the publisher and returns `publication_skipped:true` with reason `publication.disabled`. | `pkg/dashboard/api.go:95` | +| `GET` | `/api/runs/{key}/checkpoint` | Dashboard auth/session | Compact mobile/chat checkpoint review payload for a held run. Returns `run_key`, `stage`, lease `gen`, `repo`, `title`, capped plain-text `summary`, approve/reject `decisions`, dashboard deep link, `lease_gen` staleness fence, and verified-owner approver rule. | `pkg/dashboard/api.go:99` | +| `POST` | `/api/runs/{key}/checkpoint` | Owner only | Resolve a compact checkpoint decision. Body `{"action":"approve"|"reject","gen":}`; refuses stale generations with 409 and non-owner callers with 403 before mutating the plan. | `pkg/dashboard/api.go:100` | +| `GET` | `/api/runs/{key}` | Dashboard auth/session | One run by URL-escaped canonical run key (`owner/repo%23number`); early lease-shaped keys (`repo!run-key:stage`) and queued Wavefront run-stage keys (`owner/repo!graph:node`) are also accepted. Includes timeline-derived stage history in the same `Run` shape as `/api/runs` when a lease/timeline exists. History entries that came from an owner reset carry the owner's `reason`. When the key belongs to a wired convergence campaign, the detail response also includes an optional `burndown` block with `source`, `satisfied`, `remaining`, `unknown`, and `scope`; the list endpoint omits this block. | `pkg/dashboard/api.go:103` | +| `POST` | `/api/runs/{key}/reset` | Owner only | Move a run's lease back to an earlier stage (for example `implement` to `plan` after a rejected plan). Body `{"to": "", "reason": ""}`, both required. Mints a new generation so the relay holding the old one can no longer resume, persists before answering, and records the reason on the agent audit sink, the lifecycle timeline, and the `stage_completed` hook payload (`attrs.reset = "true"`). Returns `{ok, key, stage_from, stage, gen, reason}`. 400 for the same or a later stage or a missing reason, 404 when no live staged lease holds the key, 409 when the lease expired, 500 when the registry could not be written. | `pkg/dashboard/api.go:105` | +| `GET` | `/api/campaigns` | Dashboard auth/session | List resumable campaigns from archived Inception sessions and retained Spektacular (Spek) runs. Query `search`/`q`, `repo`, `stage`, `status`, and `owner`; each campaign is keyed by stable spec/session id and includes title, source, repos, current stage/step, artifact links, linked issues/PRs, contributors, last activity, status, engine, type, run link when applicable, `superseded_by` when a newer Revise revision exists, and recheck status (`enabled`, `interval`, `last_at`, `next_at`, `in_flight`, `last_delta_count`, `external_count`, `sources_failed`). | `pkg/dashboard/api.go:106` | +| `GET` | `/api/campaigns/{id}` | Dashboard auth/session | Return one campaign by URL-escaped stable campaign id or run key, using the same campaign shape as `/api/campaigns`; recheck revisions may include a `drift` block with `codebase_changed`, `prior_revision`, `delta_count`, `recheck_reason`, `external[]` (`source`, `kind`, `title`, `url`, `published_at`, `summary`), `external_count`, and `sources_failed`. | `pkg/dashboard/api.go:107` | +| `POST` | `/api/campaigns/{id}/resume` | Owner only | Resume or pick up a campaign. Archived Inception campaigns take a durable campaign lease, restore their saved state and wiki working files; Spek campaigns return the retained run detail plus a CLI status command that reloads state from Spek artifacts. Body may include `{ "surface": "chat"|"contributor"|"cli"|"dashboard" }`; 409 means another user holds the lease. | `pkg/dashboard/api.go:108` | +| `POST` | `/api/campaigns/{id}/release` | Owner only | Release the caller's Inception campaign lease and write a `campaign_release` audit entry so another user can continue it. Run-backed Spektacular campaigns return `409 Conflict`; this endpoint never revokes their run-lifecycle stage leases. | `pkg/dashboard/api.go:109` | +| `POST` | `/api/campaigns/{id}/revise` | Owner only | Create a linked revision campaign (`revision_of`, `revision`) for revisiting shipped or parked Inception/Spek work without overwriting the previous campaign. The new revision is leased to the caller and audit-logged as `campaign_revise`. | `pkg/dashboard/api.go:110` | +| `DELETE` | `/api/campaigns/{id}` | Owner only | Delete a campaign record: removes the archived Inception state and wiki snapshot for the stable campaign id and audit-logs `campaign_delete`. Knowledge-base facts, Jam threads, scaffolded repo files and other revisions in the same chain are kept; deleting the live L1 session clears it. Run-backed Spektacular campaigns return `409 Conflict`; unknown ids `404`. | `pkg/dashboard/api.go:111` | +| `POST` | `/api/campaigns/{id}/recheck` | Owner only | Create an immediate linked Spek recheck revision using the cadence path. Returns `409 Conflict` when a recheck revision is already in flight, `404 Not Found` when recheck is disabled unless `?force=true` is supplied, and records `campaign_recheck`. | `pkg/dashboard/api.go:112` | +| `GET` | `/api/campaigns/{id}/jam` | Dashboard auth/session | Return the per-campaign Jam workspace: section-anchored threads, comments, suggestions, polls, poll decisions, current spec revision and attributed revision history. | `pkg/dashboard/api.go:113` | +| `POST` | `/api/campaigns/{id}/jam` | Read-write role | Record an attributed spec revision for a campaign. Body includes `spec_content`, optional `reason`, and optional agent/model attribution fields. | `pkg/dashboard/api.go:114` | +| `GET` | `/api/campaigns/{id}/jam/threads` | Dashboard auth/session | List async Jam threads anchored to spec sections, including comments and author attribution. | `pkg/dashboard/api.go:115` | +| `POST` | `/api/campaigns/{id}/jam/threads` | Read-write role | Create a section-anchored Jam thread or add a comment to an existing thread. Body includes `section` and `body`, or `thread_id` and `body`. | `pkg/dashboard/api.go:116` | +| `POST` | `/api/campaigns/{id}/jam/agents` | Maintainer only | Invite Spektacular or a configured hive agent into a Jam thread. The agent reply is posted as an attributed thread comment and an open suggestion for human accept/reject; no spec revision is auto-applied. | `pkg/dashboard/api.go:117` | +| `GET` | `/api/campaigns/{id}/jam/suggestions` | Dashboard auth/session | List Jam suggestions with open/accepted/rejected status, proposed text, attribution and applied revision id when accepted. | `pkg/dashboard/api.go:118` | +| `POST` | `/api/campaigns/{id}/jam/suggestions` | Read-write role; maintainer for accept | Create a suggestion (`section`, `proposed_text`) or resolve it with `action: "accept"`/`"reject"` and `suggestion_id`. Accepting records a new attributed spec revision. | `pkg/dashboard/api.go:119` | +| `GET` | `/api/campaigns/{id}/jam/polls` | Dashboard auth/session | List Jam polls, advisory vote tallies and maintainer decisions stored against spec revisions. | `pkg/dashboard/api.go:120` | +| `POST` | `/api/campaigns/{id}/jam/polls` | Read-write role; maintainer for decide | Create polls (`section`, `question`, `options`), cast/update advisory votes (`action: "vote"`), or record a maintainer decision with rationale (`action: "decide"`). | `pkg/dashboard/api.go:121` | +| `GET` | `/api/campaigns/{id}/jam/project-sync` | Dashboard auth/session | Return the campaign's GitHub Projects sync configuration, last published items, last status, and retry guidance for failures. | `pkg/dashboard/api.go:122` | +| `POST` | `/api/campaigns/{id}/jam/project-sync` | Maintainer only | Enable/disable opt-in GitHub Projects sync, publish campaign specs/decisions/derived issues to the linked project, or record inbound project status updates without losing local Jam decisions. | `pkg/dashboard/api.go:123` | +| `GET` | `/api/campaigns/{id}/jam/ws` | Dashboard auth/session | Upgrade to a Jam WebSocket for live participant presence, section focus, and conflict-aware spec co-edit messages. Live edits require read-write role and stale `base_revision_id` values are rejected with a conflict message. | `pkg/dashboard/api.go:124` | +| `GET` | `/api/runs/{key}/trace` | Dashboard auth/session | Resolve `Hive-Run`, `Hive-Plan`, and `Hive-Spec` commit trailers for `?sha=...`, returning the linked plan section, spek clause, approval audit record, rationale audit entries, and typed no-linkage results when trailers are missing. | `pkg/dashboard/api.go:96` | ## Snapshots and style | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:40` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1659` | -| `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:80` | -| `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:81` | -| `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:82` | +| `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:45` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1704` | +| `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:85` | +| `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:86` | +| `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:87` | ## Auth and external accounts | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/gh-auth` | Dashboard auth/session | GitHub Auth | `pkg/dashboard/api.go:181` | -| `GET` | `/api/gh-rate-limits` | Dashboard auth/session | GitHub Rate Limits | `pkg/dashboard/api.go:182` | -| `GET` | `/api/gh-user-auth/status` | Public | GitHub User Auth Status | `pkg/dashboard/api.go:183` | -| `POST` | `/api/gh-user-auth/start` | Public | GitHub User Auth Start | `pkg/dashboard/api.go:184` | -| `POST` | `/api/gh-user-auth/poll` | Public | GitHub User Auth Poll | `pkg/dashboard/api.go:185` | -| `POST` | `/api/gh-user-auth/logout` | Public | GitHub User Auth Logout | `pkg/dashboard/api.go:186` | -| `GET` | `/api/gh-user-auth/session` | Public | GitHub User Auth Session | `pkg/dashboard/api.go:187` | +| `GET` | `/api/gh-auth` | Dashboard auth/session | GitHub Auth | `pkg/dashboard/api.go:186` | +| `GET` | `/api/gh-rate-limits` | Dashboard auth/session | GitHub Rate Limits | `pkg/dashboard/api.go:187` | +| `GET` | `/api/gh-user-auth/status` | Public | GitHub User Auth Status | `pkg/dashboard/api.go:188` | +| `POST` | `/api/gh-user-auth/start` | Public | GitHub User Auth Start | `pkg/dashboard/api.go:189` | +| `POST` | `/api/gh-user-auth/poll` | Public | GitHub User Auth Poll | `pkg/dashboard/api.go:190` | +| `POST` | `/api/gh-user-auth/logout` | Public | GitHub User Auth Logout | `pkg/dashboard/api.go:191` | +| `GET` | `/api/gh-user-auth/session` | Public | GitHub User Auth Session | `pkg/dashboard/api.go:192` | | `GET` | `/api/claude-auth/status` | Dashboard auth/session | Claude Auth Status | `pkg/dashboard/claude_auth.go:54` | | `POST` | `/api/claude-auth/start` | Dashboard auth/session | Claude Auth Start | `pkg/dashboard/claude_auth.go:55` | | `POST` | `/api/claude-auth/exchange` | Dashboard auth/session | Claude Auth Exchange | `pkg/dashboard/claude_auth.go:56` | | `POST` | `/api/claude-auth/logout` | Dashboard auth/session | Claude Auth Logout | `pkg/dashboard/claude_auth.go:57` | -| `GET` | `/api/copilot-auth/status` | Dashboard auth/session | Copilot Auth Status | `pkg/dashboard/copilot_auth.go:85` | -| `POST` | `/api/copilot-auth/start` | Owner only | Copilot Auth Start | `pkg/dashboard/copilot_auth.go:86` | -| `POST` | `/api/copilot-auth/logout` | Owner only | Copilot Auth Logout | `pkg/dashboard/copilot_auth.go:88` | +| `GET` | `/api/copilot-auth/status` | Dashboard auth/session | Copilot Auth Status | `pkg/dashboard/copilot_auth.go:84` | +| `POST` | `/api/copilot-auth/start` | Owner only | Copilot Auth Start | `pkg/dashboard/copilot_auth.go:85` | +| `POST` | `/api/copilot-auth/logout` | Owner only | Copilot Auth Logout | `pkg/dashboard/copilot_auth.go:87` | | `GET` | `/api/openrouter/connect/start` | Dashboard auth/session | Open Router Start | `pkg/dashboard/openrouter.go:45` | | `GET` | `/api/openrouter/qr` | Dashboard auth/session | Open Router QR | `pkg/dashboard/openrouter.go:46` | | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1599` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1600` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1601` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1644` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1645` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1646` | ## Configuration | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/themes` | Dashboard auth/session | Embedded dashboard/contributor theme catalog with swatches for Settings → Appearance; `?scope=contributor` filters profile themes | `pkg/dashboard/api.go:41` | -| `GET` | `/api/theme.css` | Dashboard auth/session | Effective dashboard theme stylesheet with strong ETag / 304 support; `?theme=` previews an embedded preset; `?scope=contributor` emits contributor token aliases and layers hive custom CSS | `pkg/dashboard/api.go:42` | -| `GET` | `/api/config` | Dashboard auth/session | Config | `pkg/dashboard/api.go:43` | -| `GET` | `/api/config/dashboard/theme` | Owner only | Dashboard theme selection, overrides, effective theme, and built-in catalog | `pkg/dashboard/api.go:50` | -| `PUT` | `/api/config/dashboard/theme` | Owner only | Persist `dashboard.theme` and `dashboard.theme_overrides` with validation, CSS/background guardrails, and audit logging | `pkg/dashboard/api.go:51` | -| `GET` | `/api/config/download` | Owner only | Config Download | `pkg/dashboard/api.go:44` | -| `GET` | `/api/config/sla` | Owner only | Delivery SLA targets: `{targets:{start_to_pr_h,pr_to_merge_h,open_to_merge_h}, effective_targets:{...}, defaults:{...}}`, hours, from `sla.targets` in `hive.yaml` with defaults 4/24/72 | `pkg/dashboard/api.go:263` | -| `PUT` | `/api/config/sla` | Owner only | Replace `sla.targets` (`{targets:{start_to_pr_h,pr_to_merge_h,open_to_merge_h}}`; each 0 (default) to 8760, validated before anything is persisted); the Throughput Delivery SLA panel recolours on the next status poll | `pkg/dashboard/api.go:264` | +| `GET` | `/api/themes` | Dashboard auth/session | Embedded dashboard/contributor theme catalog with swatches for Settings → Appearance; `?scope=contributor` filters profile themes | `pkg/dashboard/api.go:46` | +| `GET` | `/api/theme.css` | Dashboard auth/session | Effective dashboard theme stylesheet with strong ETag / 304 support; `?theme=` previews an embedded preset; `?scope=contributor` emits contributor token aliases and layers hive custom CSS | `pkg/dashboard/api.go:47` | +| `GET` | `/api/config` | Dashboard auth/session | Config | `pkg/dashboard/api.go:48` | +| `GET` | `/api/config/dashboard/theme` | Owner only | Dashboard theme selection, overrides, effective theme, and built-in catalog | `pkg/dashboard/api.go:55` | +| `PUT` | `/api/config/dashboard/theme` | Owner only | Persist `dashboard.theme` and `dashboard.theme_overrides` with validation, CSS/background guardrails, and audit logging | `pkg/dashboard/api.go:56` | +| `GET` | `/api/config/download` | Owner only | Config Download | `pkg/dashboard/api.go:49` | +| `GET` | `/api/config/sla` | Owner only | Delivery SLA targets: `{targets:{start_to_pr_h,pr_to_merge_h,open_to_merge_h}, effective_targets:{...}, defaults:{...}}`, hours, from `sla.targets` in `hive.yaml` with defaults 4/24/72 | `pkg/dashboard/api.go:268` | +| `PUT` | `/api/config/sla` | Owner only | Replace `sla.targets` (`{targets:{start_to_pr_h,pr_to_merge_h,open_to_merge_h}}`; each 0 (default) to 8760, validated before anything is persisted); the Throughput Delivery SLA panel recolours on the next status poll | `pkg/dashboard/api.go:269` | | `GET` | `/api/config/export` | Owner only | Redacted JSON export of the effective config, layer files, side files, and provenance | `pkg/dashboard/api_config_export.go` | -| `GET` | `/api/config/provenance` | Owner only | Config Provenance | `pkg/dashboard/api.go:47` | -| `GET` | `/api/config/variables` | Dashboard auth/session | Variables List | `pkg/dashboard/api.go:48` | -| `GET` | `/api/config/authorized-users` | Dashboard auth/session | Authorized Users List | `pkg/dashboard/api.go:49` | -| `PUT` | `/api/config/variables/{name}` | Owner only | Variable Upsert | `pkg/dashboard/api.go:52` | -| `DELETE` | `/api/config/variables/{name}` | Owner only | Variable Delete | `pkg/dashboard/api.go:53` | - -| `GET` | `/api/config/agent/{name}` | Dashboard auth/session | Agent Config Get | `pkg/dashboard/api.go:195` | -| `PUT` | `/api/config/agent/{name}/general` | Owner only | Agent Config General | `pkg/dashboard/api.go:196` | -| `PUT` | `/api/config/agent/{name}/cadences` | Owner only | Agent Config Cadences | `pkg/dashboard/api.go:197` | -| `PUT` | `/api/config/agent/{name}/models` | Owner only | Agent Config Models | `pkg/dashboard/api.go:198` | -| `PUT` | `/api/config/agent/{name}/pipeline` | Owner only | Agent Config Pipeline | `pkg/dashboard/api.go:199` | -| `PUT` | `/api/config/agent/{name}/hooks` | Owner only | Agent Config Hooks | `pkg/dashboard/api.go:200` | -| `PUT` | `/api/config/agent/{name}/restrictions` | Owner only | Agent Config Restrictions | `pkg/dashboard/api.go:201` | -| `PUT` | `/api/config/agent/{name}/stats` | Owner only | Agent Config Stats | `pkg/dashboard/api.go:202` | -| `GET` | `/api/config/agent/{name}/prompt` | Dashboard auth/session | Agent Prompt | `pkg/dashboard/api.go:203` | -| `PUT` | `/api/config/agent/{name}/prompt` | Owner only | Agent Prompt Save | `pkg/dashboard/api.go:204` | -| `GET` | `/api/config/agent/{name}/export` | Dashboard auth/session | Agent Export | `pkg/dashboard/api.go:205` | -| `PUT` | `/api/config/agent/{name}/channels` | Owner only | Agent Config Channels | `pkg/dashboard/api.go:206` | -| `PUT` | `/api/config/agent/{name}/tools` | Owner only | Agent Config Tools | `pkg/dashboard/api.go:207` | -| `PUT` | `/api/config/agent/{name}/connections` | Owner only | Agent Config Connections | `pkg/dashboard/api.go:208` | +| `GET` | `/api/config/provenance` | Owner only | Config Provenance | `pkg/dashboard/api.go:52` | +| `GET` | `/api/config/variables` | Dashboard auth/session | Variables List | `pkg/dashboard/api.go:53` | +| `GET` | `/api/config/authorized-users` | Dashboard auth/session | Authorized Users List | `pkg/dashboard/api.go:54` | +| `PUT` | `/api/config/variables/{name}` | Owner only | Variable Upsert | `pkg/dashboard/api.go:57` | +| `DELETE` | `/api/config/variables/{name}` | Owner only | Variable Delete | `pkg/dashboard/api.go:58` | + +| `GET` | `/api/config/agent/{name}` | Dashboard auth/session | Agent Config Get | `pkg/dashboard/api.go:200` | +| `PUT` | `/api/config/agent/{name}/general` | Owner only | Agent Config General | `pkg/dashboard/api.go:201` | +| `PUT` | `/api/config/agent/{name}/cadences` | Owner only | Agent Config Cadences | `pkg/dashboard/api.go:202` | +| `PUT` | `/api/config/agent/{name}/models` | Owner only | Agent Config Models | `pkg/dashboard/api.go:203` | +| `PUT` | `/api/config/agent/{name}/pipeline` | Owner only | Agent Config Pipeline | `pkg/dashboard/api.go:204` | +| `PUT` | `/api/config/agent/{name}/hooks` | Owner only | Agent Config Hooks | `pkg/dashboard/api.go:205` | +| `PUT` | `/api/config/agent/{name}/restrictions` | Owner only | Agent Config Restrictions | `pkg/dashboard/api.go:206` | +| `PUT` | `/api/config/agent/{name}/stats` | Owner only | Agent Config Stats | `pkg/dashboard/api.go:207` | +| `GET` | `/api/config/agent/{name}/prompt` | Dashboard auth/session | Agent Prompt | `pkg/dashboard/api.go:208` | +| `PUT` | `/api/config/agent/{name}/prompt` | Owner only | Agent Prompt Save | `pkg/dashboard/api.go:209` | +| `GET` | `/api/config/agent/{name}/export` | Dashboard auth/session | Agent Export | `pkg/dashboard/api.go:210` | +| `PUT` | `/api/config/agent/{name}/channels` | Owner only | Agent Config Channels | `pkg/dashboard/api.go:211` | +| `PUT` | `/api/config/agent/{name}/tools` | Owner only | Agent Config Tools | `pkg/dashboard/api.go:212` | +| `PUT` | `/api/config/agent/{name}/connections` | Owner only | Agent Config Connections | `pkg/dashboard/api.go:213` | | `GET` | `/api/config/agent/{name}/variables` | Dashboard auth/session | Per-agent Variables tab: the agent's own `${VAR}` definitions plus the hive-level ones it inherits (each flagged `overridden` when the agent redefines it) and the seed-only exec/http gate flags. Never returns values — only name, type, scope and a non-secret source hint | `pkg/dashboard/api_variables.go:298` | | `PUT` | `/api/config/agent/{name}/variables/{var}` | Owner only | Create or update one per-agent variable (`static`/`env` only; `script`/`http` are 403 and seed-only; scope `template` or `both`; secret-looking static values rejected). Persisted on the agent entry; overrides a same-named hive-level variable in this agent's kick prompt | `pkg/dashboard/api_variables.go:299` | | `DELETE` | `/api/config/agent/{name}/variables/{var}` | Owner only | Delete one per-agent variable (404 for inherited hive-level variables — delete those via `/api/config/variables/{name}`) | `pkg/dashboard/api_variables.go:300` | -| `GET` | `/api/advisor/records` | Read-write role | Advisor Records — query `?agent=` (filter by agent), `?since=`/`?until=` (RFC3339, either omitted), or `?hours=` (lookback 1..720, alternative to since), optional `?limit=` (capped at 200 by default); returns `{records: [], active?, active_reason?}` where active/reason appear only when filtered to one agent. Records are scrubbed at the boundary. Response shape in [api_advisor.go](../pkg/dashboard/api_advisor.go#L38) | `pkg/dashboard/api.go:55` | -| `GET` | `/api/advisor/spend` | Dashboard auth/session | Advisor Spend — per-agent advisor spend over one window, alongside (not folded into) the agent's own spend. Query presets `?range=hourly|daily|weekly|monthly` (Cost section timeframes) or custom `?since=`/`?until=` (RFC3339) or `?hours=` (lookback 1..720); optionally filter `?agent=`. Returns `{range, since?, until?, total_cost_usd, agents:[]}` where range is the preset or "custom". Response shape in [api_advisor.go](../pkg/dashboard/api_advisor.go#L151) | `pkg/dashboard/api.go:56` | -| `GET` | `/api/config/advisor` | Owner only | Advisor Config Get (fleet block, model roles map, and per-agent overrides; includes effective `fleet.enabled` boolean from IsEnabled()) | `pkg/dashboard/api.go:57` | -| `PUT` | `/api/config/advisor` | Owner only | Advisor Config Set (fleet-wide settings; only fields present in body change) | `pkg/dashboard/api.go:58` | -| `PUT` | `/api/config/advisor/roles` | Owner only | Advisor Model Roles Set (replaces named model_roles map; roles in use cannot be removed) | `pkg/dashboard/api.go:59` | -| `PUT` | `/api/config/advisor/agent/{name}` | Owner only | Advisor Agent Override Set (set or clear one agent's override; null/empty clears it) | `pkg/dashboard/api.go:60` | -| `GET` | `/api/config/stat-sources` | Dashboard auth/session | Stat Sources | `pkg/dashboard/api.go:209` | -| `GET` | `/api/config/governor` | Dashboard auth/session | Governor Config Get | `pkg/dashboard/api.go:211` | -| `PUT` | `/api/config/governor/sensing` | Owner only | Governor Sensing | `pkg/dashboard/api.go:214` | -| `PUT` | `/api/config/governor/thresholds` | Owner only | Governor Thresholds | `pkg/dashboard/api.go:215` | -| `PUT` | `/api/config/governor/labels` | Owner only | Governor Labels | `pkg/dashboard/api.go:222` | -| `PUT` | `/api/config/governor/budget` | Owner only | Governor Budget | `pkg/dashboard/api.go:223` | -| `PUT` | `/api/config/governor/notifications` | Owner only | Governor Notifications | `pkg/dashboard/api.go:227` | -| `PUT` | `/api/config/governor/health` | Owner only | Governor Health | `pkg/dashboard/api.go:228` | -| `PUT` | `/api/config/governor/logging` | Owner only | Governor Logging | `pkg/dashboard/api.go:244` | -| `PUT` | `/api/config/governor/attribution` | Owner only | Governor Attribution | `pkg/dashboard/api.go:245` | -| `PUT` | `/api/config/governor/hub` | Owner only | Governor Hub | `pkg/dashboard/api.go:246` | -| `PUT` | `/api/config/governor/litellm` | Owner only | Governor Lite LLM | `pkg/dashboard/api.go:247` | -| `PUT` | `/api/config/governor/trajectory` | Owner only | Governor Trajectory | `pkg/dashboard/api.go:248` | +| `GET` | `/api/config/stat-sources` | Dashboard auth/session | Stat Sources | `pkg/dashboard/api.go:214` | +| `GET` | `/api/config/governor` | Dashboard auth/session | Governor Config Get | `pkg/dashboard/api.go:216` | +| `PUT` | `/api/config/governor/sensing` | Owner only | Governor Sensing | `pkg/dashboard/api.go:219` | +| `PUT` | `/api/config/governor/thresholds` | Owner only | Governor Thresholds | `pkg/dashboard/api.go:220` | +| `PUT` | `/api/config/governor/labels` | Owner only | Governor Labels | `pkg/dashboard/api.go:227` | +| `PUT` | `/api/config/governor/budget` | Owner only | Governor Budget | `pkg/dashboard/api.go:228` | +| `PUT` | `/api/config/governor/notifications` | Owner only | Governor Notifications | `pkg/dashboard/api.go:232` | +| `PUT` | `/api/config/governor/health` | Owner only | Governor Health | `pkg/dashboard/api.go:233` | +| `PUT` | `/api/config/governor/logging` | Owner only | Governor Logging | `pkg/dashboard/api.go:249` | +| `PUT` | `/api/config/governor/attribution` | Owner only | Governor Attribution | `pkg/dashboard/api.go:250` | +| `PUT` | `/api/config/governor/hub` | Owner only | Governor Hub | `pkg/dashboard/api.go:251` | +| `PUT` | `/api/config/governor/litellm` | Owner only | Governor Lite LLM | `pkg/dashboard/api.go:252` | +| `PUT` | `/api/config/governor/trajectory` | Owner only | Governor Trajectory | `pkg/dashboard/api.go:253` | | `GET` | `/api/config/governor/backup` | Owner only | Backup Key Status (presence + safe source label; never the key value) | `pkg/dashboard/backup_key.go` | | `PUT` | `/api/config/governor/backup` | Owner only | Backup Key Set (64-hex AES-256 key; stored 0600, path-only in `hive.yaml`) | `pkg/dashboard/backup_key.go` | | `DELETE` | `/api/config/governor/backup` | Owner only | Backup Key Clear (backups are refused again) | `pkg/dashboard/backup_key.go` | -| `GET` | `/api/config/governor/bob` | Dashboard auth/session | Governor Bob Status | `pkg/dashboard/api.go:293` | -| `PUT` | `/api/config/governor/bob` | Owner only | Governor Bob Key | `pkg/dashboard/api.go:294` | -| `DELETE` | `/api/config/governor/bob` | Owner only | Governor Bob Key Clear | `pkg/dashboard/api.go:295` | -| `GET` | `/api/config/governor/cadence-scope` | Owner only | Governor Cadence Scope Get (`aggregate` or `per_repo`) | `pkg/dashboard/api.go:216` | -| `PUT` | `/api/config/governor/cadence-scope` | Owner only | Governor Cadence Scope Set (`cadenceScope`/`cadence_scope`: `aggregate`, `per_repo`, or empty for default) | `pkg/dashboard/api.go:217` | -| `POST` | `/api/config/governor/bob/test` | Dashboard auth/session | Governor Bob Key Test | `pkg/dashboard/api.go:297` | -| `POST` | `/api/config/governor/litellm/test` | Dashboard auth/session | Governor Lite LLMTest | `pkg/dashboard/api.go:298` | -| `GET` | `/api/config/governor/gateways` | Dashboard auth/session | Governor Gateways List | `pkg/dashboard/api.go:301` | -| `PUT` | `/api/config/governor/gateways` | Owner only | Governor Gateways Upsert | `pkg/dashboard/api.go:302` | -| `DELETE` | `/api/config/governor/gateways/{name}` | Owner only | Governor Gateways Delete | `pkg/dashboard/api.go:303` | -| `POST` | `/api/config/governor/gateways/{name}/test` | Dashboard auth/session | Governor Gateways Test | `pkg/dashboard/api.go:304` | -| `POST` | `/api/config/governor/gateways/discover` | Owner only | Governor Gateways Discover | `pkg/dashboard/api.go:305` | -| `POST` | `/api/config/governor/agents` | Owner only | Governor Add Agent | `pkg/dashboard/api.go:310` | -| `DELETE` | `/api/config/governor/agents/{name}` | Owner only | Governor Remove Agent | `pkg/dashboard/api.go:311` | -| `PUT` | `/api/config/governor/repos` | Owner only | Governor Repos | `pkg/dashboard/api.go:312` | -| `POST` | `/api/config/governor/repos/check-access` | Dashboard auth/session | Governor Repo Check Access | `pkg/dashboard/api.go:316` | -| `PUT` | `/api/config/github` | Owner only | Config GitHub | `pkg/dashboard/api.go:317` | -| `GET` | `/api/config/github/forge-apps` | Dashboard auth/session | Config GitHub Forge Apps | `pkg/dashboard/api.go:320` | -| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:365` | -| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:366` | -| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:367` | -| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:357` | -| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:358` | -| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:359` | -| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:360` | -| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:361` | -| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:362` | -| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:363` | -| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:535` | -| `GET` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Get | `pkg/dashboard/api.go:219` | -| `PUT` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Set | `pkg/dashboard/api.go:220` | -| `PUT` | `/api/config/governor/coverage-target` | Owner only | Governor Coverage Target | `pkg/dashboard/api.go:221` | -| `POST` | `/api/config/governor/budget/reset` | Owner only | Governor Budget Reset | `pkg/dashboard/api.go:224` | -| `PUT` | `/api/config/governor/watchdog` | Owner only | Governor Watchdog | `pkg/dashboard/api.go:229` | -| `GET` | `/api/config/escalation` | Owner only | Escalation Config Get | `pkg/dashboard/api.go:232` | -| `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:233` | -| `GET` | `/api/config/channels` | Owner only | Settings → Channels payload (GitHub mention/Actions triggers, chat bots, escalation sinks, read-only dashboard chat and admin MCP status, mention-eligible agents); secrets masked | `pkg/dashboard/api.go:236` | -| `PUT` | `/api/config/channels` | Owner only | Partial update of the same surfaces; validated against a copy so a 400 applies nothing; masked echo keeps a secret, empty clears it; hot-reloads mention poller and escalation sinks | `pkg/dashboard/api.go:237` | -| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:531` | -| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:532` | -| `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:240` | -| `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:243` | -| `PUT` | `/api/config/governor/features` | Owner only | Governor Features | `pkg/dashboard/api.go:249` | -| `GET` | `/api/config/governor/general-advanced` | Owner only | Governor General Advanced Get | `pkg/dashboard/api.go:250` | -| `PUT` | `/api/config/governor/general-advanced` | Owner only | Governor General Advanced Set | `pkg/dashboard/api.go:251` | -| `GET` | `/api/config/auto-merge` | Owner only | Auto-Merge Config Get | `pkg/dashboard/api.go:255` | -| `PUT` | `/api/config/auto-merge` | Owner only | Auto-Merge Config Set | `pkg/dashboard/api.go:256` | -| `GET` | `/api/config/convergence` | Owner only | Convergence Config Get | `pkg/dashboard/api.go:261` | -| `PUT` | `/api/config/convergence` | Owner only | Convergence Config Set | `pkg/dashboard/api.go:262` | -| `GET` | `/api/config/governor/advisory` | Owner only | Governor Advisory Get | `pkg/dashboard/api.go:266` | -| `PUT` | `/api/config/governor/advisory` | Owner only | Governor Advisory Set | `pkg/dashboard/api.go:267` | -| `GET` | `/api/config/governor/replan` | Owner only | Governor Replan Get | `pkg/dashboard/api.go:268` | -| `PUT` | `/api/config/governor/replan` | Owner only | Governor Replan Set | `pkg/dashboard/api.go:269` | -| `GET` | `/api/config/governor/work-source` | Owner only | Governor Work Source Get (Jira secrets/TLS fields reported as set/unset only) | `pkg/dashboard/api.go:279` | -| `PUT` | `/api/config/governor/work-source` | Owner only | Governor Work Source Set (including Jira Data Center TLS settings) | `pkg/dashboard/api.go:280` | -| `PUT` | `/api/config/governor/security` | Owner only | Governor Security | `pkg/dashboard/api.go:281` | -| `GET` | `/api/config/governor/project-observability` | Owner only | Governor Project Observability Get | `pkg/dashboard/api.go:283` | -| `PUT` | `/api/config/governor/project-observability` | Owner only | Governor Project Observability Set | `pkg/dashboard/api.go:284` | -| `GET` | `/api/config/governor/inference-auth` | Owner only | Governor Inference Auth Get | `pkg/dashboard/api.go:299` | -| `PUT` | `/api/config/governor/inference-auth` | Owner only | Governor Inference Auth Set | `pkg/dashboard/api.go:300` | +| `GET` | `/api/config/governor/bob` | Dashboard auth/session | Governor Bob Status | `pkg/dashboard/api.go:298` | +| `PUT` | `/api/config/governor/bob` | Owner only | Governor Bob Key | `pkg/dashboard/api.go:299` | +| `DELETE` | `/api/config/governor/bob` | Owner only | Governor Bob Key Clear | `pkg/dashboard/api.go:300` | + +| `GET` | `/api/config/governor/cadence-scope` | Owner only | Governor Cadence Scope Get (`aggregate` or `per_repo`) | `pkg/dashboard/api.go:221` | +| `PUT` | `/api/config/governor/cadence-scope` | Owner only | Governor Cadence Scope Set (`cadenceScope`/`cadence_scope`: `aggregate`, `per_repo`, or empty for default) | `pkg/dashboard/api.go:222` | +| `POST` | `/api/config/governor/bob/test` | Dashboard auth/session | Governor Bob Key Test | `pkg/dashboard/api.go:302` | +| `POST` | `/api/config/governor/litellm/test` | Dashboard auth/session | Governor Lite LLMTest | `pkg/dashboard/api.go:303` | +| `GET` | `/api/config/governor/gateways` | Dashboard auth/session | Governor Gateways List | `pkg/dashboard/api.go:306` | +| `PUT` | `/api/config/governor/gateways` | Owner only | Governor Gateways Upsert | `pkg/dashboard/api.go:307` | +| `DELETE` | `/api/config/governor/gateways/{name}` | Owner only | Governor Gateways Delete | `pkg/dashboard/api.go:308` | +| `POST` | `/api/config/governor/gateways/{name}/test` | Dashboard auth/session | Governor Gateways Test | `pkg/dashboard/api.go:309` | +| `POST` | `/api/config/governor/gateways/discover` | Owner only | Governor Gateways Discover | `pkg/dashboard/api.go:310` | +| `POST` | `/api/config/governor/agents` | Owner only | Governor Add Agent | `pkg/dashboard/api.go:315` | +| `DELETE` | `/api/config/governor/agents/{name}` | Owner only | Governor Remove Agent | `pkg/dashboard/api.go:316` | +| `PUT` | `/api/config/governor/repos` | Owner only | Governor Repos | `pkg/dashboard/api.go:317` | +| `POST` | `/api/config/governor/repos/check-access` | Dashboard auth/session | Governor Repo Check Access | `pkg/dashboard/api.go:321` | +| `PUT` | `/api/config/github` | Owner only | Config GitHub | `pkg/dashboard/api.go:322` | +| `GET` | `/api/config/github/forge-apps` | Dashboard auth/session | Config GitHub Forge Apps | `pkg/dashboard/api.go:325` | +| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:372` | +| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:373` | +| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:374` | +| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:364` | +| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:365` | +| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:366` | +| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:367` | +| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:368` | +| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:369` | +| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:370` | +| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:542` | + +| `GET` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Get | `pkg/dashboard/api.go:224` | +| `PUT` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Set | `pkg/dashboard/api.go:225` | +| `PUT` | `/api/config/governor/coverage-target` | Owner only | Governor Coverage Target | `pkg/dashboard/api.go:226` | +| `POST` | `/api/config/governor/budget/reset` | Owner only | Governor Budget Reset | `pkg/dashboard/api.go:229` | +| `PUT` | `/api/config/governor/watchdog` | Owner only | Governor Watchdog | `pkg/dashboard/api.go:234` | +| `GET` | `/api/config/escalation` | Owner only | Escalation Config Get | `pkg/dashboard/api.go:237` | +| `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:238` | +| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:538` | +| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:539` | +| `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:245` | +| `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:248` | +| `PUT` | `/api/config/governor/features` | Owner only | Governor Features | `pkg/dashboard/api.go:254` | +| `GET` | `/api/config/governor/general-advanced` | Owner only | Governor General Advanced Get | `pkg/dashboard/api.go:255` | +| `PUT` | `/api/config/governor/general-advanced` | Owner only | Governor General Advanced Set | `pkg/dashboard/api.go:256` | +| `GET` | `/api/config/auto-merge` | Owner only | Auto-Merge Config Get | `pkg/dashboard/api.go:260` | +| `PUT` | `/api/config/auto-merge` | Owner only | Auto-Merge Config Set | `pkg/dashboard/api.go:261` | +| `GET` | `/api/config/convergence` | Owner only | Convergence Config Get | `pkg/dashboard/api.go:266` | +| `PUT` | `/api/config/convergence` | Owner only | Convergence Config Set | `pkg/dashboard/api.go:267` | +| `GET` | `/api/config/governor/advisory` | Owner only | Governor Advisory Get | `pkg/dashboard/api.go:271` | +| `PUT` | `/api/config/governor/advisory` | Owner only | Governor Advisory Set | `pkg/dashboard/api.go:272` | +| `GET` | `/api/config/governor/replan` | Owner only | Governor Replan Get | `pkg/dashboard/api.go:273` | +| `PUT` | `/api/config/governor/replan` | Owner only | Governor Replan Set | `pkg/dashboard/api.go:274` | +| `GET` | `/api/config/governor/work-source` | Owner only | Governor Work Source Get (Jira secrets/TLS fields reported as set/unset only) | `pkg/dashboard/api.go:284` | +| `PUT` | `/api/config/governor/work-source` | Owner only | Governor Work Source Set (including Jira Data Center TLS settings) | `pkg/dashboard/api.go:285` | +| `PUT` | `/api/config/governor/security` | Owner only | Governor Security | `pkg/dashboard/api.go:286` | +| `GET` | `/api/config/governor/project-observability` | Owner only | Governor Project Observability Get | `pkg/dashboard/api.go:288` | +| `PUT` | `/api/config/governor/project-observability` | Owner only | Governor Project Observability Set | `pkg/dashboard/api.go:289` | +| `GET` | `/api/config/governor/inference-auth` | Owner only | Governor Inference Auth Get | `pkg/dashboard/api.go:304` | +| `PUT` | `/api/config/governor/inference-auth` | Owner only | Governor Inference Auth Set | `pkg/dashboard/api.go:305` | +| `GET` | `/api/advisor/records` | Read-write role | Advisor Records — query `?agent=` (filter by agent), `?since=`/`?until=` (RFC3339, either omitted), or `?hours=` (lookback 1..720, alternative to since), optional `?limit=` (capped at 200 by default); returns `{records: [], active?, active_reason?}` where active/reason appear only when filtered to one agent. Records are scrubbed at the boundary. Response shape in [api_advisor.go](../pkg/dashboard/api_advisor.go#L38) | `pkg/dashboard/api.go:60` | +| `GET` | `/api/advisor/spend` | Dashboard auth/session | Advisor Spend — per-agent advisor spend over one window, alongside (not folded into) the agent's own spend. Query presets `?range=hourly|daily|weekly|monthly` (Cost section timeframes) or custom `?since=`/`?until=` (RFC3339) or `?hours=` (lookback 1..720); optionally filter `?agent=`. Returns `{range, since?, until?, total_cost_usd, agents:[]}` where range is the preset or "custom". Response shape in [api_advisor.go](../pkg/dashboard/api_advisor.go#L151) | `pkg/dashboard/api.go:61` | +| `GET` | `/api/config/advisor` | Owner only | Advisor Config Get (fleet block, model roles map, and per-agent overrides; includes effective `fleet.enabled` boolean from IsEnabled()) | `pkg/dashboard/api.go:62` | +| `PUT` | `/api/config/advisor` | Owner only | Advisor Config Set (fleet-wide settings; only fields present in body change) | `pkg/dashboard/api.go:63` | +| `PUT` | `/api/config/advisor/roles` | Owner only | Advisor Model Roles Set (replaces named model_roles map; roles in use cannot be removed) | `pkg/dashboard/api.go:64` | +| `PUT` | `/api/config/advisor/agent/{name}` | Owner only | Advisor Agent Override Set (set or clear one agent's override; null/empty clears it) | `pkg/dashboard/api.go:65` | +| `GET` | `/api/config/channels` | Owner only | Settings → Channels payload (GitHub mention/Actions triggers, chat bots, escalation sinks, read-only dashboard chat and admin MCP status, mention-eligible agents); secrets masked | `pkg/dashboard/api.go:241` | +| `PUT` | `/api/config/channels` | Owner only | Partial update of the same surfaces; validated against a copy so a 400 applies nothing; masked echo keeps a secret, empty clears it; hot-reloads mention poller and escalation sinks | `pkg/dashboard/api.go:242` | ## Agents and controls | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `POST` | `/api/kick/{agent}` | Owner only | Kick — asynchronous; answers `202` once queued (see below) | `pkg/dashboard/api.go:141` | -| `GET` | `/api/kick/{agent}/status` | Dashboard auth/session | Outcome of the most recent kick | `pkg/dashboard/api.go:145` | -| `POST` | `/api/switch/{agent}/{backend}` | Owner only | Switch | `pkg/dashboard/api.go:146` | -| `POST` | `/api/model/{agent}/{model}` | Owner only | Model Set | `pkg/dashboard/api.go:147` | -| `POST` | `/api/effort/{agent}/{effort}` | Owner only | Set launch-only reasoning effort after validating against the agent's live backend (including runtime override); persists config/agent overlay, restarts the session, and `default` clears the stored effort | `pkg/dashboard/api.go:148` | -| `POST` | `/api/pause/{agent}` | Owner only | Pause | `pkg/dashboard/api.go:149` | -| `POST` | `/api/resume/{agent}` | Owner only | Resume | `pkg/dashboard/api.go:150` | -| `GET` | `/api/agent-state/{agent}` | Dashboard auth/session | Agent State | `pkg/dashboard/api.go:151` | -| `GET` | `/api/breaker` | Dashboard auth/session | Breaker State | `pkg/dashboard/api.go:152` | -| `POST` | `/api/breaker/engage` | Owner only | Breaker Engage | `pkg/dashboard/api.go:153` | -| `POST` | `/api/breaker/release` | Owner only | Breaker Release | `pkg/dashboard/api.go:154` | -| `POST` | `/api/pin/{agent}/{dimension}` | Owner only | Pin | `pkg/dashboard/api.go:155` | -| `POST` | `/api/unpin/{agent}/{dimension}` | Owner only | Unpin | `pkg/dashboard/api.go:156` | -| `POST` | `/api/restart/{agent}` | Owner only | Restart | `pkg/dashboard/api.go:160` | -| `GET` | `/api/model-advisor` | Dashboard auth/session | Model Advisor | `pkg/dashboard/api.go:176` | -| `GET` | `/api/governor/pr-models` | Dashboard auth/session | Agent-authored PR distribution by normalized attribution model/backend for `window=7d`, `30d`, or `all`, including per-model rework stats and the top 10 most-reworked PRs | `pkg/dashboard/api.go:177` | -| `GET` | `/api/reviewer/accuracy` | Dashboard auth/session | Reviewer verdict calibration for the configured window: false-approve/false-block rates and confidence buckets per perspective and reviewer model | `pkg/dashboard/api.go:242` | -| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:510` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:513` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:514` | -| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:518` | +| `POST` | `/api/kick/{agent}` | Owner only | Kick — asynchronous; answers `202` once queued (see below) | `pkg/dashboard/api.go:146` | +| `GET` | `/api/kick/{agent}/status` | Dashboard auth/session | Outcome of the most recent kick | `pkg/dashboard/api.go:150` | +| `POST` | `/api/switch/{agent}/{backend}` | Owner only | Switch | `pkg/dashboard/api.go:151` | +| `POST` | `/api/model/{agent}/{model}` | Owner only | Model Set | `pkg/dashboard/api.go:152` | +| `POST` | `/api/effort/{agent}/{effort}` | Owner only | Set launch-only reasoning effort after validating against the agent's live backend (including runtime override); persists config/agent overlay, restarts the session, and `default` clears the stored effort | `pkg/dashboard/api.go:153` | +| `POST` | `/api/pause/{agent}` | Owner only | Pause | `pkg/dashboard/api.go:154` | +| `POST` | `/api/resume/{agent}` | Owner only | Resume | `pkg/dashboard/api.go:155` | +| `GET` | `/api/agent-state/{agent}` | Dashboard auth/session | Agent State | `pkg/dashboard/api.go:156` | +| `GET` | `/api/breaker` | Dashboard auth/session | Breaker State | `pkg/dashboard/api.go:157` | +| `POST` | `/api/breaker/engage` | Owner only | Breaker Engage | `pkg/dashboard/api.go:158` | +| `POST` | `/api/breaker/release` | Owner only | Breaker Release | `pkg/dashboard/api.go:159` | +| `POST` | `/api/pin/{agent}/{dimension}` | Owner only | Pin | `pkg/dashboard/api.go:160` | +| `POST` | `/api/unpin/{agent}/{dimension}` | Owner only | Unpin | `pkg/dashboard/api.go:161` | +| `POST` | `/api/restart/{agent}` | Owner only | Restart | `pkg/dashboard/api.go:165` | +| `GET` | `/api/model-advisor` | Dashboard auth/session | Model Advisor | `pkg/dashboard/api.go:181` | +| `GET` | `/api/governor/pr-models` | Dashboard auth/session | Agent-authored PR distribution by normalized attribution model/backend for `window=7d`, `30d`, or `all`, including per-model rework stats and the top 10 most-reworked PRs | `pkg/dashboard/api.go:182` | +| `GET` | `/api/reviewer/accuracy` | Dashboard auth/session | Reviewer verdict calibration for the configured window: false-approve/false-block rates and confidence buckets per perspective and reviewer model | `pkg/dashboard/api.go:247` | +| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:517` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:520` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:521` | +| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:525` | | `GET` | `/api/review/dispatch/events` | Dashboard auth/session | Webhook review dispatch queue: `enabled`, `webhooks_configured`, `debounce_s`, `pending` (soonest first) and `recent` (newest first) dispatches with repo, number, head SHA, action, timestamps and coalesced count, plus `stats` counters; in memory, never calls GitHub | `pkg/dashboard/api_review_event_dispatch.go:16` | -| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:522` | -| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:523` | -| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:525` | -| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `///issues/.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:528` | -| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:322` | -| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:323` | -| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:324` | -| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:325` | -| `GET` | `/api/agents/{name}/log` | Dashboard auth/session | Agent Full Log | `pkg/dashboard/api.go:128` | -| `GET` | `/api/agents/{name}/terminal-urls` | Dashboard auth/session | Agent Terminal URLs | `pkg/dashboard/api.go:132` | -| `GET` | `/api/agents/{name}/kicks` | Dashboard auth/session | Agent Kick Log List | `pkg/dashboard/api.go:135` | -| `GET` | `/api/agents/{name}/kicks/{id}` | Dashboard auth/session | Agent Kick Log Get | `pkg/dashboard/api.go:136` | -| `GET` | `/agents/{name}/kicks` | Dashboard auth/session | Agent Kick History Page (HTML) | `pkg/dashboard/api.go:137` | -| `POST` | `/api/agents/{name}/login-code` | Owner only | Agent Login Code | `pkg/dashboard/api.go:159` | +| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:529` | +| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:530` | +| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:532` | +| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `///issues/.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:535` | +| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:327` | +| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:328` | +| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:329` | +| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:330` | + +| `GET` | `/api/agents/{name}/log` | Dashboard auth/session | Agent Full Log | `pkg/dashboard/api.go:133` | +| `GET` | `/api/agents/{name}/terminal-urls` | Dashboard auth/session | Agent Terminal URLs | `pkg/dashboard/api.go:137` | +| `GET` | `/api/agents/{name}/kicks` | Dashboard auth/session | Agent Kick Log List | `pkg/dashboard/api.go:140` | +| `GET` | `/api/agents/{name}/kicks/{id}` | Dashboard auth/session | Agent Kick Log Get | `pkg/dashboard/api.go:141` | +| `GET` | `/agents/{name}/kicks` | Dashboard auth/session | Agent Kick History Page (HTML) | `pkg/dashboard/api.go:142` | +| `POST` | `/api/agents/{name}/login-code` | Owner only | Agent Login Code | `pkg/dashboard/api.go:164` | ### Kick is asynchronous @@ -279,77 +282,78 @@ Read the result from `GET /api/kick/{agent}/status`, which returns `status` of ` | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:327` | -| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:328` | -| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:329` | +| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:332` | +| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:333` | +| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:334` | | `POST` | `/api/repos/rescan` | Dashboard auth/session | Re-enumerate watched repositories' open issues/PRs for the dashboard Projects cards. The scan is read-only with respect to agents/governor actions, collapses concurrent requests, and debounces repeated presses for 30 seconds (`repoRescanDebounce`) so the button cannot hammer GitHub | `pkg/dashboard/api.go`, `pkg/dashboard/api_repos_rescan.go` | | `POST` | `/api/repos/pause` | Owner | Quiet ONE repository without stopping the hive: agents stop writing to it and stop being handed work on it, while it keeps its dashboard card and ACMM eval. Body `{"repo": "...", "reason": "..."}` — the repo travels in the body because a `project.repos` entry may be an explicit cross-org `owner/name`. Records who/when/why; `changed: false` marks a no-op re-pause and leaves the original provenance intact; `persisted: false` means the pause is in force but could not be written to config. Rejects a repo outside `project.repos`. See [per-repo agent pause](repo-pause.md) | `pkg/dashboard/api_repo_pause.go` | | `POST` | `/api/repos/resume` | Owner | Lift a repository's pause. Unlike pause, does not require the repo to be in `project.repos`, so a pause left behind by a removed repo can still be cleared | `pkg/dashboard/api_repo_pause.go` | | `GET` | `/api/repos/pauses` | Dashboard auth/session | Every repo pause with its provenance. Read-only — the same state is already on the repository cards | `pkg/dashboard/api_repo_pause.go` | -| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:347`, `pkg/dashboard/api_repo_hold.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/` label; `{"held": false}` removes exactly the labels causing the hold (`hive-pause/` and/or generic hold labels) and never removes `hive/` provenance. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-.json` with audit-backed, provenance-only, and ambiguous legacy `hive/` classifications. | `pkg/dashboard/api.go:348`, `pkg/dashboard/api_repo_hold.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458"}`. On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision ` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator (the merge gate decides from there; no review, label or merge), and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR only (409 otherwise). The close variants post a comment naming the operator (and the superseding PR when `superseded_by` is given) and close the item as not planned. Never merges. Every decision writes an audit-log entry (`repo_item_decision_*`). | `pkg/dashboard/api.go:350`, `pkg/dashboard/api_repo_item_decision.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:544`, `pkg/dashboard/api_repo_item_accept_triage.go` | -| `POST` | `/api/repos/{owner}/{repo}/issues/{number}/release` | Owner or repo write/maintain/admin | Execute the typed `kind=release` owner step the hub proposed on a parked issue, with the same live preconditions as `/hive approve` ([#12635](https://github.com/hivecommons/hive/issues/12635)). Tag and target are resolved from the trusted owner-step, never from the request. Returns 503 when the GitHub client or config is unavailable; 409 when the repository is `label_driven` (the same refusal `/hive approve` gets there), when `release_on_approve` is off, or when a live precondition fails (`error` carries the refusal reason, which is also posted on the issue); 502 when the release write itself fails. Success returns `{ok, repo, number, tag, url, minStatusSeq}` plus `warning` if the follow-up issue comment failed after the release was created. | `pkg/dashboard/api.go:349`, `pkg/dashboard/api_issue_release.go` | -| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:352`, `pkg/dashboard/api_acmm_eval.go` | -| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:353` | -| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](acmm-advisor.md) | `pkg/dashboard/api.go:354` | -| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`, and on the needs-human advice `why` (plain-language reasons with counts) and `help`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](advisory.md#owner-advice) | `pkg/dashboard/api.go:356`, `pkg/dashboard/api_hive_advice.go` | +| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:352`, `pkg/dashboard/api_repo_hold.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/` label; `{"held": false}` removes exactly the labels causing the hold (`hive-pause/` and/or generic hold labels) and never removes `hive/` provenance. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-.json` with audit-backed, provenance-only, and ambiguous legacy `hive/` classifications. | `pkg/dashboard/api.go:353`, `pkg/dashboard/api_repo_hold.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458"}`. On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision ` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator (the merge gate decides from there; no review, label or merge), and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR only (409 otherwise). The close variants post a comment naming the operator (and the superseding PR when `superseded_by` is given) and close the item as not planned. Never merges. Every decision writes an audit-log entry (`repo_item_decision_*`). | `pkg/dashboard/api.go:355`, `pkg/dashboard/api_repo_item_decision.go` | +| `POST` / `GET` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated` | Same as `…/hold` (owner or repo write), signed-in GitHub user; `decisions.delegate.enabled` (403 when off) | *Let hive decide*: enqueue a delegated-decision job in which the hive decides the parked item itself as the owner's delegate under `pkg/delegatedecision.PolicyText` and records it as a human decision (`/hive decision` comment with the `` marker, park labels off, `triage/accepted`, queue approval on a PR head). Body `{"type": "issue"|"pr", "reason", "caption", "sentinel_reason"}`. `POST` answers 202 with the job (`{id, status: "running"|"done"|"failed", decision: {verdict, rule, text, rationale, actions, keeps_park}, notes, error}`); 409 with the running job while one is in flight; `GET` reads it. Rails: never merges, refuses items wearing `hold`, never grants secrets/permissions (`needs-owner` + `hold` instead), recommends only on hive policy/governance questions (keeps 085). Audited as `repo_item_decision_delegated[_failed]`. See [dashboard-sections.md](dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:329`, `pkg/dashboard/api_repo_item_decide_delegated.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:551`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:359`, `pkg/dashboard/api_acmm_eval.go` | +| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:360` | +| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](acmm-advisor.md) | `pkg/dashboard/api.go:361` | +| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`, and on the needs-human advice `why` (plain-language reasons with counts) and `help`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](advisory.md#owner-advice) | `pkg/dashboard/api.go:363`, `pkg/dashboard/api_hive_advice.go` | +| `POST` | `/api/repos/{owner}/{repo}/issues/{number}/release` | Owner or repo write/maintain/admin | Execute the typed `kind=release` owner step the hub proposed on a parked issue, with the same live preconditions as `/hive approve` ([#12635](https://github.com/hivecommons/hive/issues/12635)). Tag and target are resolved from the trusted owner-step, never from the request. Returns 503 when the GitHub client or config is unavailable; 409 when the repository is `label_driven` (the same refusal `/hive approve` gets there), when `release_on_approve` is off, or when a live precondition fails (`error` carries the refusal reason, which is also posted on the issue); 502 when the release write itself fails. Success returns `{ok, repo, number, tag, url, minStatusSeq}` plus `warning` if the follow-up issue comment failed after the release was created. | `pkg/dashboard/api.go:354`, `pkg/dashboard/api_issue_release.go` | ## Cost, tokens, telemetry | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/trends` | Dashboard auth/session | Trends | `pkg/dashboard/api.go:84` | -| `GET` | `/api/token-access` | Dashboard auth/session | Token Access | `pkg/dashboard/api.go:163` | -| `GET` | `/api/tokens` | Dashboard auth/session | Tokens | `pkg/dashboard/api.go:164` | -| `GET` | `/api/cost` | Dashboard auth/session | Cost; `merged_prs` / `closed_issues` come from the webhook-incremented ForgeState counters reconciled hourly against search, with `count_stale`, `count_stale_since` and `count_source` (`forgestate` / `search`) when last-good counts are being served (#13376), `count_partial` when only some configured repos have an attributed baseline yet, and `count_error` naming why the latest attributed refresh failed or was partial (the reconcile asks GitHub about a budgeted slice of repos per pass so a many-repo hive stays inside search's rate limit); `daily_rollup` lists per-UTC-day estimated spend (`day`, `cost_usd`, `input_tokens`, `output_tokens`, `samples`) kept for ~2 years so the Cost chart can reach past the 30-day snapshot ring; `recording` reports `agent_models`, `unrecorded_models`, `recent_models`, `last_priced_growth_at`, `last_agent_model_growth_at`, `stale_after_ms` and `stale` so the card can flag stale live usage recording (#13485) | `pkg/dashboard/api.go:165` | +| `GET` | `/api/trends` | Dashboard auth/session | Trends | `pkg/dashboard/api.go:89` | +| `GET` | `/api/token-access` | Dashboard auth/session | Token Access | `pkg/dashboard/api.go:168` | +| `GET` | `/api/tokens` | Dashboard auth/session | Tokens | `pkg/dashboard/api.go:169` | +| `GET` | `/api/cost` | Dashboard auth/session | Cost; `merged_prs` / `closed_issues` come from the webhook-incremented ForgeState counters reconciled hourly against search, with `count_stale`, `count_stale_since` and `count_source` (`forgestate` / `search`) when last-good counts are being served (#13376), `count_partial` when only some configured repos have an attributed baseline yet, and `count_error` naming why the latest attributed refresh failed or was partial (the reconcile asks GitHub about a budgeted slice of repos per pass so a many-repo hive stays inside search's rate limit); `daily_rollup` lists per-UTC-day estimated spend (`day`, `cost_usd`, `input_tokens`, `output_tokens`, `samples`) kept for ~2 years so the Cost chart can reach past the 30-day snapshot ring; `recording` reports `agent_models`, `unrecorded_models`, `recent_models`, `last_priced_growth_at`, `last_agent_model_growth_at`, `stale_after_ms` and `stale` so the card can flag stale live usage recording (#13485) | `pkg/dashboard/api.go:170` | | `GET` | `/api/repo-activity` | Dashboard auth/session | Per-repo audited activity | `pkg/dashboard/api.go` | | `GET` | `/api/repo-cost` | Dashboard auth/session | Per-repo estimated token cost (interval join, cached) | `pkg/dashboard/api.go` | | `GET` | `/api/launch-cost` | Dashboard auth/session | Per-issue / PR / head-SHA estimated token cost from kick launch attribution (cached; `?ref=owner/repo#N` narrows) | `pkg/dashboard/api.go` | -| `GET` | `/api/cost/history` | Dashboard auth/session | Cost History | `pkg/dashboard/api.go:170` | -| `GET` | `/api/trend/history` | Dashboard auth/session | Trend History; entries recorded since #13646 carry `yourTurnRootIssues` and `yourTurnRootPrs` | `pkg/dashboard/api.go:173` | -| `GET` | `/api/timeseries` | Dashboard auth/session | Time Series | `pkg/dashboard/api.go:175` | -| `GET` | `/api/providers/headroom` | Dashboard auth/session | Providers Headroom | `pkg/dashboard/api.go:61` | -| `GET` | `/api/budget/history` | Dashboard auth/session | Budget History | `pkg/dashboard/api.go:172` | +| `GET` | `/api/cost/history` | Dashboard auth/session | Cost History | `pkg/dashboard/api.go:175` | +| `GET` | `/api/trend/history` | Dashboard auth/session | Trend History; entries recorded since #13646 carry `yourTurnRootIssues` and `yourTurnRootPrs` | `pkg/dashboard/api.go:178` | +| `GET` | `/api/timeseries` | Dashboard auth/session | Time Series | `pkg/dashboard/api.go:180` | +| `GET` | `/api/providers/headroom` | Dashboard auth/session | Providers Headroom | `pkg/dashboard/api.go:66` | +| `GET` | `/api/budget/history` | Dashboard auth/session | Budget History | `pkg/dashboard/api.go:177` | ## Knowledge | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:372` | -| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:373` | -| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; 404 otherwise | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](public-knowledge-mcp.md)) | `pkg/dashboard/api.go:381` | -| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:374` | -| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](knowledge-toc.md)) | `pkg/dashboard/api.go:375` | -| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:376` | -| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](knowledge-toc.md)) | `pkg/dashboard/api.go:377` | -| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:382` | -| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:383` | -| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:384` | -| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:385` | -| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:386` | -| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:387` | -| `POST` | `/api/knowledge/create` | Dashboard auth/session | Knowledge Create | `pkg/dashboard/api.go:388` | -| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:389` | -| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:395` | -| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:396` | -| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:397` | -| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:398` | -| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:399` | -| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:400` | -| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:401` | -| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:402` | -| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:403` | -| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:404` | -| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:405` | -| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:406` | -| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:407` | -| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:408` | -| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:409` | -| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:410` | +| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:379` | +| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:380` | +| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; 404 otherwise | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](public-knowledge-mcp.md)) | `pkg/dashboard/api.go:388` | +| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:381` | +| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](knowledge-toc.md)) | `pkg/dashboard/api.go:382` | +| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:383` | +| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](knowledge-toc.md)) | `pkg/dashboard/api.go:384` | +| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:389` | +| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:390` | +| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:391` | +| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:392` | +| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:393` | +| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:394` | +| `POST` | `/api/knowledge/create` | Dashboard auth/session | Knowledge Create | `pkg/dashboard/api.go:395` | +| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:396` | +| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:402` | +| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:403` | +| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:404` | +| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:405` | +| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:406` | +| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:407` | +| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:408` | +| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:409` | +| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:410` | +| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:411` | +| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:412` | +| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:413` | +| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:414` | +| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:415` | +| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:416` | +| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:417` | | `GET` | `/api/config/knowledge/connectors` | Dashboard auth/session | Knowledge connectors config (name, type, layer, scope, env/file auth reference) and registered types | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/connectors` | Owner only | Replace `knowledge.connectors` after validation; inline secrets rejected | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/connectors/status` | Dashboard auth/session | Per-connector status pill, last sync, pages/facts, last error, plus the `publish` mirror status | `pkg/dashboard/api.go` | @@ -357,34 +361,34 @@ Read the result from `GET /api/kick/{agent}/status`, which returns `status` of ` | `POST` | `/api/config/knowledge/connectors/{name}/sync` | Owner only | Start an immediate background sync of one connector | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/agent-scopes` | Dashboard auth/session | Per-agent knowledge scopes (`knowledge.agent_scopes`: layers, repos, types, tags, include_states) ([docs](knowledge-toc.md#per-agent-scopes)) | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/agent-scopes` | Owner only | Replace `knowledge.agent_scopes` after validating layer and state names; persisted and audited as `config_knowledge_agent_scopes` | `pkg/dashboard/api.go` | -| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:420` | -| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:421` | -| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:422` | -| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:423` | -| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:424` | -| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:425` | -| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:426` | -| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:427` | -| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:428` | -| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:429` | -| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:430` | -| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:393` | -| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:394` | +| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:427` | +| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:428` | +| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:429` | +| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:430` | +| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:431` | +| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:432` | +| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:433` | +| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:434` | +| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:435` | +| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:436` | +| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:437` | +| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:400` | +| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:401` | | `GET` | `/api/config/knowledge/pipeline` | Owner only | Scalar bead synthesizer, curator, code map and primer settings, plus the read-only engine, vault paths and primer merge strategy ([docs](settings.md#knowledge)) | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/pipeline` | Owner only | Update those settings after `config.ValidateKnowledgePipeline`; 0 or empty restores the default; the bead synthesizer switch applies at once, the rest after a restart; audited as `config_knowledge_pipeline` | `pkg/dashboard/api.go` | -| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:420` | -| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:421` | -| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:422` | -| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:423` | -| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:424` | -| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:425` | -| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:426` | -| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:427` | -| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:428` | -| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:429` | -| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:430` | -| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:393` | -| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:394` | +| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:427` | +| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:428` | +| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:429` | +| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:430` | +| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:431` | +| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:432` | +| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:433` | +| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:434` | +| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:435` | +| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:436` | +| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:437` | +| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:400` | +| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:401` | | `GET` | `/api/knowledge/connectors` | Dashboard auth/session | Knowledge connector statuses and publish mirror status (`{"connectors": [...], "publish": {...}\|null}`, see [knowledge-connectors.md](knowledge-connectors.md)) | `pkg/dashboard/api_knowledge_connectors.go:23` | | `POST` | `/api/knowledge/connectors/{name}/sync` | Owner only | Start one connector sync now (202; 404 unknown, 409 running) | `pkg/dashboard/api_knowledge_connectors.go:24` | | `POST` | `/api/knowledge/publish/sync` | Owner only | Run a publish mirror batch now (202 with publish status; 503 mirror not running, 409 batch in progress) | `pkg/dashboard/api_knowledge_connectors.go:25` | @@ -393,50 +397,50 @@ Read the result from `GET /api/kick/{agent}/status`, which returns `status` of ` | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/contribute` | Public | Contribute Landing | `pkg/dashboard/api_contribute.go:119` | -| `GET` | `/contribute/{tab}` | Public | Contribute Landing | `pkg/dashboard/api_contribute.go:125` | -| `GET` | `/api/contribute/ws` | Public | s.contribute Hub.Handle WS | `pkg/dashboard/api_contribute.go:133` | -| `POST` | `/api/contribute/register` | Public | Contribute Register | `pkg/dashboard/api_contribute.go:134` | -| `POST` | `/api/contribute/invite` | Public | Contribute Invite | `pkg/dashboard/api_contribute.go:140` | -| `POST` | `/api/contribute/reissue-token` | Public | Contribute Reissue Token | `pkg/dashboard/api_contribute.go:141` | -| `GET` | `/api/contribute/status` | Public | Contribute Status | `pkg/dashboard/api_contribute.go:142` | -| `GET` | `/api/contribute/repositories` | Public | Contribution-eligible repositories for a contributor `repo_scope`: `{repositories:[{id,forge,host,full_name}],generated_at}`, the monitored repos minus `hub.disabled_repos`; `id` is `{forge}/{host}/{owner}/{name}` lower-cased ([#12805](https://github.com/hivecommons/hive/issues/12805)) | `pkg/dashboard/api_contribute.go:145` | -| `GET` | `/api/contribute/activity` | Public | Contribute Activity | `pkg/dashboard/api_contribute.go:149` | -| `GET` | `/api/contribute/fleet` | Public | Contribute Fleet | `pkg/dashboard/api_contribute.go:150` | -| `GET` | `/api/contribute/events` | Public | Contribute Events (SSE). Every frame carries a monotonic `seq`; a connection whose channel filled gets a `gap` frame naming how many events it missed, so a long-lived client can tell "nothing happened" from "I missed events" ([#6218](https://github.com/hivecommons/hive/issues/6218)) | `pkg/dashboard/api_contribute.go:154` | -| `GET` | `/api/contribute/queue` | Public | Contribute Queue | `pkg/dashboard/api_contribute.go:157` | -| `GET` | `/api/contribute/opportunistic` | Public | Contribute Opportunistic | `pkg/dashboard/api_contribute.go:161` | -| `GET` | `/api/contribute/limits` | Public | Contribute Limits | `pkg/dashboard/api_contribute.go:166` | -| `GET` | `/api/contribute/metrics` | Public | Contribute Metrics | `pkg/dashboard/api_contribute.go:172` | -| `GET` | `/api/contribute/me` | Public path; caller identity resolved server-side (401 anonymous; a signed-in caller with no profile on this hive gets 200 with `has_profile:false` and no counters, [#12111](https://github.com/hivecommons/hive/issues/12111)) | The caller's OWN contribution stats — issues worked (24h and all-time), completions that produced a verified PR (24h and all-time, [#7894](https://github.com/hivecommons/hive/issues/7894)), and failures. No username parameter, so it only ever answers for its caller ([#6543](https://github.com/hivecommons/hive/issues/6543)) | `pkg/dashboard/api_contribute.go:182` | +| `GET` | `/contribute` | Public | Contribute Landing | `pkg/dashboard/api_contribute.go:125` | +| `GET` | `/contribute/{tab}` | Public | Contribute Landing | `pkg/dashboard/api_contribute.go:131` | +| `GET` | `/api/contribute/ws` | Public | s.contribute Hub.Handle WS | `pkg/dashboard/api_contribute.go:139` | +| `POST` | `/api/contribute/register` | Public | Contribute Register | `pkg/dashboard/api_contribute.go:140` | +| `POST` | `/api/contribute/invite` | Public | Contribute Invite | `pkg/dashboard/api_contribute.go:146` | +| `POST` | `/api/contribute/reissue-token` | Public | Contribute Reissue Token | `pkg/dashboard/api_contribute.go:147` | +| `GET` | `/api/contribute/status` | Public | Contribute Status | `pkg/dashboard/api_contribute.go:149` | +| `GET` | `/api/contribute/repositories` | Public | Contribution-eligible repositories for a contributor `repo_scope`: `{repositories:[{id,forge,host,full_name}],generated_at}`, the monitored repos minus `hub.disabled_repos`; `id` is `{forge}/{host}/{owner}/{name}` lower-cased ([#12805](https://github.com/hivecommons/hive/issues/12805)) | `pkg/dashboard/api_contribute.go:152` | +| `GET` | `/api/contribute/activity` | Public | Contribute Activity | `pkg/dashboard/api_contribute.go:156` | +| `GET` | `/api/contribute/fleet` | Public | Contribute Fleet | `pkg/dashboard/api_contribute.go:157` | +| `GET` | `/api/contribute/events` | Public | Contribute Events (SSE). Every frame carries a monotonic `seq`; a connection whose channel filled gets a `gap` frame naming how many events it missed, so a long-lived client can tell "nothing happened" from "I missed events" ([#6218](https://github.com/hivecommons/hive/issues/6218)) | `pkg/dashboard/api_contribute.go:161` | +| `GET` | `/api/contribute/queue` | Public | Contribute Queue | `pkg/dashboard/api_contribute.go:164` | +| `GET` | `/api/contribute/opportunistic` | Public | Contribute Opportunistic | `pkg/dashboard/api_contribute.go:168` | +| `GET` | `/api/contribute/limits` | Public | Contribute Limits | `pkg/dashboard/api_contribute.go:173` | +| `GET` | `/api/contribute/metrics` | Public | Contribute Metrics | `pkg/dashboard/api_contribute.go:179` | +| `GET` | `/api/contribute/me` | Public path; caller identity resolved server-side (401 anonymous; a signed-in caller with no profile on this hive gets 200 with `has_profile:false` and no counters, [#12111](https://github.com/hivecommons/hive/issues/12111)) | The caller's OWN contribution stats — issues worked (24h and all-time), completions that produced a verified PR (24h and all-time, [#7894](https://github.com/hivecommons/hive/issues/7894)), and failures. No username parameter, so it only ever answers for its caller ([#6543](https://github.com/hivecommons/hive/issues/6543)) | `pkg/dashboard/api_contribute.go:189` | | `GET` | `/api/contribute/run-stats/models` | Public | Per-model run evidence for the contributor wall: runs, verified-PR share, and failure rate over the selected window. | `pkg/dashboard/api_contribute.go` | | `GET` | `/api/contribute/wall` | Public | Contributor wall posts when the hive-wide wall switch is enabled; anonymous readers see visible posts, authors can still see their own hidden posts. | `pkg/dashboard/api_contribute.go` | | `POST` | `/api/contribute/wall` | Signed-in contributor | Create a 500-character plain-text wall post or one-level reply, with optional `model`, `backend`, and `repo` tags. Returns 401 anonymous, 403 when disabled/revoked/muted, and 429 when rate-limited. | `pkg/dashboard/api_contribute.go` | | `DELETE` | `/api/contribute/wall/{id}` | Post author | Author deletes their own wall post; action is audit logged. | `pkg/dashboard/api_contribute.go` | | `POST` | `/api/contribute/wall/{id}/hide` | Owner/read-write | Hide a wall post from public readers while leaving it visible to the author; action is audit logged and pushed over `/api/contribute/events`. | `pkg/dashboard/api_contribute.go` | | `POST` | `/api/contribute/wall/{id}/flag` | Signed-in contributor | Flag a wall post for Management review; action is audit logged. | `pkg/dashboard/api_contribute.go` | -| `GET` | `/api/contribute/triage` | Public | Contribute Triage | `pkg/dashboard/api_contribute.go:208` | -| `PUT` | `/api/contribute/queue/order` | Public | Contribute Queue Order | `pkg/dashboard/api_contribute.go:212` | -| `POST` | `/api/contribute/queue/hold` | Public | Contribute Queue Hold | `pkg/dashboard/api_contribute.go:217` | -| `POST` | `/api/contribute/queue/hold/clear` | Public | Contribute Queue Hold Clear | `pkg/dashboard/api_contribute.go:221` | -| `GET` | `/api/contribute/interests` | Public | Contribute Interests | `pkg/dashboard/api_contribute.go:234` | -| `PUT` | `/api/contribute/interests` | Public | Contribute Interests | `pkg/dashboard/api_contribute.go:235` | -| `GET` | `/api/contributors` | Dashboard auth/session | Contributors List | `pkg/dashboard/api_contribute.go:241` | -| `GET` | `/api/contributors/{id}` | Dashboard auth/session | Contributor Get | `pkg/dashboard/api_contribute.go:242` | +| `GET` | `/api/contribute/triage` | Public | Contribute Triage | `pkg/dashboard/api_contribute.go:215` | +| `PUT` | `/api/contribute/queue/order` | Public | Contribute Queue Order | `pkg/dashboard/api_contribute.go:219` | +| `POST` | `/api/contribute/queue/hold` | Public | Contribute Queue Hold | `pkg/dashboard/api_contribute.go:224` | +| `POST` | `/api/contribute/queue/hold/clear` | Public | Contribute Queue Hold Clear | `pkg/dashboard/api_contribute.go:228` | +| `GET` | `/api/contribute/interests` | Public | Contribute Interests | `pkg/dashboard/api_contribute.go:241` | +| `PUT` | `/api/contribute/interests` | Public | Contribute Interests | `pkg/dashboard/api_contribute.go:242` | +| `GET` | `/api/contributors` | Dashboard auth/session | Contributors List | `pkg/dashboard/api_contribute.go:250` | +| `GET` | `/api/contributors/{id}` | Dashboard auth/session | Contributor Get | `pkg/dashboard/api_contribute.go:251` | | `POST` | `/api/contributors/{id}/wall-mute` | Owner only | Mute or unmute a contributor from posting to the contributor wall; action is audit logged. | `pkg/dashboard/api_contribute.go` | -| `PUT` | `/api/contributors/{id}/trust` | Owner only | Contributor Trust | `pkg/dashboard/api_contribute.go:244` | -| `PUT` | `/api/contributors/{id}/agent-role` | Owner only | Contributor Agent Role | `pkg/dashboard/api_contribute.go:245` | -| `PUT` | `/api/contributors/{id}/agent-role-grants` | Owner only | Contributor Agent Role Grants | `pkg/dashboard/api_contribute.go:246` | -| `POST` | `/api/contributors/{id}/revoke` | Owner only | Contributor Revoke | `pkg/dashboard/api_contribute.go:247` | -| `POST` | `/api/contributors/{id}/requeue` | Dashboard auth/session | Contributor Requeue | `pkg/dashboard/api_contribute.go:248` | -| `DELETE` | `/api/contributors/{id}` | Owner only | Contributor Delete | `pkg/dashboard/api_contribute.go:249` | -| `GET` | `/contribute/dossier/{username}` | Public | Contributor Dossier Page (HTML) | `pkg/dashboard/api_contribute.go:132` | -| `GET` | `/api/contribute/run-stats` | Public | Contribute Run Stats | `pkg/dashboard/api_contribute.go:195` | -| `GET` | `/api/contribute/runs` | Public | Contribute Per-Run History | `pkg/dashboard/api_contribute.go:290` | -| `GET` | `/api/contribute/decisions` | Owner/read-write | Contribute Hub Decisions | `pkg/dashboard/api_contribute.go:300` | -| `GET` | `/api/contribute/dossier` | Public | Contribute Dossier Get | `pkg/dashboard/api_contribute.go:239` | -| `POST` | `/api/contribute/dossier` | Public | Contribute Dossier Update | `pkg/dashboard/api_contribute.go:240` | -| `GET` | `/api/leaderboard/contributor/{username}/heraldry` | Public | Contributor Heraldry | `pkg/dashboard/api_contribute.go:270` | +| `PUT` | `/api/contributors/{id}/trust` | Owner only | Contributor Trust | `pkg/dashboard/api_contribute.go:253` | +| `PUT` | `/api/contributors/{id}/agent-role` | Owner only | Contributor Agent Role | `pkg/dashboard/api_contribute.go:254` | +| `PUT` | `/api/contributors/{id}/agent-role-grants` | Owner only | Contributor Agent Role Grants | `pkg/dashboard/api_contribute.go:255` | +| `POST` | `/api/contributors/{id}/revoke` | Owner only | Contributor Revoke | `pkg/dashboard/api_contribute.go:256` | +| `POST` | `/api/contributors/{id}/requeue` | Dashboard auth/session | Contributor Requeue | `pkg/dashboard/api_contribute.go:257` | +| `DELETE` | `/api/contributors/{id}` | Owner only | Contributor Delete | `pkg/dashboard/api_contribute.go:258` | +| `GET` | `/contribute/dossier/{username}` | Public | Contributor Dossier Page (HTML) | `pkg/dashboard/api_contribute.go:138` | +| `GET` | `/api/contribute/run-stats` | Public | Contribute Run Stats | `pkg/dashboard/api_contribute.go:202` | +| `GET` | `/api/contribute/runs` | Public | Contribute Per-Run History | `pkg/dashboard/api_contribute.go:299` | +| `GET` | `/api/contribute/decisions` | Owner/read-write | Contribute Hub Decisions | `pkg/dashboard/api_contribute.go:309` | +| `GET` | `/api/contribute/dossier` | Public | Contribute Dossier Get | `pkg/dashboard/api_contribute.go:246` | +| `POST` | `/api/contribute/dossier` | Public | Contribute Dossier Update | `pkg/dashboard/api_contribute.go:247` | +| `GET` | `/api/leaderboard/contributor/{username}/heraldry` | Public | Contributor Heraldry | `pkg/dashboard/api_contribute.go:279` | | `GET` | `/api/cards/player/{asset}` | Public | OpenAPI alias for a cacheable contributor social-card SVG (`asset` is `.svg`). | `pkg/dashboard/social_cards.go:112` | | `GET` | `/api/cards/achievement/{login}/{asset}` | Public | OpenAPI alias for a cacheable attained-achievement SVG (`asset` is `.svg`). | `pkg/dashboard/social_cards.go:113` | | `GET` | `/api/cards/leaderboard/{asset}` | Public | OpenAPI alias for a cacheable leaderboard-summary SVG (`contributors.svg`, `teams.svg`, or `swarm.svg`). | `pkg/dashboard/social_cards.go:114` | @@ -446,10 +450,10 @@ Read the result from `GET /api/kick/{agent}/status`, which returns `status` of ` | `GET` | `/share/player/{login}` | Public | Share landing page with Open Graph/Twitter metadata for a contributor card and link to the public dossier. | `pkg/dashboard/social_cards.go:115` | | `GET` | `/share/achievement/{login}/{achievementID}` | Public | Share landing page with metadata for an unlocked achievement card and link to the public dossier. | `pkg/dashboard/social_cards.go:116` | | `GET` | `/share/leaderboard/{board}` | Public | Share landing page with metadata for a public leaderboard card and link to live standings. | `pkg/dashboard/social_cards.go:117` | -| `PUT` | `/api/contribute/help-links` | Owner/read-write | Contribute Help Links | `pkg/dashboard/api_contribute.go:148` | -| `GET` | `/api/contribute/operators/message` | Public path; caller identity resolved server-side (401 anonymous, 403 without a profile) | Returns only the signed-in contributor's unacknowledged operator messages; no username/body field can select another recipient ([#8461](https://github.com/hivecommons/hive/issues/8461)). | `pkg/dashboard/api_contribute.go:188` | -| `POST` | `/api/contribute/operators/message` | Owner/read-write | Sends a sanitized, length-capped one-to-one operator note to a contributor profile, persists it until acknowledgement, and pushes it to matching live relay sockets only ([#8461](https://github.com/hivecommons/hive/issues/8461)). | `pkg/dashboard/api_contribute.go:189` | -| `POST` | `/api/contribute/operators/message/ack` | Public path; caller identity resolved server-side | Acknowledges the signed-in contributor's own message, optionally recording a short sanitized reply for operators. | `pkg/dashboard/api_contribute.go:190` | +| `PUT` | `/api/contribute/help-links` | Owner/read-write | Contribute Help Links | `pkg/dashboard/api_contribute.go:155` | +| `GET` | `/api/contribute/operators/message` | Public path; caller identity resolved server-side (401 anonymous, 403 without a profile) | Returns only the signed-in contributor's unacknowledged operator messages; no username/body field can select another recipient ([#8461](https://github.com/hivecommons/hive/issues/8461)). | `pkg/dashboard/api_contribute.go:195` | +| `POST` | `/api/contribute/operators/message` | Owner/read-write | Sends a sanitized, length-capped one-to-one operator note to a contributor profile, persists it until acknowledgement, and pushes it to matching live relay sockets only ([#8461](https://github.com/hivecommons/hive/issues/8461)). | `pkg/dashboard/api_contribute.go:196` | +| `POST` | `/api/contribute/operators/message/ack` | Public path; caller identity resolved server-side | Acknowledges the signed-in contributor's own message, optionally recording a short sanitized reply for operators. | `pkg/dashboard/api_contribute.go:197` | ### Issue claims (hivecommons/hive#8380) @@ -496,74 +500,117 @@ always resolved server-side from the validated token. | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status | `pkg/dashboard/api.go:465` | -| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:466` | -| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:467` | -| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:468` | -| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:469` | -| `POST` | `/api/nous/promote` | Owner only | Nous Promote (manual verdict promotion to an issue or Spektacular run; see [experiment-lab.md](experiment-lab.md#verdict-card-and-manual-promote)) | `pkg/dashboard/api.go:470` | -| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:471` | -| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:472` | -| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:473` | -| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:474` | -| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:475` | -| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:476` | -| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:477` | -| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:478` | -| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals | `pkg/dashboard/api.go:479` | -| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos | `pkg/dashboard/api.go:480` | -| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:481` | -| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:482` | -| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:483` | -| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:484` | -| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:485` | -| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:486` | -| `POST` | `/api/nous/artifacts/sync` | Owner only | Nous Artifact Sync | `pkg/dashboard/api.go:487` | -| `POST` | `/api/nous/reset` | Owner only | Nous Reset (wipe baseline snapshots, learned knowledge or everything; see [experiment-lab.md](experiment-lab.md#resetting-data)) | `pkg/dashboard/api.go:488` | -| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:489` | -| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:490` | -| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:491` | -| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:492` | +| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:472` | +| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:473` | +| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:474` | +| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:475` | +| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:476` | +| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:478` | +| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:479` | +| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:480` | +| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:481` | +| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:482` | +| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:483` | +| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:484` | +| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:485` | +| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:486` | +| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:487` | +| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:488` | +| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:489` | +| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:490` | +| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:491` | +| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:492` | +| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:493` | +| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:496` | +| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:497` | +| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:498` | +| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:499` | +| `POST` | `/api/nous/promote` | Owner only | Nous Promote (manual verdict promotion to an issue or Spektacular run; see [experiment-lab.md](experiment-lab.md#verdict-card-and-manual-promote)) | `pkg/dashboard/api.go:477` | +| `POST` | `/api/nous/artifacts/sync` | Owner only | Nous Artifact Sync | `pkg/dashboard/api.go:494` | +| `POST` | `/api/nous/reset` | Owner only | Nous Reset (wipe baseline snapshots, learned knowledge or everything; see [experiment-lab.md](experiment-lab.md#resetting-data)) | `pkg/dashboard/api.go:495` | ## Inception | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:435` | -| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:436` | -| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:437` | -| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:438` | -| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:439` | -| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:440` | -| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:441` | -| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:442` | -| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:443` | -| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:444` | -| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:445` | -| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:446` | -| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:447` | -| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:448` | +| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:442` | +| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:443` | +| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:444` | +| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:445` | +| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:446` | +| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:447` | +| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:448` | +| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:449` | +| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:450` | +| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:451` | +| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:452` | +| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:453` | +| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:454` | +| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:455` | ## Beads | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:494` | -| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:495` | -| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:496` | -| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:497` | -| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:498` | +| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:501` | +| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:502` | +| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:503` | +| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:504` | +| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:505` | ## Dashboard miscellaneous | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/audit` | Read-write role | Audit Log — `{"entries": [...]}` envelope, newest first, capped at 200; response shape and the serve-time `user_name` field in [audit-log.md](audit-log.md#get-apiaudit) | `pkg/dashboard/api.go:54` | -| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:505` | +| `GET` | `/api/audit` | Read-write role | Audit Log — `{"entries": [...]}` envelope, newest first, capped at 200; response shape and the serve-time `user_name` field in [audit-log.md](audit-log.md#get-apiaudit) | `pkg/dashboard/api.go:59` | +| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:512` | +| `POST` | `/api/presence` | Dashboard auth/session | Presence | `pkg/dashboard/api.go:68` | +| `GET` | `/api/prompt-history` | Dashboard auth/session | Prompt History | `pkg/dashboard/api.go:69` | +| `POST` | `/api/self-upgrade` | Owner only | Self Upgrade | `pkg/dashboard/api.go:70` | +| `GET` | `/api/backup/status` | Owner only | Backup Status — `available:false` with a reason when no encryption key is configured | `pkg/dashboard/api.go:74` | +| `POST` | `/api/backup` | Owner only | Backup Download — `412` when no encryption key is configured (never an unencrypted archive) | `pkg/dashboard/api.go:75` | +| `POST` | `/api/banner-dismissed` | Dashboard auth/session | Banner Dismissed | `pkg/dashboard/api.go:76` | +| `GET` | `/api/history` | Dashboard auth/session | History | `pkg/dashboard/api.go:88` | +| `GET` | `/api/timeline` | Dashboard auth/session | Timeline | `pkg/dashboard/api.go:90` | +| `GET` | `/api/lifecycle-timeline` | Dashboard auth/session | Issue→PR lifecycle journeys plus derived stage timeline. Each journey keeps the dense `stages` map and adds a backwards-compatible `details` object for expandable rows: chronological stages with durations, current-stage age, agent, classification attrs, issue/PR links, blocked reason, and kick/run identifiers when recorded. | `pkg/dashboard/api.go:91` | +| `GET` | `/api/pr-throughput` | Dashboard auth/session | Throughput: pull/merge requests and issues across tracked forges. Counts hive-created PRs/issues, hive/observed reviews, observed PR merges/closures, and issue closes over `?hours=N` (default 24, capped at audit retention; `hours=0` = all time from durable counters in `/data/pr-throughput-counters.json`) with optional exact case-insensitive `?repo=owner/name` and `?role=created\|reviewed\|merged` for the actor trend. The response keeps legacy `opened`/`merged`/`closed`, `merged_by_path`, `buckets`, velocity metrics, previous-window trends, top agents/repos/reasons, and `recorded_since`, and adds `by_actor` (`pr`/`issue` × `created`/`reviewed`/`merged`/`closed` × `hive`/`human`/`other`) plus `series` points (`t`, `hive`, `human`, `other`) for the selected role. | `pkg/dashboard/api.go:92` | + + +| `GET` | `/api/convergence/soak` | Owner only | Convergence Soak Status | `pkg/dashboard/api.go:270` | +| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:460` | +| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:463` | +| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:464` | +| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:466` | +| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:467` | +| `POST` | `/api/linear/agent/install` | Owner only | Linear Agent Install (OAuth start) | `pkg/dashboard/api_linear_agent.go:219` | +| `GET` | `/api/linear/agent/status` | Owner only | Linear Agent Status | `pkg/dashboard/api_linear_agent.go:220` | +| `POST` | `/api/linear/agent/disconnect` | Owner only | Linear Agent Disconnect | `pkg/dashboard/api_linear_agent.go:221` | +| `GET` | `/api/upstream-watch` | Owner only | Upstream-watch divergence: per watched fork the upstream followed, the watermark and last-run time, surfaced/ported/dismissed/skipped counts, and the recent upstream refs with their upstream URL, diff URL, fork issue number and state. Read-only — it reads the config and the watch state file, never GitHub. Also carries `interval`, `project_repos` and each repo's configured entry (`config`) for the editor. | `pkg/dashboard/api.go:313` | +| `PUT` | `/api/config/upstream-watch` | Owner only | Upstream Watch Config Update: edit `upstream_watch` from the Features tab. Body `{enabled?, interval?, repos?: {: {upstream?, sources?, pr_labels?, max_issues_per_run?, label?} \| null}}` — absent keys are unchanged, `null` removes a repo, keys are stored as the bare repo name. Validated with the loader's rules before the config changes (400 on failure), then persisted and audited as `config_upstream_watch`. | `pkg/dashboard/api.go:314` | + +| `GET` | `/api/widget` | Dashboard auth/session | Widget | `pkg/dashboard/api.go:125` | +| `GET` | `/api/pane/{agent}` | Dashboard auth/session | Pane | `pkg/dashboard/api.go:130` | +| `GET` | `/api/role` | Dashboard auth/session | Role | `pkg/dashboard/api.go:144` | +| `POST` | `/api/reset-restarts/{agent}` | Owner only | Reset Restarts | `pkg/dashboard/api.go:166` | +| `GET` | `/api/budget-ignore` | Dashboard auth/session | Budget Ignore Get | `pkg/dashboard/api.go:183` | +| `POST` | `/api/budget-ignore` | Owner only | Budget Ignore Set | `pkg/dashboard/api.go:184` | +| `GET` | `/api/summaries` | Dashboard auth/session | Summaries | `pkg/dashboard/api.go:196` | +| `GET` | `/api/stage-tracker` | Dashboard auth/session | Stage Tracker: every open issue and pull request of the watched repos for the Hive activity monitor — actionable, held, claimed, in-progress, covered-by-PR, filtered (`needs-*`), draft and exempt-labelled items alike, each with `stage`, `stage_since`, `labels`, `created_at`, `author_kind`, `modifiers`, (PRs) `sentinel_alert` / `sentinel_findings` / `sentinel_reason` when the sentinel flagged the PR, `finisher` / `finisher_verb` / `finisher_reason` (who can finish the item, from the your-turn decision: `hive`, `person` when it is the operator's turn, `elsewhere` when it waits on someone else (a hold, a covering PR, a reporter, an open blocker, or its author on a returned PR or a person's draft), absent when the snapshot has no decision; also on each `linked_prs` entry), and (issues) `linked_prs`; items with no stage yet report stage `0` "Open"; `needs_human_labels` is the "a person must act" label set behind the monitor's Needs human facet (the same list as `/api/status.labelPresets[needs-human]`). Standing meta issues are omitted from the monitor but still counted in the Overview breakdown; `recently_merged` lists the PRs (`repo`, `number`, `merged_at`) whose `pr_merged` audit entry is under 30 minutes old, so a client diffing two payloads can tell a merge from a close. Served from a server-side cache rebuilt at most every `dashboard.stage_tracker.cache_ttl_s` (default `governor.eval_interval_s`) from the spoke's existing snapshot — a request never calls GitHub. Carries `generated_at`, `source` (`cache` or `fresh`), `cache_ttl_s`, an `ETag` and `Cache-Control: private, max-age=`; `If-None-Match` yields 304. Each hit is recorded under REST caller `dashboard:stage-tracker` so `/api/gh-rate-limits` shows it charged nothing. | `pkg/dashboard/api.go:197` | +| `POST` | `/api/prs/{owner}/{repo}/{number}/queue-automerge` | Merger/owner role | Queue PRAuto Merge | `pkg/dashboard/api.go:198` | +| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:377` | +| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:439` | +| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:440` | +| `POST` | `/api/chat` | Dashboard auth/session | Chat. JSON body `{"query": "", "history": [...]}`; `message` is accepted as an alias for `query` (`query` wins when both are set). A missing or blank field returns 400 `query is required`. `!`-prefixed commands go to the chat spine (`{"accepted":true,"seq":N}`, read-write); while an inception is in `clarify` with unanswered questions, an owner's unprefixed reply also goes to the spine, which submits it as the answer to the next unanswered question. | `pkg/dashboard/api.go:469` | +| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:507` | +| `GET` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:260` | +| `POST` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:261` | +| `GET` | `/api/docs` | Dashboard auth/session | APIDocs | `pkg/dashboard/api_contribute.go:262` | +| `GET` | `/leaderboard` | Public | Leaderboard Page | `pkg/dashboard/api_contribute.go:264` | +| `GET` | `/api/leaderboard` | Public | Leaderboard API | `pkg/dashboard/api_contribute.go:265` | +| `GET` | `/api/leaderboard/teams` | Public | Team Leaderboards | `pkg/dashboard/api_contribute.go:266` | +| `GET` | `/api/leaderboard/style` | Public | Leaderboard Style | `pkg/dashboard/api_contribute.go:267` | | `GET` | `/api/logs/sources` | Dashboard auth/session | Log sources (#11942): every source name this hive knows (`hive`, the default, plus `proxy`, `hive-output`, `litellm` and `requests` from #11944, and `starts` and `crash` from #11945), each marked `available` or not with a `reason`; names only, no log text. Wrapped by the admin MCP `log_sources` tool; see [design/admin-mcp.md](design/admin-mcp.md#log-read) | `pkg/dashboard/api_logs.go:19` | | `GET` | `/api/logs/read` | Read-write role (checked by the handler) | Log read (#11942): the newest matching lines of the hive's own log and its rotated, compressed history, or of another source (#11944: auth proxy output, early hive output, litellm output, dashboard request log), oldest first, as `{time, source, level, stream, text}`; `?source=&lines=&since=&until=&min_level=&include=&exclude=&cursor=`. Default 100 lines, at most 1000, 128 KiB per result, 8 KiB per line; `since`/`until` need an explicit offset; scrubbed at read time with redactions counted; `coverage`, `next_cursor`, `status` `incomplete` (stopped on the scan bound; follow `next_cursor`) / `not_available` / `cursor_expired`. Wrapped by the admin MCP `log_read` tool; see [design/admin-mcp.md](design/admin-mcp.md#log-read) | `pkg/dashboard/api_logs.go:20` | -| `POST` | `/api/presence` | Dashboard auth/session | Presence | `pkg/dashboard/api.go:63` | -| `GET` | `/api/prompt-history` | Dashboard auth/session | Prompt History | `pkg/dashboard/api.go:64` | | `GET` | `/api/diag/agents/{name}/log` | Dashboard auth/session (role of the wrapped endpoint) | Diagnostic read (#11943): wraps `GET /api/agents/{name}/log` as `{text, content_type, truncated, bytes_total, ...}` — the newest `?max_bytes=` of the scrubbed log (default 64 KiB, at most 128 KiB, clamping reported); `?explain=` passed through. Wrapped by the admin MCP `agent_log_read` tool; see [admin-mcp.md](admin-mcp.md#diagnostic-reads) | `pkg/dashboard/api_diag.go:41` | | `GET` | `/api/diag/agents/{name}/kicks` | Dashboard auth/session (role of the wrapped endpoint) | Diagnostic read (#11943): wraps `GET /api/agents/{name}/kicks`, scrubbed, newest first, oldest entries dropped (and reported in `trimmed`) to stay under 128 KiB. Wrapped by `agent_kicks_list`; see [admin-mcp.md](admin-mcp.md#diagnostic-reads) | `pkg/dashboard/api_diag.go:42` | | `GET` | `/api/diag/agents/{name}/kicks/{id}` | Dashboard auth/session (role of the wrapped endpoint) | Diagnostic read (#11943): wraps `GET /api/agents/{name}/kicks/{id}` in the same text shape and bounds as the agent log. Wrapped by `agent_kick_read`; see [admin-mcp.md](admin-mcp.md#diagnostic-reads) | `pkg/dashboard/api_diag.go:43` | @@ -572,57 +619,17 @@ always resolved server-side from the validated token. | `GET` | `/api/diag/health` | Public (as `/api/health/deep`) | Diagnostic read (#11943): wraps `GET /api/health/deep`, scrubbed and returned whole. Wrapped by `health_deep_read`; see [admin-mcp.md](admin-mcp.md#diagnostic-reads) | `pkg/dashboard/api_diag.go:46` | | `GET` | `/api/diag/prompt-history` | Read-write role (checked by the wrapped endpoint) | Diagnostic read (#11943): wraps `GET /api/prompt-history` (`?agent=&windowHours=` passed through); every entry scrubbed, each prompt cut at 8 KiB (`cut`, `full_bytes`), newest entries kept within 128 KiB with `truncated` and `entries_total`. Wrapped by `prompt_history_read`; see [admin-mcp.md](admin-mcp.md#diagnostic-reads) | `pkg/dashboard/api_diag.go:47` | | `GET` | `/api/diag/config` | Owner only, verified by this hive (dashboard token or verified owner session) | Diagnostic read (#11943): the effective configuration as `GET /api/config/provenance` computes it — every key and environment name with the layer that set it (`also_set_by` for overridden layers, rejected overlay and last-good fallback in `provenance`); values only for keys and names on the documented `value_list`, all others set/not set; shown values scrubbed; the file itself never returned; `?offset=` pages keys. Wrapped by `config_effective_read`; see [admin-mcp.md](admin-mcp.md#diagnostic-reads) | `pkg/dashboard/api_diag.go:48` | -| `POST` | `/api/self-upgrade` | Owner only | Self Upgrade | `pkg/dashboard/api.go:65` | -| `GET` | `/api/backup/status` | Owner only | Backup Status — `available:false` with a reason when no encryption key is configured | `pkg/dashboard/api.go:69` | -| `POST` | `/api/backup` | Owner only | Backup Download — `412` when no encryption key is configured (never an unencrypted archive) | `pkg/dashboard/api.go:70` | -| `POST` | `/api/banner-dismissed` | Dashboard auth/session | Banner Dismissed | `pkg/dashboard/api.go:71` | -| `GET` | `/api/history` | Dashboard auth/session | History | `pkg/dashboard/api.go:83` | -| `GET` | `/api/timeline` | Dashboard auth/session | Timeline | `pkg/dashboard/api.go:85` | -| `GET` | `/api/lifecycle-timeline` | Dashboard auth/session | Issue→PR lifecycle journeys plus derived stage timeline. Each journey keeps the dense `stages` map and adds a backwards-compatible `details` object for expandable rows: chronological stages with durations, current-stage age, agent, classification attrs, issue/PR links, blocked reason, and kick/run identifiers when recorded. | `pkg/dashboard/api.go:86` | -| `GET` | `/api/pr-throughput` | Dashboard auth/session | Throughput: pull/merge requests and issues across tracked forges. Counts hive-created PRs/issues, hive/observed reviews, observed PR merges/closures, and issue closes over `?hours=N` (default 24, capped at audit retention; `hours=0` = all time from durable counters in `/data/pr-throughput-counters.json`) with optional exact case-insensitive `?repo=owner/name` and `?role=created\|reviewed\|merged` for the actor trend. The response keeps legacy `opened`/`merged`/`closed`, `merged_by_path`, `buckets`, velocity metrics, previous-window trends, top agents/repos/reasons, and `recorded_since`, and adds `by_actor` (`pr`/`issue` × `created`/`reviewed`/`merged`/`closed` × `hive`/`human`/`other`) plus `series` points (`t`, `hive`, `human`, `other`) for the selected role. | `pkg/dashboard/api.go:87` | -| `GET` | `/api/convergence/soak` | Owner only | Convergence Soak Status | `pkg/dashboard/api.go:265` | -| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:453` | -| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:456` | -| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:457` | -| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:459` | -| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:460` | -| `POST` | `/api/linear/agent/install` | Owner only | Linear Agent Install (OAuth start) | `pkg/dashboard/api_linear_agent.go:219` | -| `GET` | `/api/linear/agent/status` | Owner only | Linear Agent Status | `pkg/dashboard/api_linear_agent.go:220` | -| `POST` | `/api/linear/agent/disconnect` | Owner only | Linear Agent Disconnect | `pkg/dashboard/api_linear_agent.go:221` | -| `GET` | `/api/upstream-watch` | Owner only | Upstream-watch divergence: per watched fork the upstream followed, the watermark and last-run time, surfaced/ported/dismissed/skipped counts, and the recent upstream refs with their upstream URL, diff URL, fork issue number and state. Read-only — it reads the config and the watch state file, never GitHub. Also carries `interval`, `project_repos` and each repo's configured entry (`config`) for the editor. | `pkg/dashboard/api.go:308` | -| `PUT` | `/api/config/upstream-watch` | Owner only | Upstream Watch Config Update: edit `upstream_watch` from the Features tab. Body `{enabled?, interval?, repos?: {: {upstream?, sources?, pr_labels?, max_issues_per_run?, label?} \| null}}` — absent keys are unchanged, `null` removes a repo, keys are stored as the bare repo name. Validated with the loader's rules before the config changes (400 on failure), then persisted and audited as `config_upstream_watch`. | `pkg/dashboard/api.go:309` | -| `GET` | `/api/widget` | Dashboard auth/session | Widget | `pkg/dashboard/api.go:120` | -| `GET` | `/api/pane/{agent}` | Dashboard auth/session | Pane | `pkg/dashboard/api.go:125` | -| `GET` | `/api/role` | Dashboard auth/session | Role | `pkg/dashboard/api.go:139` | -| `POST` | `/api/reset-restarts/{agent}` | Owner only | Reset Restarts | `pkg/dashboard/api.go:161` | -| `GET` | `/api/budget-ignore` | Dashboard auth/session | Budget Ignore Get | `pkg/dashboard/api.go:178` | -| `POST` | `/api/budget-ignore` | Owner only | Budget Ignore Set | `pkg/dashboard/api.go:179` | -| `GET` | `/api/summaries` | Dashboard auth/session | Summaries | `pkg/dashboard/api.go:191` | -| `POST` | `/api/prs/{owner}/{repo}/{number}/queue-automerge` | Merger/owner role | Queue PRAuto Merge | `pkg/dashboard/api.go:193` | -| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:370` | -| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:432` | -| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:433` | -| `POST` | `/api/chat` | Dashboard auth/session | Chat. JSON body `{"query": "", "history": [...]}`; `message` is accepted as an alias for `query` (`query` wins when both are set). A missing or blank field returns 400 `query is required`. `!`-prefixed commands go to the chat spine (`{"accepted":true,"seq":N}`, read-write); while an inception is in `clarify` with unanswered questions, an owner's unprefixed reply also goes to the spine, which submits it as the answer to the next unanswered question. | `pkg/dashboard/api.go:462` | -| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:500` | -| `GET` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:251` | -| `POST` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:252` | -| `GET` | `/api/docs` | Dashboard auth/session | APIDocs | `pkg/dashboard/api_contribute.go:253` | -| `GET` | `/leaderboard` | Public | Leaderboard Page | `pkg/dashboard/api_contribute.go:255` | -| `GET` | `/api/leaderboard` | Public | Leaderboard API | `pkg/dashboard/api_contribute.go:256` | -| `GET` | `/api/leaderboard/teams` | Public | Team Leaderboards | `pkg/dashboard/api_contribute.go:257` | -| `GET` | `/api/leaderboard/style` | Public | Leaderboard Style | `pkg/dashboard/api_contribute.go:258` | -| `GET` | `/api/stage-tracker` | Dashboard auth/session | Stage Tracker: every open issue and pull request of the watched repos for the Hive activity monitor — actionable, held, claimed, in-progress, covered-by-PR, filtered (`needs-*`), draft and exempt-labelled items alike, each with `stage`, `stage_since`, `labels`, `created_at`, `author_kind`, `modifiers`, (PRs) `sentinel_alert` / `sentinel_findings` / `sentinel_reason` when the sentinel flagged the PR, `finisher` / `finisher_verb` / `finisher_reason` (who can finish the item, from the your-turn decision: `hive`, `person` when it is the operator's turn, `elsewhere` when it waits on someone else (a hold, a covering PR, a reporter, an open blocker, or its author on a returned PR or a person's draft), absent when the snapshot has no decision; also on each `linked_prs` entry), and (issues) `linked_prs`; items with no stage yet report stage `0` "Open"; `needs_human_labels` is the "a person must act" label set behind the monitor's Needs human facet (the same list as `/api/status.labelPresets[needs-human]`). Standing meta issues are omitted from the monitor but still counted in the Overview breakdown; `recently_merged` lists the PRs (`repo`, `number`, `merged_at`) whose `pr_merged` audit entry is under 30 minutes old, so a client diffing two payloads can tell a merge from a close. Served from a server-side cache rebuilt at most every `dashboard.stage_tracker.cache_ttl_s` (default `governor.eval_interval_s`) from the spoke's existing snapshot — a request never calls GitHub. Carries `generated_at`, `source` (`cache` or `fresh`), `cache_ttl_s`, an `ETag` and `Cache-Control: private, max-age=`; `If-None-Match` yields 304. Each hit is recorded under REST caller `dashboard:stage-tracker` so `/api/gh-rate-limits` shows it charged nothing. | `pkg/dashboard/api.go:192` | | `GET` | `/api/leaderboard/battle-log` | Public | Scrubbed public Battle Log activity feed for leaderboard widgets. | `pkg/dashboard/battle_log.go:108` | | `GET` | `/api/leaderboard/hive-of-week` | Public | Featured weekly project metadata, configured render URLs (empty when none), gource source URL, and embed snippets. | `pkg/dashboard/battle_log.go:109` | | `GET` | `/api/leaderboard/gource-log` | Public | Gource custom log source for a project/week, derived from public hive event data. | `pkg/dashboard/battle_log.go:110` | | `POST` | `/api/leaderboard/hive-of-week/archive` | Owner only | Archive the selected project's weekly gource source log into knowledge; stores source only, never rendered video. | `pkg/dashboard/battle_log.go:111` | -| `GET` | `/api/leaderboard/contributor/{username}` | Public | Contributor Profile | `pkg/dashboard/api_contribute.go:265` | -| `GET` | `/api/hives` | Dashboard auth/session | Hives List | `pkg/dashboard/api_contribute.go:276` | -| `POST` | `/api/hives/register` | Dashboard auth/session | Hives Register | `pkg/dashboard/api_contribute.go:277` | -| `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:278` | -| `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:279` | -| `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:280` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1606` | +| `GET` | `/api/leaderboard/contributor/{username}` | Public | Contributor Profile | `pkg/dashboard/api_contribute.go:274` | +| `GET` | `/api/hives` | Dashboard auth/session | Hives List | `pkg/dashboard/api_contribute.go:285` | +| `POST` | `/api/hives/register` | Dashboard auth/session | Hives Register | `pkg/dashboard/api_contribute.go:286` | +| `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:287` | +| `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:288` | +| `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:289` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1651` | ## Hub SaaS @@ -752,31 +759,31 @@ always resolved server-side from the validated token. | `GET` | `/api/hub/clusters` | Hub auth | List Clusters | `pkg/hub/saas.go:528` | | `GET` | `/api/hub/image-pulls` | Hub auth | Per-Release Image Pull Series | `pkg/hub/saas.go:377` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:519` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1746` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1747` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1687` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1688` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1696` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1697` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1698` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1699` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1704` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1714` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1715` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1716` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1717` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1718` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1719` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1720` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1721` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1724` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1727` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1728` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1735` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1736` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1737` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1754` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1759` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1761` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1760` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1761` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1701` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1702` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1710` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1711` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1712` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1713` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1718` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1728` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1729` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1730` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1731` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1732` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1733` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1734` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1735` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1738` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1741` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1742` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1749` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1750` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1751` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1768` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1773` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1775` | diff --git a/src/docs/architecture.md b/src/docs/architecture.md index eaff927769..0647de65b1 100644 --- a/src/docs/architecture.md +++ b/src/docs/architecture.md @@ -293,7 +293,7 @@ flowchart LR L3["L3 Quality-Gated
quality opens PRs"] L4["L4 Security-Aware
more agents file/PR"] L5["L5 Semi-Autonomous
all PRs, hold gated"] - L6["L6 Fully Autonomous
auto-merge switches on"] + L6["L6 Autonomous
auto-merge switches on"] L1 --> L2 --> L3 --> L4 --> L5 --> L6 ``` @@ -304,7 +304,7 @@ flowchart LR | L3 | Quality-Gated (Measured) | `quality` opens PRs gated by literal `hold` about testing gaps, coverage, and CI health; others advisory. This measurement foundation is what earns automation at higher levels | | L4 | Security-Aware (Adaptive) | All agents file issues; quality, ci-maintainer, and sec-check can open PRs gated by literal `hold` | | L5 | Semi-Autonomous (Semi-Automated) | All agents open PRs — every PR carries literal `hold` for human review | -| L6 | Fully Autonomous | Agents open PRs and **auto-merge on green CI**; switching to L6 enables auto-merge for every active repo, owners may toggle repos afterward, no level hold, except cross-repo outreach PRs (target repo not managed by this hive) remain held | +| L6 | Autonomous | Agents open PRs and **auto-merge on green CI**; switching to L6 enables auto-merge for every active repo, owners may toggle repos afterward, no level hold, except cross-repo outreach PRs (target repo not managed by this hive) remain held | `supervisor` is always advisory. The full matrix is in [`acmm-policy-matrix.md`](acmm-policy-matrix.md). diff --git a/src/docs/assets/logos/SOURCES.md b/src/docs/assets/logos/SOURCES.md new file mode 100644 index 0000000000..10383678a9 --- /dev/null +++ b/src/docs/assets/logos/SOURCES.md @@ -0,0 +1,58 @@ +# Logo sources and licenses + +Every file in this directory is listed here with where it came from and under +which license. Logos are trademarks of their owners and are used for +identification only. See [integrations.md](../../integrations.md). + +## Simple Icons (CC0 1.0) + +Fetched from [simple-icons/simple-icons](https://github.com/simple-icons/simple-icons) at commit `90646e4bf7f3b89df6c37e65baf3e7bc759a517c`. The SVG files are released under [CC0 1.0](https://github.com/simple-icons/simple-icons/blob/develop/LICENSE.md). The only edit is a neutral `fill="#6b7280"` on the root element so the icon reads on light and dark backgrounds. + +| File | Brand | Source URL | +| --- | --- | --- | +| `github.svg` | GitHub | | +| `githubcopilot.svg` | GitHub Copilot | | +| `gitlab.svg` | GitLab | | +| `gitea.svg` | Gitea | | +| `forgejo.svg` | Forgejo | | +| `jira.svg` | Jira | | +| `linear.svg` | Linear | | +| `claude.svg` | Claude | | +| `nvidia.svg` | NVIDIA | | +| `kubernetes.svg` | Kubernetes | | +| `podman.svg` | Podman | | +| `cncf.svg` | CNCF | | + +## Text badges (generated, no third-party asset) + +No clearly redistributable logo was found for the projects below, or the brand +guidelines do not permit redistribution. Each file is a small SVG generated in +this repository (rounded rectangle plus the project name). It contains no +third-party artwork and is covered by the repository license. + +| File | Project | Note | +| --- | --- | --- | +| `goose-badge.svg` | Goose | no redistributable logo found | +| `antigravity-badge.svg` | Google Antigravity | no redistributable logo found | +| `codex-badge.svg` | OpenAI Codex | no redistributable logo found | +| `openai-agents-badge.svg` | OpenAI Agents API | no redistributable logo found | +| `omnigent-badge.svg` | Omnigent | no redistributable logo found | +| `opensandbox-badge.svg` | AAIF OpenSandbox | no redistributable logo found | +| `gateway-badge.svg` | OpenAI-compatible gateways (vLLM, llm-d, LiteLLM, watsonx, OpenRouter) | no redistributable logo found | +| `bob-badge.svg` | IBM Bob | no redistributable logo found | +| `pi-badge.svg` | Pi | no redistributable logo found | +| `aider-badge.svg` | Aider | no redistributable logo found | +| `kilo-badge.svg` | Kilo Code | no redistributable logo found | +| `muse-badge.svg` | Muse Code | no redistributable logo found | +| `opencode-badge.svg` | OpenCode | no redistributable logo found | +| `omp-badge.svg` | Oh My Pi | no redistributable logo found | +| `openhands-badge.svg` | OpenHands | no redistributable logo found | +| `gemini-badge.svg` | Gemini | no redistributable logo found | +| `gh-aw-badge.svg` | GitHub Agentic Workflows | no redistributable logo found | + +## Rules for adding a logo + +- Prefer the project's own brand page or a Simple Icons SVG (CC0). +- Record the exact source URL and license here in the same PR. +- Keep each SVG under 20 KB with no scripts, external references, or embedded images. +- If no redistributable asset exists, add a text badge instead of copying a logo. diff --git a/src/docs/assets/logos/aider-badge.svg b/src/docs/assets/logos/aider-badge.svg new file mode 100644 index 0000000000..09585aa1d3 --- /dev/null +++ b/src/docs/assets/logos/aider-badge.svg @@ -0,0 +1 @@ +AiderAider diff --git a/src/docs/assets/logos/antigravity-badge.svg b/src/docs/assets/logos/antigravity-badge.svg new file mode 100644 index 0000000000..25149cbd1f --- /dev/null +++ b/src/docs/assets/logos/antigravity-badge.svg @@ -0,0 +1 @@ +AntigravityAntigravity diff --git a/src/docs/assets/logos/bob-badge.svg b/src/docs/assets/logos/bob-badge.svg new file mode 100644 index 0000000000..60d1eea36c --- /dev/null +++ b/src/docs/assets/logos/bob-badge.svg @@ -0,0 +1 @@ +BobBob diff --git a/src/docs/assets/logos/claude.svg b/src/docs/assets/logos/claude.svg new file mode 100644 index 0000000000..18fc6850de --- /dev/null +++ b/src/docs/assets/logos/claude.svg @@ -0,0 +1 @@ +Claude \ No newline at end of file diff --git a/src/docs/assets/logos/cncf.svg b/src/docs/assets/logos/cncf.svg new file mode 100644 index 0000000000..0a2921b959 --- /dev/null +++ b/src/docs/assets/logos/cncf.svg @@ -0,0 +1 @@ +CNCF \ No newline at end of file diff --git a/src/docs/assets/logos/codex-badge.svg b/src/docs/assets/logos/codex-badge.svg new file mode 100644 index 0000000000..41ebd506ae --- /dev/null +++ b/src/docs/assets/logos/codex-badge.svg @@ -0,0 +1 @@ +CodexCodex diff --git a/src/docs/assets/logos/forgejo.svg b/src/docs/assets/logos/forgejo.svg new file mode 100644 index 0000000000..86384dfef7 --- /dev/null +++ b/src/docs/assets/logos/forgejo.svg @@ -0,0 +1 @@ +Forgejo \ No newline at end of file diff --git a/src/docs/assets/logos/gateway-badge.svg b/src/docs/assets/logos/gateway-badge.svg new file mode 100644 index 0000000000..597f9c28cf --- /dev/null +++ b/src/docs/assets/logos/gateway-badge.svg @@ -0,0 +1 @@ +OpenAI-compatibleOpenAI-compatible diff --git a/src/docs/assets/logos/gemini-badge.svg b/src/docs/assets/logos/gemini-badge.svg new file mode 100644 index 0000000000..b086e2680d --- /dev/null +++ b/src/docs/assets/logos/gemini-badge.svg @@ -0,0 +1 @@ +GeminiGemini diff --git a/src/docs/assets/logos/gh-aw-badge.svg b/src/docs/assets/logos/gh-aw-badge.svg new file mode 100644 index 0000000000..da84b119bf --- /dev/null +++ b/src/docs/assets/logos/gh-aw-badge.svg @@ -0,0 +1 @@ +gh-awgh-aw diff --git a/src/docs/assets/logos/gitea.svg b/src/docs/assets/logos/gitea.svg new file mode 100644 index 0000000000..a9a0a88bdb --- /dev/null +++ b/src/docs/assets/logos/gitea.svg @@ -0,0 +1 @@ +Gitea \ No newline at end of file diff --git a/src/docs/assets/logos/github.svg b/src/docs/assets/logos/github.svg new file mode 100644 index 0000000000..eeb4b4f5c7 --- /dev/null +++ b/src/docs/assets/logos/github.svg @@ -0,0 +1 @@ +GitHub \ No newline at end of file diff --git a/src/docs/assets/logos/githubcopilot.svg b/src/docs/assets/logos/githubcopilot.svg new file mode 100644 index 0000000000..9865d1668f --- /dev/null +++ b/src/docs/assets/logos/githubcopilot.svg @@ -0,0 +1 @@ +GitHub Copilot \ No newline at end of file diff --git a/src/docs/assets/logos/gitlab.svg b/src/docs/assets/logos/gitlab.svg new file mode 100644 index 0000000000..cab7e79802 --- /dev/null +++ b/src/docs/assets/logos/gitlab.svg @@ -0,0 +1 @@ +GitLab \ No newline at end of file diff --git a/src/docs/assets/logos/goose-badge.svg b/src/docs/assets/logos/goose-badge.svg new file mode 100644 index 0000000000..13ad5750a9 --- /dev/null +++ b/src/docs/assets/logos/goose-badge.svg @@ -0,0 +1 @@ +GooseGoose diff --git a/src/docs/assets/logos/jira.svg b/src/docs/assets/logos/jira.svg new file mode 100644 index 0000000000..9df595bdff --- /dev/null +++ b/src/docs/assets/logos/jira.svg @@ -0,0 +1 @@ +Jira \ No newline at end of file diff --git a/src/docs/assets/logos/kilo-badge.svg b/src/docs/assets/logos/kilo-badge.svg new file mode 100644 index 0000000000..b11b15e8be --- /dev/null +++ b/src/docs/assets/logos/kilo-badge.svg @@ -0,0 +1 @@ +KiloKilo diff --git a/src/docs/assets/logos/kubernetes.svg b/src/docs/assets/logos/kubernetes.svg new file mode 100644 index 0000000000..8924ebaf7c --- /dev/null +++ b/src/docs/assets/logos/kubernetes.svg @@ -0,0 +1 @@ +Kubernetes \ No newline at end of file diff --git a/src/docs/assets/logos/linear.svg b/src/docs/assets/logos/linear.svg new file mode 100644 index 0000000000..312705d95a --- /dev/null +++ b/src/docs/assets/logos/linear.svg @@ -0,0 +1 @@ +Linear \ No newline at end of file diff --git a/src/docs/assets/logos/muse-badge.svg b/src/docs/assets/logos/muse-badge.svg new file mode 100644 index 0000000000..7301482289 --- /dev/null +++ b/src/docs/assets/logos/muse-badge.svg @@ -0,0 +1 @@ +MuseMuse diff --git a/src/docs/assets/logos/nvidia.svg b/src/docs/assets/logos/nvidia.svg new file mode 100644 index 0000000000..7fd6a137b1 --- /dev/null +++ b/src/docs/assets/logos/nvidia.svg @@ -0,0 +1 @@ +NVIDIA \ No newline at end of file diff --git a/src/docs/assets/logos/omnigent-badge.svg b/src/docs/assets/logos/omnigent-badge.svg new file mode 100644 index 0000000000..e4931e7619 --- /dev/null +++ b/src/docs/assets/logos/omnigent-badge.svg @@ -0,0 +1 @@ +OmnigentOmnigent diff --git a/src/docs/assets/logos/omp-badge.svg b/src/docs/assets/logos/omp-badge.svg new file mode 100644 index 0000000000..8fd7f7e693 --- /dev/null +++ b/src/docs/assets/logos/omp-badge.svg @@ -0,0 +1 @@ +OMPOMP diff --git a/src/docs/assets/logos/openai-agents-badge.svg b/src/docs/assets/logos/openai-agents-badge.svg new file mode 100644 index 0000000000..8e35a7ccab --- /dev/null +++ b/src/docs/assets/logos/openai-agents-badge.svg @@ -0,0 +1 @@ +OpenAI AgentsOpenAI Agents diff --git a/src/docs/assets/logos/opencode-badge.svg b/src/docs/assets/logos/opencode-badge.svg new file mode 100644 index 0000000000..1d83433ed6 --- /dev/null +++ b/src/docs/assets/logos/opencode-badge.svg @@ -0,0 +1 @@ +OpenCodeOpenCode diff --git a/src/docs/assets/logos/openhands-badge.svg b/src/docs/assets/logos/openhands-badge.svg new file mode 100644 index 0000000000..58e4f3df0d --- /dev/null +++ b/src/docs/assets/logos/openhands-badge.svg @@ -0,0 +1 @@ +OpenHandsOpenHands diff --git a/src/docs/assets/logos/opensandbox-badge.svg b/src/docs/assets/logos/opensandbox-badge.svg new file mode 100644 index 0000000000..3e018c072c --- /dev/null +++ b/src/docs/assets/logos/opensandbox-badge.svg @@ -0,0 +1 @@ +OpenSandboxOpenSandbox diff --git a/src/docs/assets/logos/pi-badge.svg b/src/docs/assets/logos/pi-badge.svg new file mode 100644 index 0000000000..eafdf9c7c9 --- /dev/null +++ b/src/docs/assets/logos/pi-badge.svg @@ -0,0 +1 @@ +PiPi diff --git a/src/docs/assets/logos/podman.svg b/src/docs/assets/logos/podman.svg new file mode 100644 index 0000000000..d4687355bb --- /dev/null +++ b/src/docs/assets/logos/podman.svg @@ -0,0 +1 @@ +Podman \ No newline at end of file diff --git a/src/docs/backend-gap-claude-managed-agents.md b/src/docs/backend-gap-claude-managed-agents.md new file mode 100644 index 0000000000..c6be17a30b --- /dev/null +++ b/src/docs/backend-gap-claude-managed-agents.md @@ -0,0 +1,244 @@ +# Gap analysis: Claude Managed Agents as a Hive backend + +This note checks [Claude Managed Agents](https://claude.com/resources/articles/claude-managed-agents) +against the acceptance bar in [backend-support-tiers.md](backend-support-tiers.md): +the "All tiers" criteria (1-7) and the T3 criterion (8). It is the first child +of [#14093](https://github.com/hivecommons/hive/issues/14093) and closes +[#14106](https://github.com/hivecommons/hive/issues/14106). It is analysis +only. It changes no code and does not assign a tier. The tier decision belongs +to [#14111](https://github.com/hivecommons/hive/issues/14111), which reads this +note next to the OpenAI Agents API note from +[#14108](https://github.com/hivecommons/hive/issues/14108). + +Verified 2026-10-25 against the Managed Agents API docs (public beta, header +`managed-agents-2026-04-01`). Vendor behaviour below is what those docs state; +nothing here has been run against a live session yet. That evidence is the job +of the prototype in [#14124](https://github.com/hivecommons/hive/issues/14124). + +## What the product is + +Managed Agents splits one agent run across four objects +([overview](https://platform.claude.com/docs/en/managed-agents/overview)): + +| Object | What it holds | Hive analogue | +| --- | --- | --- | +| **Agent** (`/v1/agents`) | Model, system prompt, tools, MCP servers, skills. Persisted and versioned; every update is a new immutable version. | The agent definition plus the policy prompt | +| **Environment** (`/v1/environments`) | A container template: `cloud` (Anthropic runs the container) or `self_hosted` (a worker you run executes the tools). Networking policy lives here. | The contributor pod image and its egress rules | +| **Session** (`/v1/sessions`) | One run of an agent in an environment. Carries resources (files, repos), vault IDs, an optional spend budget, and an event stream. | One leased task | +| **Container** | Where tools (bash, file ops, code) execute. The agent loop itself runs on Anthropic's orchestration layer, not in the container. | The agent's tmux pane and workdir | + +The important difference from every current Hive backend: **the agent loop does +not run on Hive's hardware at all.** Even with a `self_hosted` environment only +the tool calls run locally; the model sees tool inputs and outputs on +Anthropic's side. There is no binary to launch, so Hive would drive it as an +API client, not as a CLI. + +## How the four things the issue asks about map + +### 1. Task prompt + +The task prompt goes in as a `user.message` event, either in `initial_events` +on `POST /v1/sessions` or posted to the session afterwards. Hive's policy +prompt maps to the agent's `system` field. A session can override agent fields +for one run, so a per-lane prompt does not need a new agent version. For a task +with one deliverable (a PR-sized change), `user.define_outcome` adds a rubric +and a grader loop; that is optional and costs extra model calls. + +No gap. This is a straight API call. + +### 2. Repo workspace + +There are three ways to put a repository in front of the agent: + +| Option | How | Credential held by Anthropic | Can the agent push? | +| --- | --- | --- | --- | +| **A. `github_repository` resource** ([docs](https://platform.claude.com/docs/en/managed-agents/github)) | Anthropic clones the repo into `/workspace/` before the run. | A GitHub token (`authorization_token`). It never enters the container; an Anthropic-side git proxy injects it into `git` and GitHub REST calls for that repo. It can be rotated on a running session. | Yes, if the token has `Contents: write`. | +| **B. File resource** | Hive uploads a tarball or `git bundle` with the Files API; it mounts read-only under `/mnt/session/uploads/`. | None. | No. The agent writes its result to `/mnt/session/outputs/`, which Hive downloads. | +| **C. `self_hosted` environment** ([docs](https://platform.claude.com/docs/en/managed-agents/self-hosted-sandboxes)) | Hive's own worker claims the work item and runs the tools in a container Hive controls. `file` and `github_repository` resources are refused; Hive clones the repo itself, as it does today. | None for GitHub. The worker holds an environment key and a per-session work secret. | Only through whatever egress Hive's container allows. | + +Getting the change back out is the open part. Under options B and C, and under +A with a read-only token, the agent cannot write to GitHub itself. Hive takes +the patch from `/mnt/session/outputs/` (B), or from its own workdir (C), and +opens the PR through `hive-open-pr` like any other lane. That fits the +`pkg/extwork` receipt model: `OpenArtifact` opens an artifact by execution key +and relative path, and the receipt is checked against a declared digest before +parsing. + +### 3. GitHub credential boundary + +Hive's rule is that agents never hold the App installation token. Writes go +through the relays in [github-write-surface.md](github-write-surface.md), and +the GitHub proxy hard-denies two calls for every agent mode (`denyRules` in +`src/crux/proxy/rules.go`): + +- `POST /repos/{o}/{r}/pulls` (direct PR creation), and +- `PUT /repos/{o}/{r}/pulls/{n}/merge` (direct merge). + +Those denials only work because the agent's traffic is forced through Hive's +proxy by the iptables redirect described in +[net-admin-requirement.md](net-admin-requirement.md). A `cloud` session's +traffic never touches that proxy. So the question per option is: **can the +sandbox reach GitHub with a write credential by a path Hive does not see?** + +| Path to GitHub from a session | Seen by Hive's proxy? | Write-capable? | Verdict | +| --- | --- | --- | --- | +| `github_repository` resource with a write token | No. Anthropic's git proxy injects the token. The docs say GitHub REST calls for the attached repo go the same way (they also say opening a PR needs the GitHub MCP server); treat `POST /pulls` as reachable until a test shows otherwise. | Yes | **Direct egress. Advisory ceiling.** | +| `github_repository` resource with a read-only, single-repo token | No | No: GitHub itself refuses the write. | Allowed. The proxy denials hold because there is nothing to deny. | +| Vault `environment_variable` credential (e.g. `GH_TOKEN`) | No. Substituted at Anthropic's egress for the hosts listed on the credential. | Yes | **Direct egress. Advisory ceiling.** | +| GitHub MCP server through a vault credential | No. Anthropic's MCP proxy adds the credential. | Yes | **Direct egress. Advisory ceiling.** | +| `cloud` environment, `networking: unrestricted`, no GitHub credential | No | Only with a credential the agent finds or is given. Anonymous reads work. | Not acceptable as the default; use `limited`. | +| `cloud` environment, `networking: limited`, no GitHub host in `allowed_hosts` | n/a | No | Allowed. | +| Custom tool (`agent.custom_tool_use`) executed by Hive | Yes. Hive runs the call on its own side and answers with `user.custom_tool_result` over the same authenticated event stream. | Only what Hive's relay permits. | Allowed. This is how a hosted session asks Hive to write. | +| `self_hosted` worker inside a Hive pod with forced egress | Yes. Same proxy, same `denyRules`. | Only through the relays. | Allowed. Same posture as a local CLI backend. | + +Two ways of running Managed Agents keep both proxy denials in force: + +1. **Cloud, credential-less.** `cloud` environment, `networking: limited`, + no GitHub host listed, `web_search` and `web_fetch` disabled, no GitHub MCP + server, no `environment_variable` GitHub secret. The repo arrives as a file + resource or under a read-only token that Hive mints for that one repo. Any + write is a custom tool that Hive turns into a relay request. +2. **Self-hosted.** A `self_hosted` environment whose worker runs in a Hive + pod. Tool traffic goes through Hive's proxy exactly as it does for a CLI + backend. The worker must not hold `ANTHROPIC_API_KEY` (the docs warn it + exposes an organisation-scoped key to tool calls); it needs only the + environment key and the per-session work secret. + +Every other setup (a write token on the repo resource, a vaulted GitHub +secret, the GitHub MCP server) gives the hosted sandbox a GitHub write path +Hive cannot see. Per the #14093 constraint, such a setup **cannot exceed +advisory mode**. In `pkg/extwork` terms that is `shadow` or `report-only` +(`ModeShadow`, `ModeReportOnly` in `admission.go`): report-only "starts work +but carries no publication credential of any kind". + +One more custody point. Even a read-only token in option A is a GitHub +credential stored by a third party. It should be a short-lived installation +token scoped to one repository and `contents: read`, minted per session and +rotated or left to expire. It must never be the App's own key or a broad PAT. + +### 4. Transcript and cost telemetry + +Managed Agents reports usage at three levels +([events](https://platform.claude.com/docs/en/managed-agents/events-and-streaming), +[sessions](https://platform.claude.com/docs/en/managed-agents/sessions)): + +| Source | Fields | Use for metering | +| --- | --- | --- | +| `span.model_request_end` event | `id`, `processed_at`, `is_error`, `model_request_start_id`, `model_usage.{input_tokens, output_tokens, cache_read_input_tokens, cache_creation_input_tokens}` | One record per model request, with a timestamp and a stable event ID to deduplicate on. This is the per-call row. | +| `session.usage` event, and `usage` on the session object | Cumulative token totals; `list_cost` as `{amount, currency}` (integer string in cents, USD only, rounded to the cent); `active_seconds`; `server_tool_use.web_search_requests` and `web_fetch_requests` (the latter always 0 today) | The authoritative session total, including charges that are not tokens. | +| Per-thread `usage` (multi-agent sessions) | Per-thread `list_cost`, `active_seconds` | Breakdown only. The docs say per-thread figures do not sum to the session figure. | +| `stats.active_seconds` | Per-thread time, summed | Not the billed figure; `usage.active_seconds` is. | + +The model name is not on the usage event. It comes from the agent version the +session pinned (or a session override), so the adapter has to record it at +dispatch. + +`list_cost` is priced at public list rates, not the operator's contracted +price. Besides tokens it counts web searches ($10 per 1,000) and session run +time ($0.08 per hour). Hive's token pipeline (`pkg/tokens`) has no place for +those last two today. `InferenceSink` records tokens per agent and model, and +the closest existing shape is the four-field record behind +`RecordCopilotWithCache`. The issue calls the target the "lineage ledger"; +there is no package by that name. The nearest thing is the collector's token +totals that feed the dashboard's spend headline and its "live token summary" +(see [dashboard-sections.md](dashboard-sections.md)). + +The transcript is the event list: `GET /v1/sessions/{id}/events` is durable +and paginated, and `GET /v1/sessions/{id}/events/stream` is live SSE with no +replay. A client that reconnects must open the stream, then list, then +deduplicate by event ID. `agent.thinking` events mark progress but carry no +thinking text. Each session also has a Console trace URL. Hive's +`claude_scanner.go` reads Claude Code session JSONL and does not apply here; +a hosted backend needs its own sink. + +Managed Agents also offers what Hive's budget gate would otherwise have to +build: a session `budget` (`{type: "limit", max_list_cost: {amount, currency}}`) +is a hard cap the platform checks before every model request. A session at its +cap pauses with `stop_reason: budget_reached` instead of overspending. The +adapter can set it from the governor's remaining weekly budget at dispatch. It +is create-only and can only be changed or removed afterwards. + +## Acceptance criteria, row by row + +Status key: **met** (the product does what the criterion needs, or the +criterion is satisfied by construction), **gap** (Hive work, vendor evidence, +or both are missing), **N/A** (the criterion is about a local CLI binary and +has no counterpart). + +| # | Criterion | Status | Reason | +| --- | --- | --- | --- | +| 1 | List parity (`KNOWN_BACKENDS`, `backend_binary`, `backend_perm_flag`, `CLIBackends`) | **N/A** | There is no binary to launch or flag to pass. The `claude-managed-agents` engine is registered through `pkg/extwork` only under `extwork_claude_managed_agents`; it deliberately never enters the CLI registry. | +| 2 | Declared local posture (`localBackendPostures`) | **met** for the selected `self_hosted` shape | The prototype pins a `self_hosted` environment and requires `forced-egress` or `proxy-only`. The worker runs in the Hive pod and the direct-path refusal test covers the write boundary. | +| 3 | Flag-honoring proof | **partial** | The safety controls are API/config fields, not flags. Adapter validation rejects cloud environments, unsupported network policies, over-cap budgets, and unrecognised write paths; tests exercise the direct-path refusal. No live vendor invalid-policy transcript is claimed, so the promotion evidence remains open. | +| 4 | Honest credential detection (`detect_cli`) | **met** for the hosted contract | The adapter resolves only an API-key environment-variable or `/data/secrets`/`/secrets` file reference, authenticates with `GET /v1/environments/{id}` before dispatch, and never puts `ANTHROPIC_API_KEY` in the worker bundle. | +| 5 | Documentation rows | **met** for the hosted tier | Hosted rows now live in `agent-configuration.md`, `sandbox-isolation.md`, `contributor-relay.md`, `backend-support-tiers.md`, and the external-admission design. `backend-setup.md` and the `AGENT_BACKEND` CLI row remain unchanged because this is not a CLI backend. | +| 6 | Pinned install | **N/A** for `cloud`, **met by construction** for `self_hosted` | Nothing is installed for `cloud`. The pinned runtime identity is the explicit agent version and toolset version in the session request; the adapter itself is selected by a build tag. | +| 7 | Changelog fragment | **met** | `changelog.d/added-hosted-managed-agents.md` records the shipped prototype. | +| 8 | T3 refusal gate (`HIVE__DANGEROUSLY_RUN_UNCONFINED`) | **met for the hosted counterpart** | The binding defaults off, resolves enabled-without-mode to `shadow`, dispatches only in explicit `report-only`, and refuses write-capable stages for direct GitHub resource, vaulted-secret, and GitHub-MCP paths. | + +The T1 criteria are outside this note's scope, but they are not where the gap +is. An API is headless by nature (criterion 10), and an API key or workload +identity in a fresh pod is an unattended credential (criterion 11). The hard +parts are 1, 2 and 8: the bar assumes a local binary that Hive confines, and +here the agent loop is not Hive's to confine. + +## Other facts the tier decision needs + +- **Availability.** Managed Agents is beta on the Claude API and Claude Platform + on AWS. It is not offered on Bedrock, Vertex AI or Foundry. +- **Lifecycle mapping to `pkg/extwork`.** Session status (`running`, `idle`, + `rescheduling`, `terminated`) maps onto `Observe`'s running / waiting / + terminal states, with a transport failure reported as unknown. `terminated` + covers both success and unrecoverable error, so the adapter must fetch the + session to tell them apart. `Cancel` maps to a `user.interrupt` event. + `Start` needs idempotency on the execution key; the docs reviewed say nothing + about idempotent session creation, so the adapter must persist the session ID + against the key (session `metadata` holds up to 8 keys) and reconcile before + any retry. +- **Permission policies.** Server-run tools can be `always_allow`, + `always_ask` (pauses for a `user.tool_confirmation`), or `auto` (the server + decides; calls it denies cannot be overridden by the client). This gives Hive + a second approval point, but it is the vendor's classifier, not Hive's + policy, and it does not cover custom tools. +- **Web tools.** `web_search` and `web_fetch` run on Anthropic's servers in + both environment types and are not limited by a self-hosted worker's egress. + Disable them unless a task needs them. +- **Spend.** The API key owner pays: tokens at list price, plus run time and + searches. On a contributor-private spoke that is the contributor's key. The + prototype persists those fields in its ledger, reserves the remaining + per-day budget before dispatch, and hard-caps the spoke at USD 25 per UTC day. + +## Recommendation + +**A new "hosted execution" tier, not CLI-style and not gateway-path.** + +- It is not **CLI-style**: criteria 1, 2 and 8 assume a local binary whose + flags Hive controls, and there is none. +- It is not **gateway-path**: that tier is about routing model calls from the + Claude CLI through `pkg/proxy/anthropic_translate.go`. Managed Agents runs + the whole loop, tools included, so the gateway rows (model picker, login, + terminal) do not describe it. +- It fits the existing **`pkg/extwork`** external-host contract almost one to + one: start keyed by execution key, observe by status, cancel by interrupt, + receipt by artifact path. That contract already caps admission at shadow and + report-only, which is exactly the advisory ceiling the issue asks for. + +Suggested acceptance criteria for that tier, for #14111 to adopt or change: + +1. The engine is registered as a `pkg/extwork` adapter and passes its + conformance tests (refusal, receipt verification, cancellation). +2. The PR states which environment type it uses and, for each row of the + GitHub egress table that applies, why the `POST /pulls` and + `PUT /pulls/{n}/merge` denials still hold. Any direct write path caps the + engine at `report-only`, enforced in code. +3. Metering: every `span.model_request_end` becomes a token record and + `session.usage.list_cost` is reconciled against it; non-token charges + (run time, searches) are recorded or the backend is shown as partly + unmetered. +4. A session `budget` is set at dispatch whenever the hive is budget-gated. + +Of the two shapes above, **self-hosted** is closer to how Hive already confines +agents, since tool egress goes through Hive's own proxy and needs no new +custody argument for GitHub tokens. The accepted hosted-tier decision in #14111 +selected that shape for #14124; the OpenAI Agents API remains a separate +credential-less adapter proposal under #14108. diff --git a/src/docs/backend-gap-openai-agents-api.md b/src/docs/backend-gap-openai-agents-api.md new file mode 100644 index 0000000000..08b7c27cfd --- /dev/null +++ b/src/docs/backend-gap-openai-agents-api.md @@ -0,0 +1,225 @@ +# Gap analysis: the OpenAI Agents API as a Hive backend + +This note checks the OpenAI hosted agent stack (the +[Responses API](https://platform.openai.com/docs/api-reference/responses), its +hosted tools, and the [Agents SDK](https://openai.github.io/openai-agents-python/)) +against the acceptance bar in [backend-support-tiers.md](backend-support-tiers.md): +the "All tiers" criteria (1-7) and the T3 criterion (8). It is a child of +[#14093](https://github.com/hivecommons/hive/issues/14093) and closes +[#14108](https://github.com/hivecommons/hive/issues/14108). It is analysis +only. It changes no code and does not assign a tier; that decision belongs to +[#14111](https://github.com/hivecommons/hive/issues/14111), which reads this +note next to the Claude Managed Agents note from +[#14106](https://github.com/hivecommons/hive/issues/14106). + +Vendor behaviour below is taken from OpenAI's public documentation +([shell tool](https://developers.openai.com/api/docs/guides/tools-shell), +[new tools in the Responses API](https://openai.com/index/new-tools-and-features-in-the-responses-api/), +[computer environment](https://openai.com/index/equip-responses-api-computer-environment/), +[Agents SDK tools](https://openai.github.io/openai-agents-python/tools/)). +Nothing here has been run against a live account. Exact field names and +network-policy options should be confirmed by the prototype in +[#14124](https://github.com/hivecommons/hive/issues/14124). + +## What the product is + +"OpenAI Agents API" is not one endpoint. Three pieces matter: + +| Piece | What it does | Hive analogue | +| --- | --- | --- | +| **Responses API** (`/v1/responses`) | One model turn, or a server-side tool loop. Supports `background: true` (poll or stream later), `previous_response_id` and Conversations for state, and `store` for retention. | The model call plus the transcript | +| **Hosted tools** | Run on OpenAI's side: hosted shell/container (`environment: container_auto`), code interpreter, file search, web search, remote MCP. | The agent's tool surface | +| **Agents SDK** | A client library (Python, JS) that runs the agent loop on the caller's machine, calling the Responses API for model turns. Shell and file tools can be local or hosted. | The CLI binary | + +Unlike Claude Managed Agents there is no persisted, versioned "agent" or +"session" object that owns the run. State is the chain of responses (or a +Conversation) and, for hosted shell, a container. Two shapes are therefore +possible, and they have different answers below: + +1. **Hosted loop.** Hive calls the Responses API with hosted tools. The loop + and the container are on OpenAI's side. +2. **SDK loop in a Hive pod.** An Agents SDK program (or `codex`, which Hive + already has as a T1/T2 backend) runs in a Hive pod with local tools. Only + model calls leave the pod. + +Shape 2 is not a new backend: it is the existing CLI path with a different +client. The rest of this note is about shape 1 unless it says otherwise. + +## How the four things the issue asks about map + +### 1. Task prompt + +The task prompt is the `input` of a Responses call; Hive's policy prompt is +`instructions`. Multi-turn follow-ups chain with `previous_response_id`. +`background: true` returns immediately and the run is polled with +`GET /v1/responses/{id}` or streamed, which suits an unattended lane. Cancel +is `POST /v1/responses/{id}/cancel`. + +No gap. This is a straight API call. + +### 2. Repo workspace + +A hosted container starts empty. The options: + +| Option | How | GitHub credential held by OpenAI | Can the agent push? | +| --- | --- | --- | --- | +| **A. Upload** | Hive uploads a tarball or `git bundle` as a container file; the agent works in the container and Hive downloads the result files. | None | No | +| **B. Clone in the container** | The agent runs `git clone` inside the hosted shell. | Whatever token it is given (env, header, MCP secret) | Yes, if the token allows it, and only if the container can reach `github.com` | +| **C. Remote MCP server (GitHub or Hive-run)** | The model calls an MCP server that OpenAI's side contacts directly. | The MCP authorization value passed in the tool definition | Depends on the server | +| **D. Local shell via the SDK** | Hive clones the repo into its own pod, as today; the hosted API sees only tool inputs and outputs. | None | Only through Hive's egress | + +Getting the change back out under A and D uses the same path as every other +lane: Hive takes the patch and opens the PR through `hive-open-pr`. + +### 3. GitHub credential boundary and egress path + +Hive's rule is that agents never hold the App installation token. Writes go +through the relays in [github-write-surface.md](github-write-surface.md), and +the proxy hard-denies, for every agent mode (`denyRules` in +`src/crux/proxy/rules.go`): + +- `POST /repos/{o}/{r}/pulls` (direct PR creation), and +- `PUT /repos/{o}/{r}/pulls/{n}/merge` (direct merge). + +Those denials hold only because agent traffic is forced through the proxy by +the iptables redirect in [net-admin-requirement.md](net-admin-requirement.md). +**A hosted container's traffic never touches that proxy.** Whether the hosted +sandbox reaches GitHub directly therefore depends on the container's network +policy, not on anything Hive controls: + +| Path to GitHub | Seen by Hive's proxy? | Write-capable? | Verdict | +| --- | --- | --- | --- | +| Hosted container with open outbound access and any GitHub token | No | Yes | **Direct egress. Advisory ceiling.** | +| Hosted container with a domain allowlist that includes `github.com` / `api.github.com` and a write token | No | Yes | **Direct egress. Advisory ceiling.** | +| Hosted container with a domain allowlist that excludes GitHub, or no network | n/a | No | Allowed | +| Remote MCP server with a GitHub credential in the tool definition | No. OpenAI's side calls the server. | Yes | **Direct egress. Advisory ceiling.** | +| Function tool (client-executed) that Hive answers | Yes. Hive runs the call and returns the output in the next request. | Only what Hive's relay permits | Allowed. This is how a hosted run asks Hive to write. | +| Read-only, single-repo token, short-lived | No | No: GitHub refuses the write | Allowed. Nothing for the proxy to deny. | +| SDK loop in a Hive pod with forced egress | Yes | Only through the relays | Allowed. Same posture as a CLI backend. | + +Hosted web search and any hosted tool with its own outbound access also run on +OpenAI's side and are not bound by a container allowlist; disable them unless +a task needs them. + +**Stated plainly.** Credential boundary: the full App token is never sent to +OpenAI, in any shape. At most OpenAI holds a short-lived, one-repository, +`contents: read` installation token minted per run by Hive. GitHub egress: a +hosted container that can reach GitHub with a write credential egresses +directly, bypasses the proxy, and is capped at advisory mode (`shadow` or +`report-only` in `pkg/extwork`, `ModeShadow` and `ModeReportOnly` in +`admission.go`). The proxy denials for `POST /pulls` and +`PUT /pulls/{n}/merge` are only guaranteed in the credential-less hosted setup +(no GitHub host, no GitHub token, no GitHub MCP), where Hive turns writes into +function-tool calls, and in the SDK-in-pod shape. + +### 4. Transcript and cost telemetry + +The Responses object carries usage and a full output list: + +| Source | Fields | Use for metering | +| --- | --- | --- | +| `response.usage` | `input_tokens`, `output_tokens`, `total_tokens`, `input_tokens_details.cached_tokens`, `output_tokens_details.reasoning_tokens` | The per-response token record. Deduplicate on `response.id`. | +| `response.id`, `created_at`, `completed_at`, `status`, `model`, `previous_response_id` | Identity, timing, final state, and the model that served the call (unlike Managed Agents, the model is on the response) | Join key, timestamps, per-model attribution | +| `response.output[]` | Messages, reasoning items, tool calls and outputs | The transcript | +| `GET /v1/responses/{id}/input_items` | The inputs of a stored response | Transcript completeness when the chain is rebuilt | +| Streaming events (`response.completed`, tool-call events) | Same usage, live | Progress while a background run is in flight | + +Gaps in metering: the usage object counts tokens only. Hosted tool charges +(container sessions, code interpreter, file-search calls, web-search calls) are +billed separately and are not documented as fields on `usage`; they would have +to be derived from `output[]` items or read from the organisation usage API. +There is no cost field and no server-enforced per-run spend cap comparable to a +session budget, so a budget gate has to be Hive's: cap `max_output_tokens` and +`max_tool_calls` per call and stop chaining when the governor says so. A stored +response is retained by OpenAI (`store`), which is a data-custody point to +record for any contributor-private spoke. + +Mapping to Hive: the issue calls the target the "lineage ledger". There is no +package by that name. The nearest existing sink is the token pipeline in +`pkg/tokens` (see [token-tracking.md](token-tracking.md)), which records +tokens per agent and model with a cache split. + +| Ledger field | OpenAI source | +| --- | --- | +| agent / lane | Hive-side, set at dispatch | +| model | `response.model` | +| input tokens | `usage.input_tokens` minus `cached_tokens` | +| cache-read tokens | `usage.input_tokens_details.cached_tokens` | +| cache-write tokens | not reported (OpenAI caching is automatic and has no write charge) | +| output tokens | `usage.output_tokens` (includes reasoning) | +| reasoning tokens | `usage.output_tokens_details.reasoning_tokens` (informational) | +| timestamp | `completed_at` | +| dedupe key | `response.id` | +| non-token charges | not in `usage`; unmetered until derived | + +## Acceptance criteria, row by row + +Status key: **met**, **gap** (Hive work or vendor evidence missing), **N/A** +(the criterion is about a local CLI binary with no counterpart). + +| # | Criterion | Status | Reason | +| --- | --- | --- | --- | +| 1 | List parity (`KNOWN_BACKENDS`, `backend_binary`, `backend_perm_flag`, `CLIBackends`) | **N/A** for the hosted loop; **met** for the SDK-in-pod shape if it runs as `codex` | The hosted loop has no binary; a name in `KNOWN_BACKENDS` would make the shell launch path try to exec it. The counterpart is an engine adapter registered with `pkg/extwork` (`registry.go`), not wired for this engine. | +| 2 | Declared local posture (`localBackendPostures`) | **N/A** hosted, **gap** SDK-in-pod | A hosted run has no contributor-local path. An SDK worker's posture is that of its pod and no test pins it. | +| 3 | Flag-honoring proof | **gap** | The controls are request fields (container network policy, tool list, `max_tool_calls`, `max_output_tokens`), not flags. No transcript yet shows the API refusing an invalid network policy or tool config rather than ignoring it. | +| 4 | Honest credential detection (`detect_cli`) | **gap** | The credential is `OPENAI_API_KEY`. A cheap authenticated read (for example `GET /v1/models`) proves it works, the `codex_is_authenticated` pattern. Not written. | +| 5 | Documentation rows | **gap** | The tables named all describe CLIs. A hosted backend needs rows for auth, credential storage, container network policy, and the GitHub egress verdict above. | +| 6 | Pinned install | **N/A** hosted, **met by construction** SDK-in-pod | Nothing is installed for the hosted loop; the pin that matters is the model snapshot and tool version in the request. An SDK worker is pinned by lockfile. | +| 7 | Changelog fragment | **met by construction** | Any PR that wires the adapter adds one. This note is docs-only and needs none. | +| 8 | T3 refusal gate (`HIVE__DANGEROUSLY_RUN_UNCONFINED`) | **gap** | The env-var gate concerns launching an unconfined local CLI and does not transfer. The counterpart is a per-repo opt-in plus code that refuses a run config with a direct GitHub write path (every "advisory ceiling" row above) when the mode would allow writes. `pkg/extwork` already defaults to off and admits only shadow and report-only. | + +The T1 criteria are outside this note's scope. An API is headless (criterion +10) and an API key in a fresh pod is an unattended credential (criterion 11). +The hard parts are 1, 2 and 8: the bar assumes a local binary Hive confines, +and in the hosted shape the loop and sandbox are not Hive's to confine. + +## Other facts the tier decision needs + +- **Lifecycle mapping to `pkg/extwork`.** Response `status` (`queued`, + `in_progress`, `completed`, `failed`, `cancelled`, `incomplete`) maps onto + `Observe`'s running and terminal states; `Cancel` is the cancel endpoint. + `Start` needs idempotency on the execution key; the adapter should persist + the response ID against the key (response `metadata` can carry it) and + reconcile before any retry. +- **Approvals.** Remote MCP tools support `require_approval`, which gives a + second approval point, but it is the client's choice per call, not Hive + policy, and it does not cover hosted shell. +- **Spend.** The API key owner pays tokens plus hosted-tool charges. On a + contributor-private spoke that is the contributor's key, which is why + #14124 is parked for a maintainer decision (risk: spend). +- **Codex overlap.** Hive already drives OpenAI models through the `codex` + CLI. The incremental value of a hosted OpenAI backend is the hosted + container, which is also the part Hive cannot confine. + +## Recommendation + +**CLI-style for anything that must write; a new "hosted execution" tier for the +hosted loop, capped at advisory mode.** + +- The **SDK-in-pod** shape is CLI-style and already covered by `codex`. It + keeps both proxy denials and needs no new tier. +- The **hosted loop** is not CLI-style (criteria 1, 2 and 8 assume a local + binary) and not gateway-path (there is no CLI whose model calls Hive can + reroute; the loop and tools run remotely). It fits the `pkg/extwork` + external-host contract: start by execution key, observe by status, cancel, + receipt by artifact. That contract caps admission at shadow and report-only, + which is the advisory ceiling the issue requires. + +Suggested acceptance criteria for the hosted tier, for #14111 to adopt or +change: + +1. Registered as a `pkg/extwork` adapter and passes its conformance tests. +2. The PR states which container network policy it uses and, per row of the + egress table, why the `POST /pulls` and `PUT /pulls/{n}/merge` denials + still hold. Any direct GitHub write path caps the engine at `report-only`, + enforced in code. +3. Metering: each response's `usage` becomes a token record keyed on + `response.id`; hosted-tool charges are recorded or the backend is shown as + partly unmetered. +4. Per-call `max_output_tokens` and `max_tool_calls` are set from the governor's + remaining budget, since there is no server-side spend cap. + +Against Claude Managed Agents, the OpenAI hosted loop has the larger gap on +confinement (no self-hosted environment type that keeps tool traffic on Hive's +proxy, and no platform budget), and the smaller gap on telemetry (model and +usage on every response). Which to prototype first is for #14111. diff --git a/src/docs/backend-omnigent-cli-contract.md b/src/docs/backend-omnigent-cli-contract.md new file mode 100644 index 0000000000..957e23ff1b --- /dev/null +++ b/src/docs/backend-omnigent-cli-contract.md @@ -0,0 +1,172 @@ +# Omnigent CLI Contract and Policy Composition + +This note records the observed contract of the **Omnigent** CLI ([omnigent-ai/omnigent](https://github.com/omnigent-ai/omnigent), Apache-2.0) against Hive's backend acceptance bar (`src/docs/backend-support-tiers.md`), covering the six required contract facts, safety flag enforcement, credential detection, local confinement posture, and policy composition with Hive. + +Tracking issue: [#14094](https://github.com/hivecommons/hive/issues/14094) (child issues: [#14119](https://github.com/hivecommons/hive/issues/14119), [#14121](https://github.com/hivecommons/hive/issues/14121), [#14122](https://github.com/hivecommons/hive/issues/14122)). + +--- + +## 0. What Hive actually launches + +Hive wires omnigent on the **interactive** contributor path only. `backend_perm_flag omnigent` (`config/backends.conf`) emits `run --server local`, so every launch path — the contributor tmux line (`bin/contributor-agent.sh`), the Justfile local-mode line and `bin/agent-launch.sh` argv — produces: + +```bash +omnigent run --server local [--model ] +``` + +Both words are load-bearing and neither is a sandbox: + +- **`run`** is the agent REPL. A bare `omnigent` on a TTY is rewritten by upstream `omnigent.cli:main` to `omnigent start` (spawns the background server/host and returns to the shell prompt — no agent), and a bare `omnigent --model X` is a root-level Click `UsageError` (exit 2). Without `run`, the tmux pane would show a shell prompt or help text and the relay would type the task into bash. +- **`--server local`** pins the session to a local Omnigent server. `omnigent login ` persists a `server:` default in `~/.omnigent/config.yaml`, and `omnigent run` falls back to it when `--server` is absent. A remote Omnigent server hands the user's GitHub token to its managed runners, which push branches and open PRs from outside the Hive container and egress proxy — exactly the path the precedence rule in §5 says must not exist. `local` is Omnigent's documented sentinel for `run` ("overriding any configured server default"; `--server ""` is the historical spelling). **Remote-server mode is unsupported by Hive**, and `contribute-check-backend omnigent` says so. + +omnigent is **not** in the relay's `HEADLESS_BACKENDS` (`bin/contributor-relay.js`) and is declared non-headless in `bin/test_backend_smoke.sh` and the Kubernetes capability map (`api_contribute_test.go`). The one-shot `omnigent run -p` shape in Facts 1–3 below is recorded from upstream's CLI source and docs so the contract is complete; Hive has **not** wired or verified it end to end. Wiring it would need a small relay extension, because `flagsAfterCommand` cannot place `-p ` last after `--model`. + +## 1. Six CLI Contract Facts + +### Fact 1: Non-interactive invocation + +> **Status: documented upstream, not wired or verified by Hive** (see §0). Hive launches the interactive REPL (`omnigent run --server local`), not this path. + +Omnigent provides a non-interactive entry point via `omnigent run` with the `-p` / `--prompt` option. Without `-p`, `omnigent run` launches an interactive prompt_toolkit / Rich REPL and browser co-drive. With `-p ""`, the CLI runs headlessly, sends one turn, writes the assistant text response to stdout, and exits. + +```bash +omnigent run --harness codex -p "Implement the requested feature" +``` + +Per upstream source and docs: +- Runs in headless mode (REPL and browser auto-open are bypassed when `-p` is set unless explicitly configured). +- `--no-session` (ephemeral mode) runs in-process with a temporary SQLite database and artifact directory that are cleaned up on process exit. +- `omnigent run -p "..."` runs the declared agent spec headlessly. + +### Fact 2: How a prompt is passed in + +The prompt is passed via the `-p` or `--prompt` CLI option. The positional argument `[AGENT]` to `omnigent run` is reserved for the agent spec YAML file or agent directory path; passing the prompt as a positional argument causes an `Agent path not found` error. + +```bash +# Correct: prompt via -p +omnigent run --harness claude -p "Explain this codebase" + +# Passing prompt as positional argument fails: +omnigent run "Explain this codebase" +# Output on stderr: +# Error: Agent path not found: Explain this codebase +# Exit code: 1 +``` + +### Fact 3: Exit codes on success, failure, and denial + +- **Success (0):** When the agent turn completes normally, the assistant response is emitted on stdout and the CLI exits with status `0`. +- **Execution Failure / Server Error (1):** When a run fails (e.g. runner disconnection without completed turn, agent spec resolution error, or uncaught server exception), Omnigent outputs `Error: ` to stderr and exits with status `1`. +- **Policy Denial (1 or handled):** When a tool-call policy emits `DENY`, the policy engine rejects the action and returns an error payload to the agent model turn. If the overall turn or session fails as a result, or an admission policy blocks session initialization, the command exits with code `1`. +- **CLI Option / Argument Parsing Error (2):** Omnigent uses Click. Passing invalid flags or illegal values produces a `UsageError` on stderr and exits with status `2`. + +```bash +# Invalid option: +omnigent run --invalid-safety-flag +# Output on stderr: +# Error: no such option: --invalid-safety-flag +# Exit code: 2 +``` + +### Fact 4: Transcript written to disk and where + +- **Persistent sessions:** By default, Omnigent persists sessions, turns, and artifacts in an SQLite store under `${HOME}/.omnigent/` (`omnigent.db` and runner artifacts). +- **Session logs (`--log`):** In interactive mode, passing `--log` writes full JSON conversation dumps to `${HOME}/.omnigent/logs/`. +- **Event tapes (`--debug-events`):** When enabled, SSE event logs are streamed to `${HOME}/.omnigent/debug/`. +- **Ephemeral mode (`--no-session`):** Uses an isolated temporary directory (`tmpdir`) for SQLite storage and session state, which is unlinked on completion. + +### Fact 5: Model and cost reporting + +- **Model selection:** Specified via `--model ` or in the agent spec (`executor.model`). +- **Cost tracking & budgets:** Omnigent includes builtin cost guardrail policies (`omnigent.policies.builtins.cost.cost_budget` and `user_daily_cost_budget`). These track cumulative USD spend per session or per user day across both LLM request turns and tool calls, triggering `ASK` soft checkpoints or `DENY` hard caps on expensive model tiers. +- **Usage reporting:** Operators and scripts query cumulative metrics via the `omnigent usage` command. + +### Fact 6: Tool-denial list and policy enforcement + +Omnigent features a multi-tiered policy engine evaluating at three phases: session, agent spec, and server-wide. Policies evaluate at tool-call checkpoints and emit one of three verdicts: +- `ALLOW`: The tool call proceeds. +- `DENY`: The tool call is blocked; the agent receives an error payload. +- `ASK`: The tool pauses for user approval. + +Builtin tool policies include: +- `ask_on_os_tools`: Requires approval before OS file read, write, edit, or shell execution. +- `block_skills`: Denies loading specific skill bundles. +- `max_tool_calls_per_session`: Denies tool invocations once a count limit is reached. +- `github_policy`: Restricts GitHub MCP operations and `git`/`gh` shell commands to an allowlist of repositories and branches. + +--- + +## 2. Safety Flag Enforcement + +Per Criterion 3 of `src/docs/backend-support-tiers.md`, CLI safety flags must be actively parsed and enforced: an invalid option must exit non-zero rather than being silently ignored. + +Because Omnigent is implemented with Click, unrecognized options and invalid flag values fail immediately at argv parse time with exit code `2`: + +```bash +$ omnigent run --dangerously-bypass-everything +Error: no such option: --dangerously-bypass-everything +[exit status: 2] +``` + +Unattended mode is triggered by `-p` / `--prompt` (with no interactive REPL). Unattended runs must not configure policies that emit `ASK`, as an unanswered prompt in a headless container would stall. Hive does not currently wire this mode (§0). + +--- + +## 3. Credential Detection (`detect_cli`) + +Omnigent provides a standard `--version` flag: +```bash +$ omnigent --version +omnigent, version 0.2.0 +[exit status: 0] +``` +`omnigent --version` is an eager Click callback that exits `0` even when no credentials exist on the host. Probing `--version` alone would falsely report `OK` for unauthenticated environments. + +To provide honest credential detection, `detect_cli` in `bin/contributor-agent.sh` checks for valid authentication sources: +1. Ambient provider API keys: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`. +2. Databricks profile credentials: `DATABRICKS_TOKEN`, or non-empty `${HOME}/.databrickscfg`. +3. Omnigent session credentials: non-empty `${HOME}/.omnigent/auth_tokens.json` or `${HOME}/.omnigent/config.yaml`. + +If the binary exists but none of these credential paths are present, `detect_cli` outputs `NOT_AUTHED`. + +**Container staging is an allowlist, never the directory.** `just contribute-hive omnigent` stages only `~/.omnigent/auth_tokens.json` and `~/.omnigent/config.yaml` into the container's `~/.omnigent`, plus `~/.databrickscfg` as a single file when present, and forwards `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `DATABRICKS_TOKEN` and `OMNIGENT_API_KEY` by name — exactly the sources `detect_cli` accepts, so the container's verdict matches the host preflight. The rest of the directory — `omnigent.db` (every prior session and turn), `logs/` and `debug/` (Fact 4) — is the contributor's full Omnigent history and stays on the host, for the same reason `~/.claude` is staged as an allowlist (#7836). A `server:` key in the staged `config.yaml` is inert: the launch line's `--server local` overrides it (§0). + +--- + +## 4. Confinement Posture and Refusal Gate + +- **Local confinement posture:** `postureRefusalGated` (`refusal-gated`). +- **Support tier:** T3 (experimental). +- **Reasoning:** On the contributor local launch path (`just contribute-hive omnigent local`), the bare `omnigent` CLI runs directly on the host. While Omnigent supports an `enforce_sandbox` policy (using bubblewrap on Linux or seatbelt on macOS) in agent specifications, and supports external remote sandboxes (e.g. Modal, Daytona), Hive does not wire an OS-level filesystem sandbox directly through launch flags. Local mode therefore **refuses to launch** by default and requires the explicit opt-in: + ```bash + export HIVE_OMNIGENT_DANGEROUSLY_RUN_UNCONFINED=1 + ``` + Container mode (`just contribute-hive omnigent`, the default) provides standard container boundary isolation and requires no escape hatch. + +--- + +## 5. Policy Composition with Hive + +Omnigent functions as an open-source meta-harness over Claude Code, Codex, Cursor, Pi, and custom agent runtimes. Because both Omnigent and Hive feature policy layers, composition must follow strict precedence rules: + +```mermaid +graph TD + Agent[Model Agent Action] --> OmnigentPolicy[Omnigent Policy Engine] + OmnigentPolicy -- DENY --> Rejected[Action Blocked / Error to Model] + OmnigentPolicy -- ALLOW --> HiveProxy[Hive Egress & API Proxy] + HiveProxy -- Denied by Hive --> ProxyBlock[HTTP 403 / Push Refusal] + HiveProxy -- Approved --> GitHub[GitHub API / Git Remote] +``` + +### Precedence Rules + +1. **Hive policy strictly wins on GitHub writes.** + This holds only while the Omnigent server runs inside the Hive container, which is why every launch pins `--server local` (§0); a remote Omnigent server would route GitHub writes through its own runners and is unsupported. + All repository interactions (pull request creation `POST /repos/{owner}/{repo}/pulls`, merge requests `PUT /repos/{owner}/{repo}/pulls/{n}/merge`, issue comments, releases, and Git pushes) are routed through Hive's egress proxy and GitHub App authentication broker. + Even if an Omnigent policy (e.g. `github_policy`) or agent bundle permits a write, Hive's proxy checks (DCO trailers, ACMM tier gates, merge policy) remain authoritative. A proxy denial produces an HTTP 403 or push refusal that Omnigent cannot bypass. + +2. **Omnigent's sandbox is additive, never a substitute for Hive proxy or container isolation.** + Omnigent's optional sandbox (`enforce_sandbox`) or managed cloud sandboxes add host-level defense-in-depth. However, they do not replace Hive's API proxy or the contributor container boundary. + +3. **Underlying harness credential flow.** + Omnigent orchestrates underlying harnesses like Claude Code or Codex. In Hive, provider credentials reach the harness via ambient environment variables or staged configs. At no point are raw GitHub personal access tokens or administrative tokens exposed to Omnigent or its child harnesses. diff --git a/src/docs/backend-support-tiers.md b/src/docs/backend-support-tiers.md index 8a4c1070f5..8ecfc11ef2 100644 --- a/src/docs/backend-support-tiers.md +++ b/src/docs/backend-support-tiers.md @@ -15,14 +15,14 @@ implementation. ## The three tiers -The 12 shell-known CLIs (`KNOWN_BACKENDS` in `config/backends.conf`) already +The 15 shell-known CLIs (`KNOWN_BACKENDS` in `config/backends.conf`) already fall into three classes that are enforced in code. This document names them. | Tier | Meaning | Backends today | Enforcement point | | --- | --- | --- | --- | | **T1 - core (headless-pod)** | Runs unattended in a TTY-less pod from staged or env-injected credentials | `claude`, `litellm`, `copilot`, `codex`, `goose` | `HEADLESS_BACKENDS` in the `Justfile` (`contribute-k8s`) and `K8S_HEADLESS_BACKENDS` in `src/pkg/dashboard/api_contribute.go` | | **T2 - supported (confined)** | Has a confinement floor hive can wire on the contributor local path: an OS sandbox, or at least a host-state deny-list | `claude`/`litellm`, `codex`, `copilot`, `muse` (sandboxed); `opencode` (deny-listed) | `backend_perm_flag` and the `*_local_perm_flag_shell` helpers in `config/backends.conf` | -| **T3 - experimental (unconfined)** | No confinement hive can wire; local mode refuses to launch without a per-backend opt-in, container mode is the default | `goose`, `agy`, `bob`, `pi`, `aider`, `kilo`, `omp` | `unconfined_local_perm_flag_shell` and `HIVE__DANGEROUSLY_RUN_UNCONFINED=1` (#4918) | +| **T3 - experimental (unconfined)** | No confinement hive can wire; local mode refuses to launch without a per-backend opt-in, container mode is the default | `goose`, `agy`, `bob`, `pi`, `aider`, `kilo`, `omp`, `openhands`, `omnigent` | `unconfined_local_perm_flag_shell` and `HIVE__DANGEROUSLY_RUN_UNCONFINED=1` (#4918) | Tier assignment is per launch path. `goose` is T1 on the pod path (it has a one-shot `goose run` and an env-key credential) and T3 on the local path (no @@ -35,6 +35,8 @@ Upstream considers that CLI/headless path legacy and no longer actively maintained (OpenHands/OpenHands#18046), and Hive will not promote OpenHands beyond T3 on that surface. +`omnigent` is T3 experimental (open-source meta-harness over Claude Code, Codex, Cursor, Pi, and custom agents). On the contributor local launch path, the bare CLI runs with no OS sandbox wired by Hive, so local launches refuse by default without `HIVE_OMNIGENT_DANGEROUSLY_RUN_UNCONFINED=1`. While Omnigent has its own internal policy engine and optional agent sandboxing (`enforce_sandbox`), Hive treats these as additive: Hive's API/egress proxy strictly decides and controls all GitHub writes (`POST /repos/{owner}/{repo}/pulls`, `PUT /repos/{owner}/{repo}/pulls/{n}/merge`, Git pushes), and Omnigent policies can never override or bypass Hive proxy denials. See [backend-omnigent-cli-contract.md](backend-omnigent-cli-contract.md). + The T3 tier also bounds what an already-running OMP workbench may take as an external host (#8361 step 9, #6899): it is admissible only in report-only and shadow modes, where it holds no repository credential and no dashboard token @@ -43,6 +45,10 @@ and returns a receipt Hive verifies rather than trusts, and the adapter code. The escape-hatch env var above concerns launching `omp` locally under Hive; it does not widen what an external OMP host may do. +Hosted agent runtimes (Claude Managed Agents, the OpenAI Agents API) are not +CLI backends either. They have their own +[hosted execution tier](#hosted-execution-tier) below. + `gemini` is Go-only (`cliBackendExceptions`): it is launched by the hub-side manager and has no contributor-relay wiring at all, so the local-path tiers do not apply to it. Adding it to the relay would go through this bar as a new @@ -155,7 +161,7 @@ sandbox, no deny-list, and no unattended-credential story still lands here. `CLAUDE_HOST_DENY_TOOLS` (privilege escalation plus the boot/deployment tools named in #4918), with a `HIVE__DANGEROUSLY_ALLOW_HOST_STATE` opt-out. `TestOpencodeLocalModeDeniesHostStateCommands` and - `TestShellAndGoDenyListsAgree` (`src/pkg/agent/host_state_deny_test.go`) + `TestShellAndGoDenyListsAgree` (`src/crux/agent/host_state_deny_test.go`) are the templates. The local-mode banner must not call a deny-listed backend "confined" (`TestLocalModeBannerNeverCallsOpencodeConfined`). @@ -248,6 +254,201 @@ case, entirely hermetically (no network, no sleeps) as an ordinary PR-time Go test rather than a new CI workflow arm. +## Hosted execution tier + +Vendors now sell hosted agent runtimes: a platform that runs the agent loop, +keeps its state, and executes its tools in a sandbox the vendor operates. +[#14093](https://github.com/hivecommons/hive/issues/14093) asked whether Hive +can drive two of them, Claude Managed Agents and the OpenAI Agents API, as +backends. This section records the decision +([#14111](https://github.com/hivecommons/hive/issues/14111)), made from two +gap analyses: + +- Claude Managed Agents: + [backend-gap-claude-managed-agents.md](backend-gap-claude-managed-agents.md) + ([#14106](https://github.com/hivecommons/hive/issues/14106)). +- OpenAI Agents API: the note proposed in + [#14130](https://github.com/hivecommons/hive/pull/14130) + ([#14108](https://github.com/hivecommons/hive/issues/14108)). + +Neither existing on-ramp fits a hosted runtime: + +- **Not CLI-style.** Criteria 1, 2 and 8 assume a local binary whose flags + Hive controls. A hosted runtime has no binary. A name in `KNOWN_BACKENDS` + would make the shell launch path try to exec it. +- **Not gateway-path.** That tier routes the Claude CLI's model calls through + `pkg/proxy/anthropic_translate.go`. A hosted runtime runs the whole loop, + tools included, so the gateway rows (model picker, login, terminal) do not + describe it. + +What does fit is the `pkg/extwork` external-host contract that Flue, OMP and +vibe-kanban already use (see +[design/external-workflow-admission.md](design/external-workflow-admission.md)): +start keyed by execution key, observe by status, cancel, and a receipt fetched +by artifact path and checked against a declared digest. So a hosted runtime +gets its own tier, **hosted execution**. It has no T1/T2/T3 assignment and is +never added to `KNOWN_BACKENDS` or `CLIBackends`. It is an engine adapter +under `pkg/extwork`, switched on per engine under `runs.external.` in +`hive.yaml`, like the existing engines. + +### Decision per runtime + +| Runtime and shape | Placement | Ceiling | +| --- | --- | --- | +| Claude Managed Agents, `self_hosted` environment whose worker runs in a Hive pod with forced egress | Hosted execution | `report-only` at entry; can be promoted | +| Claude Managed Agents, `cloud` environment, credential-less: `networking: limited` with no GitHub host, no vaulted GitHub secret, no GitHub MCP server, web tools off; repo delivered as a file resource or under a read-only one-repo token | Hosted execution | `report-only` at entry; can be promoted | +| Claude Managed Agents, any setup with a direct GitHub write path: a write token on the `github_repository` resource, a vaulted GitHub secret, or the GitHub MCP server | Hosted execution | **Advisory for good** (`shadow` or `report-only`) | +| OpenAI hosted loop (Responses API with a hosted container), credential-less: no network or an allowlist without GitHub, no GitHub token, no GitHub MCP server, writes only as Hive-answered function tools | Hosted execution | `report-only` at entry; can be promoted | +| OpenAI hosted loop where the container can reach GitHub with a write credential, or a remote MCP server holds one | Hosted execution | **Advisory for good** | +| OpenAI Agents SDK loop in a Hive pod with local tools | CLI-style | Not a new backend. `codex` already covers it. Any other SDK-based CLI goes through the T1-T3 bar above. | + +### Advisory-mode ceiling + +The GitHub proxy hard-denies `POST /repos/{o}/{r}/pulls` and +`PUT /repos/{o}/{r}/pulls/{n}/merge` for every agent mode (`denyRules` in +`src/crux/proxy/rules.go`). Those denials only bind traffic that the iptables +redirect forces through the proxy +([net-admin-requirement.md](net-admin-requirement.md)). Traffic from a hosted +sandbox does not pass through it. + +A hosted sandbox **egresses GitHub directly** when it can reach GitHub with a +write-capable credential by a path Hive's proxy does not see: from the +sandbox itself, through a vendor-side git or MCP proxy, or through a secret +the vendor substitutes at its own egress. **Such a backend cannot exceed +advisory mode**, meaning `shadow` or `report-only` (`ModeShadow` and +`ModeReportOnly` in `src/pkg/extwork/admission.go`). No promotion lifts this +ceiling. The only way past it is a setup with no direct path. The adapter +enforces the ceiling in code. Before `Start`, it checks the session or +request configuration and refuses a write-capable stage if any direct path is +present. It refuses; it does not downgrade, matching how the existing +engines treat an item bound to an engine they cannot honour. + +Two consequences: + +- Today `pkg/extwork` has no write-capable mode at all, so every hosted + backend enters at advisory regardless of shape. The ceiling above is the + part that still holds after such a mode exists. +- A read-only, one-repo token is not a write path, but it is still a GitHub + credential held by a third party. It must be a short-lived installation + token scoped to one repository with `contents: read`, minted per run. It is + never the App's own key and never a broad PAT. + +### Acceptance criteria + +These replace criteria 1-12 for a hosted backend. They are numbered H1-H6 so +a PR can cite them next to the CLI criteria without confusion. + +- **H1. Engine adapter.** The runtime is an `extwork.Adapter` registered in + `src/pkg/extwork/registry.go`. It is linked into `hive` only behind its own + `extwork_` build tag, and offered only to relays that declare its + `ext-exec/` capability. It passes the #8201 conformance rows that + the Flue and OMP adapters pin in their `conformance_test.go` (admitted task + and replay, held task starts nothing, capability and version refused, + conflict and recreated engine, crash windows, cancel ignored and late + success, outage and waiting, artifact and verifier refusals, store write + fails). `Start` is idempotent on the execution key: the adapter stores the + vendor's session or response ID against the key (vendor `metadata` can hold + it) and reconciles before any retry. +- **H2. Egress statement and refusal.** The PR names the environment type or + container network policy it uses. For each row of that runtime's egress + table in its gap note, it says why the `POST /pulls` and + `PUT /pulls/{n}/merge` denials still hold. Tests pin that the adapter + refuses a write-capable stage for each direct-path row. +- **H3. Default off.** `runs.external.` defaults to off and to + `shadow` when enabled with no mode, the same as `FlueBindingConfig`. It + dispatches only in an explicitly chosen mode. This is the hosted + counterpart of criterion 8's refusal gate. +- **H4. Credential handling.** Before dispatch, an authenticated read proves + the vendor key works (for example `GET /v1/environments/{id}` or + `GET /v1/models`), the counterpart of criterion 4. The vendor key never + enters the sandbox. A Managed Agents `self_hosted` worker holds only its + environment key and the per-session work secret, never `ANTHROPIC_API_KEY`. +- **H5. Metering and spend cap.** Each model call becomes a token record in + `pkg/tokens`, deduplicated on the vendor's ID (the `span.model_request_end` + event ID, or `response.id`), with the model recorded at dispatch when the + usage record omits it. Non-token charges (session run time, web searches, + hosted-tool calls) are recorded, or the dashboard shows the backend as + partly unmetered, under the policy in + [Metering coverage and budget-gated hives](#metering-coverage-and-budget-gated-hives). + On a budget-gated hive the dispatch caps spend from the governor's + remaining budget. For Managed Agents that is the session `budget`; for + OpenAI it is per-call `max_output_tokens` and `max_tool_calls`, and the + adapter stops chaining calls when the governor says so. +- **H6. Pins, docs and changelog.** The request pins what the runtime runs: the + agent version and toolset version for Managed Agents, and the model + snapshot and tool version for OpenAI. The PR adds the documentation rows + listed below and a changelog fragment (criterion 7). + +### Promotion beyond advisory + +A hosted backend in a setup with no direct path may later run a stage beyond +`report-only`. Promotion is a PR that supplies all of: + +1. A write-capable `pkg/extwork` mode, designed in its own issue, in which the + change still reaches GitHub only through Hive's relays (`hive-open-pr`). + The sandbox never writes to GitHub itself. +2. Live evidence from a real run: the vendor refusing an invalid network + policy or tool configuration with an error instead of ignoring it (the + counterpart of criterion 3), a test showing both proxy denials still hold, + and metering for that run reconciled against the vendor's own total + (`session.usage.list_cost` or the organisation usage API). + +A setup with a direct path is never promoted. + +### Which backend is prototyped first + +**Claude Managed Agents, with a `self_hosted` environment**, has the smaller +gap: + +- Tool traffic goes through Hive's own proxy, so the two denials hold through + the existing `denyRules`. No new argument about who holds GitHub tokens is + needed. +- A session `budget` is a hard spend cap that the platform checks before every + model request. +- Agent, environment and session objects map directly onto `Start`, `Observe` + and `Cancel`. The durable, paginated event list is the transcript. + +The OpenAI hosted loop has the smaller gap on telemetry: model and usage are +on every response. But it has no environment type that keeps tool traffic on +Hive's proxy, no server-side spend cap, and hosted-tool charges are missing +from `usage`. Its SDK-in-pod shape is already the `codex` backend. + +The prototype is [#14124](https://github.com/hivecommons/hive/issues/14124). +It is implemented as the `claude-managed-agents` `pkg/extwork` adapter: +`runs.external.claude_managed_agents` is default off, `shadow` when enabled +without a mode, and dispatches only in explicit `report-only` mode. The adapter +is linked only with `-tags extwork_claude_managed_agents`, requires a +`self_hosted` environment and `forced-egress` or `proxy-only` policy, and +stores vendor session bindings under the extwork record directory. It does not +touch `KNOWN_BACKENDS`, `CLIBackends`, or CLI parity/posture tests. + +### Follow-ups + +These are filed as #14124 completes, as its checklist requires: + +- **Second backend.** The OpenAI hosted loop as a hosted-execution adapter in + the credential-less shape, under H1-H6, after #14124 lands. +- **Tier promotion.** The write-capable `pkg/extwork` mode and the live + evidence that the promotion criteria above call for. + +### Rows shipped for the hosted prototype + +The first hosted row is now present in +[`agent-configuration.md`](agent-configuration.md#methods-subscription-clis-vs-self-hosted-inference): + +- **Methods table:** `hosted / claude-managed-agents`, authenticated by an API + key reference (`api_key_env` or `api_key_file`) and pinned agent/model + metadata. There is no CLI login or model discovery. +- **Engine block:** a hosted engine is not a `backend:` value. It is selected + under `runs.external.claude_managed_agents`. +- **Per-engine block:** the endpoint, workflow/agent/toolset versions, pinned + `self_hosted` environment, forced-egress policy, report-only mode, state + directory, and USD 25 daily ledger cap are documented. Token events are + metered in `pkg/tokens`; cumulative session list cost, runtime, and web + search usage are persisted in the managed-agents ledger. The adapter stops + at the ledger cap and the vendor session budget is set to the remaining + cents, so the hosted path is not treated as unmetered. + ## Metering coverage and budget-gated hives Backend support is also a metering contract. Budget gates in `pkg/governor` @@ -265,6 +466,7 @@ Metered sources today are the scanner and sink implementations in `pkg/tokens`: | `copilot_scanner.go` | Copilot CLI `events.jsonl` | Reads `session.shutdown.modelMetrics` and avoids double-counting live proxy captures. | | `bob_scanner.go` | Bob CLI chat recordings | Uses Bob's explicit token fields, falling back to content-size estimates only when a recording lacks token data. | | `inference_sink.go` | vLLM, llm-d, LiteLLM, live Copilot proxy usage | Hive-written JSONL under the metrics directory. | +| `managed_agents_ledger.go` | Claude Managed Agents self_hosted sessions | Deduplicates `span.model_request_end` IDs into collector JSONL and persists cumulative USD list cost, active seconds, and web searches with a hard USD 25 daily cap. | ### Tagging Bob sessions for Bobalytics @@ -336,6 +538,7 @@ agent spend, and the `advisor_records` admin MCP tool. | `kilo` | No | n/a | Nothing; reported not active | | `muse` | No | n/a | Nothing; reported not active | | `openhands` | No | n/a | Nothing; reported not active | +| `omnigent` | No | n/a | Nothing; reported not active | | `vllm` | No: inference backend, no turn-end hook | n/a | Nothing; reported not active | | `llm-d` | No: inference backend, no turn-end hook | n/a | Nothing; reported not active | | `litellm` | No: inference backend, no turn-end hook | n/a | Nothing; reported not active | diff --git a/src/docs/benchmarks/README.md b/src/docs/benchmarks/README.md new file mode 100644 index 0000000000..3e651f8e33 --- /dev/null +++ b/src/docs/benchmarks/README.md @@ -0,0 +1,44 @@ +# Outcome benchmarks + +Reproducible comparison of pull requests that went through Hive's review swarm (**mediated**) with pull requests merged without it (**unmediated**), computed from the per-PR evidence bundles a spoke already writes under `/data/evidence`. No new telemetry; every rate carries its n and a 95% Wilson interval, and n < 10 is printed as `not reported`. + +## Reports + +| Month | Report | Charts | +|---|---|---| +| 2026-10 | [outcomes-2026-10.md](outcomes-2026-10.md) | [bars](assets/outcomes-2026-10-bars.svg) · [trend](assets/outcomes-2026-10-trend.svg) | + +Each report ships with `outcomes-YYYY-MM.json`, the computed `Report` the markdown, charts and README block are rendered from. `pkg/outcomebench` `TestCommittedReportMatchesReadme` re-renders the newest JSON and fails when the committed markdown, SVGs or the repository README's `…` block differ, so none of them can drift. + +## Regenerate + +```sh +# on a spoke, or anywhere with a copy of its evidence tree +src/scripts/outcome-benchmark.sh --root /data/evidence --month 2026-10 \ + --spend cost.json # saved GET /api/cost, for cost per merged PR + --hub hub.json # saved GET /api/hub/evidence (admin session), optional + --source "hive-oke spoke snapshot 2026-10-27" \ + --readme # also rewrite README.md's outcomes block +``` + +The script wraps `hivectl evidence benchmark`; see [hivectl.md](../hivectl.md#evidence--benchmark) for every flag. To take a read-only copy of a spoke's evidence: `kubectl exec deploy/hive -c hive -- tar czf - -C /data evidence | tar xzf -`. + +## Metrics + +| Metric | Definition | +|---|---| +| Revert rate | merged PRs the evidence linker tagged `reverted_by` ÷ merged PRs observed ≥ 14 days after merge | +| Change-failure rate | `reverted_by` or `followed_up_by` ÷ the same matured denominator | +| Median time-to-merge | lower median of merge − open time over merged PRs whose bundle recorded an open time | +| CI failure at merged head | captured `failure` ÷ captured success/failure on merged heads; not comparable when a cohort has a captured status on < 50 % of its merges | +| Changes requested pre-merge | mediated merged PRs with ≥ 1 `changes_requested` verdict on any head before the merge | +| Cost per merged PR | `/api/cost` estimated spend ÷ its hive-attributed merged-PR count, same window | + +A PR is mediated when any of its heads carries a Hive verdict or a posted Hive review. Backfilled history is unmediated by construction, so the window must start no earlier than the first mediated merge or the comparison is confounded by time; the report states that date. + +## Caveats + +- Revert and follow-up links depend on the evidence linker having seen the later PR; a spoke that was down misses them. +- CI status is captured only for heads Hive reviewed, so the unmediated CI cell is a small sample until the sweeper records it for every merged head. +- Cost is an estimate: model prices are approximate and some models are unpriced. +- The hub's `GET /api/hub/evidence` needs an admin session; when it is not supplied the report says so and the numbers are one spoke's. diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-bars.svg b/src/docs/benchmarks/assets/outcomes-2026-10-bars.svg new file mode 100644 index 0000000000..1c6e386026 --- /dev/null +++ b/src/docs/benchmarks/assets/outcomes-2026-10-bars.svg @@ -0,0 +1,36 @@ + +Mediated vs unmediated — 2026-10 (95% Wilson intervals) +Mediated vs unmediated — 2026-10 (95% Wilson intervals) +mediated +unmediated +0% +10% +20% +30% +40% + + +0.0% (n=43) + + +0.1% (n=1513) +Revert rate + + +2.3% (n=43) + + +1.1% (n=1513) +Change-failure rate + + +6.9% (n=798) +n=62 +CI failure at head + + +33.7% (n=1455) +Changes requested pre-merge + +Hatched = n < 10 or CI not captured for most of the cohort: not reported. Change-failure = revert or follow-up fix within 14 days. + diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-trend.svg b/src/docs/benchmarks/assets/outcomes-2026-10-trend.svg new file mode 100644 index 0000000000..c811f256ea --- /dev/null +++ b/src/docs/benchmarks/assets/outcomes-2026-10-trend.svg @@ -0,0 +1,43 @@ + +Median time-to-merge by month (hours) — revert rate annotated +Median time-to-merge by month (hours) — revert rate annotated +mediated +unmediated +0h +0.1h +0.2h +0.3h +0.4h +0.5h +2026-04 +2026-05 +2026-06 +2026-07 +2026-08 +2026-09 +2026-10 + +0.01h · rev 0.0% + +0.01h · rev 0.0% + + +0.01h · rev 0.1% + + +0.09h · rev 0.0% + + +0.31h · rev 0.1% + + +0.34h · rev 0.1% + + +0.19h · rev 0.1% + + +0.44h · rev 0.0% + +Months with n < 10 are gaps. Mediated series starts at the first mediated merge. + diff --git a/src/docs/benchmarks/outcomes-2026-10.json b/src/docs/benchmarks/outcomes-2026-10.json new file mode 100644 index 0000000000..2e74e841b2 --- /dev/null +++ b/src/docs/benchmarks/outcomes-2026-10.json @@ -0,0 +1,425 @@ +{ + "GeneratedAt": "2026-10-27T00:00:00Z", + "Since": "2026-10-01T00:00:00Z", + "Until": "2026-11-01T00:00:00Z", + "Now": "2026-10-27T00:00:00Z", + "Source": "hive's own spoke (hive-oke) evidence snapshot taken 2026-10-27", + "Bundles": 13060, + "Unreadable": 10, + "PRs": 11571, + "Repos": [ + { + "Repo": "hivecommons/hive", + "Merged": 2451 + }, + { + "Repo": "hivecommons/docs", + "Merged": 223 + }, + { + "Repo": "hivecommons/dibs", + "Merged": 179 + }, + { + "Repo": "hivecommons/rationguard", + "Merged": 172 + }, + { + "Repo": "hivecommons/pluk", + "Merged": 155 + }, + { + "Repo": "hivecommons/promptargs", + "Merged": 147 + }, + { + "Repo": "hivecommons/hivecommons.github.io", + "Merged": 145 + }, + { + "Repo": "hivecommons/hotshot", + "Merged": 122 + }, + { + "Repo": "hivecommons/homebrew-hive", + "Merged": 78 + }, + { + "Repo": "hivecommons/crux", + "Merged": 7 + } + ], + "Verdicts": 15054, + "Mediated": { + "Name": "mediated", + "Merged": 1455, + "Matured": 43, + "RevertRate": { + "K": 0, + "N": 43, + "Rate": 0, + "Lo": 0, + "Hi": 0.08201257002322722, + "Reported": true + }, + "ChangeFailure": { + "K": 1, + "N": 43, + "Rate": 0.023255813953488372, + "Lo": 0.004116981177419918, + "Hi": 0.12059267861216882, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.4418315681741667, + "N": 1455, + "Reported": true + }, + "CIFailureAtHead": { + "K": 55, + "N": 798, + "Rate": 0.06892230576441102, + "Lo": 0.05333222787551566, + "Hi": 0.08864294527588978, + "Reported": true + }, + "CICoverage": { + "K": 798, + "N": 1455, + "Rate": 0.5484536082474227, + "Lo": 0.5227885036932317, + "Hi": 0.5738635248751888, + "Reported": true + }, + "ChangesRequested": { + "K": 490, + "N": 1455, + "Rate": 0.33676975945017185, + "Lo": 0.312943612754814, + "Hi": 0.36145558178419573, + "Reported": true + }, + "ByAuthorKind": { + "agent": 857, + "bot": 70, + "human": 528 + } + }, + "Unmediated": { + "Name": "unmediated", + "Merged": 2224, + "Matured": 1513, + "RevertRate": { + "K": 1, + "N": 1513, + "Rate": 0.0006609385327164573, + "Lo": 0.00011667840798450777, + "Hi": 0.0037344817922631644, + "Reported": true + }, + "ChangeFailure": { + "K": 16, + "N": 1513, + "Rate": 0.010575016523463317, + "Lo": 0.006519662289637756, + "Hi": 0.01710943649035351, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.19166666666666668, + "N": 2224, + "Reported": true + }, + "CIFailureAtHead": { + "K": 17, + "N": 62, + "Rate": 0.27419354838709675, + "Lo": 0.17880918599092133, + "Hi": 0.39592775843327255, + "Reported": true + }, + "CICoverage": { + "K": 62, + "N": 2224, + "Rate": 0.02787769784172662, + "Lo": 0.021807488449304663, + "Hi": 0.035576124461056645, + "Reported": true + }, + "ChangesRequested": { + "K": 0, + "N": 2224, + "Rate": 0, + "Lo": 0, + "Hi": 0.0017243595774493123, + "Reported": true + }, + "ByAuthorKind": { + "agent": 1186, + "bot": 103, + "human": 935 + } + }, + "FirstMediatedMerge": "2026-10-09T18:04:32Z", + "Months": [ + "2026-04", + "2026-05", + "2026-06", + "2026-07", + "2026-08", + "2026-09", + "2026-10" + ], + "Trend": { + "mediated": [ + { + "Month": "2026-04", + "Merged": 0, + "RevertRate": { + "K": 0, + "N": 0, + "Rate": 0, + "Lo": 0, + "Hi": 0, + "Reported": false + }, + "TimeToMerge": { + "Value": 0, + "N": 0, + "Reported": false + } + }, + { + "Month": "2026-05", + "Merged": 0, + "RevertRate": { + "K": 0, + "N": 0, + "Rate": 0, + "Lo": 0, + "Hi": 0, + "Reported": false + }, + "TimeToMerge": { + "Value": 0, + "N": 0, + "Reported": false + } + }, + { + "Month": "2026-06", + "Merged": 0, + "RevertRate": { + "K": 0, + "N": 0, + "Rate": 0, + "Lo": 0, + "Hi": 0, + "Reported": false + }, + "TimeToMerge": { + "Value": 0, + "N": 0, + "Reported": false + } + }, + { + "Month": "2026-07", + "Merged": 0, + "RevertRate": { + "K": 0, + "N": 0, + "Rate": 0, + "Lo": 0, + "Hi": 0, + "Reported": false + }, + "TimeToMerge": { + "Value": 0, + "N": 0, + "Reported": false + } + }, + { + "Month": "2026-08", + "Merged": 0, + "RevertRate": { + "K": 0, + "N": 0, + "Rate": 0, + "Lo": 0, + "Hi": 0, + "Reported": false + }, + "TimeToMerge": { + "Value": 0, + "N": 0, + "Reported": false + } + }, + { + "Month": "2026-09", + "Merged": 0, + "RevertRate": { + "K": 0, + "N": 0, + "Rate": 0, + "Lo": 0, + "Hi": 0, + "Reported": false + }, + "TimeToMerge": { + "Value": 0, + "N": 0, + "Reported": false + } + }, + { + "Month": "2026-10", + "Merged": 1455, + "RevertRate": { + "K": 0, + "N": 43, + "Rate": 0, + "Lo": 0, + "Hi": 0.08201257002322722, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.4418315681741667, + "N": 1455, + "Reported": true + } + } + ], + "unmediated": [ + { + "Month": "2026-04", + "Merged": 184, + "RevertRate": { + "K": 0, + "N": 184, + "Rate": 0, + "Lo": 0, + "Hi": 0.02045127383923476, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.005555555555555556, + "N": 184, + "Reported": true + } + }, + { + "Month": "2026-05", + "Merged": 432, + "RevertRate": { + "K": 0, + "N": 432, + "Rate": 0, + "Lo": 0, + "Hi": 0.008814211401573416, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.013888888888888888, + "N": 432, + "Reported": true + } + }, + { + "Month": "2026-06", + "Merged": 943, + "RevertRate": { + "K": 1, + "N": 943, + "Rate": 0.0010604453870625664, + "Lo": 0.00018721436692565457, + "Hi": 0.005982349792486016, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.013333333333333334, + "N": 943, + "Reported": true + } + }, + { + "Month": "2026-07", + "Merged": 611, + "RevertRate": { + "K": 0, + "N": 611, + "Rate": 0, + "Lo": 0, + "Hi": 0.006248113335206986, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.08694444444444445, + "N": 611, + "Reported": true + } + }, + { + "Month": "2026-08", + "Merged": 1881, + "RevertRate": { + "K": 2, + "N": 1881, + "Rate": 0.001063264221158958, + "Lo": 0.00029162773813870926, + "Hi": 0.0038687218636527625, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.30833333333333335, + "N": 1881, + "Reported": true + } + }, + { + "Month": "2026-09", + "Merged": 3088, + "RevertRate": { + "K": 4, + "N": 3088, + "Rate": 0.0012953367875647669, + "Lo": 0.0005038347363301571, + "Hi": 0.0033261156725135554, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.33694444444444444, + "N": 3088, + "Reported": true + } + }, + { + "Month": "2026-10", + "Merged": 2224, + "RevertRate": { + "K": 1, + "N": 1513, + "Rate": 0.0006609385327164573, + "Lo": 0.00011667840798450777, + "Hi": 0.0037344817922631644, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.19166666666666668, + "N": 2224, + "Reported": true + } + } + ] + }, + "Spend": { + "usd": 6124.75144225, + "merged_prs": 2869, + "window_start": "2026-08-08T00:00:00Z", + "window_end": "2026-10-27T14:50:41.723Z", + "basis": "hive-attributed" + }, + "Hub": null +} diff --git a/src/docs/benchmarks/outcomes-2026-10.md b/src/docs/benchmarks/outcomes-2026-10.md new file mode 100644 index 0000000000..ffee1b9301 --- /dev/null +++ b/src/docs/benchmarks/outcomes-2026-10.md @@ -0,0 +1,63 @@ +# Outcome benchmark — 2026-10 + +Generated 2026-10-27 from hive's own spoke (hive-oke) evidence snapshot taken 2026-10-27. Window: merged 2026-10-01 → 2026-11-01; maturation 14 days (observed as of 2026-10-27). + +Pull requests that went through Hive's review swarm (**mediated**: at least one recorded verdict or posted review) versus pull requests merged without it (**unmediated**). Every rate carries its n and a 95% Wilson interval; anything with n < 10 is `not reported`. + +## Headline + +| Metric | Mediated | Unmediated | +|---|---|---| +| Merged PRs in window | 1455 | 2224 | +| Revert rate (matured) | 0.0% [0.0–8.2] (n=43) | 0.1% [0.0–0.4] (n=1513) | +| Change-failure rate (revert or follow-up fix, matured) | 2.3% [0.4–12.1] (n=43) | 1.1% [0.7–1.7] (n=1513) | +| Median time-to-merge | 0.44 h (n=1455) | 0.19 h (n=2224) | +| CI failure at merged head | 6.9% [5.3–8.9] (n=798) | 27.4% [17.9–39.6] (n=62) — captured on only 3% of merges, not comparable | +| Had a `changes_requested` verdict before merge | 33.7% [31.3–36.1] (n=1455) | — (no verdicts by definition) | +| Cost per merged PR | ≈$2.13 est. (n=2869 merged, 2026-08-08 → 2026-10-27) | — | + +![mediated vs unmediated](assets/outcomes-2026-10-bars.svg) + +### Reading the numbers + +- Revert rates are indistinguishable at this n: 0.0% [0.0–8.2] (n=43) mediated vs 0.1% [0.0–0.4] (n=1513) unmediated (overlapping intervals). +- Median time-to-merge is 0.44 h (n=1455) for mediated vs 0.19 h (n=2224) for unmediated: review adds about 15 minutes to the median PR. +- 33.7% [31.3–36.1] (n=1455) of mediated merged PRs were flagged `changes_requested` on an earlier head before merging — findings an unmediated PR would have carried to main unseen. +- CI at merged head is only captured for reviewed heads (mediated 6.9% [5.3–8.9] (n=798)); the unmediated cell is not comparable. +- Estimated agent spend is ≈$2.13 per hive-attributed merged PR (n=2869). + +## Trend by month + +Whole evidence history, not just the window; months with n < 10 are gaps. + +![trend](assets/outcomes-2026-10-trend.svg) + +| Month | Mediated merged | Mediated revert | Mediated TTM | Unmediated merged | Unmediated revert | Unmediated TTM | +|---|---|---|---|---|---|---| +| 2026-04 | 0 | not reported (n=0) | not reported (n=0) | 184 | 0.0% [0.0–2.0] (n=184) | 0.01 h (n=184) | +| 2026-05 | 0 | not reported (n=0) | not reported (n=0) | 432 | 0.0% [0.0–0.9] (n=432) | 0.01 h (n=432) | +| 2026-06 | 0 | not reported (n=0) | not reported (n=0) | 943 | 0.1% [0.0–0.6] (n=943) | 0.01 h (n=943) | +| 2026-07 | 0 | not reported (n=0) | not reported (n=0) | 611 | 0.0% [0.0–0.6] (n=611) | 0.09 h (n=611) | +| 2026-08 | 0 | not reported (n=0) | not reported (n=0) | 1881 | 0.1% [0.0–0.4] (n=1881) | 0.31 h (n=1881) | +| 2026-09 | 0 | not reported (n=0) | not reported (n=0) | 3088 | 0.1% [0.1–0.3] (n=3088) | 0.34 h (n=3088) | +| 2026-10 | 1455 | 0.0% [0.0–8.2] (n=43) | 0.44 h (n=1455) | 2224 | 0.1% [0.0–0.4] (n=1513) | 0.19 h (n=2224) | + +## Scale + +- 13060 evidence bundles (10 unreadable) folding to 11571 pull requests across 10 repositories; 15054 verdicts recorded. +- Merged in window by repository: hivecommons/hive 2451, hivecommons/docs 223, hivecommons/dibs 179, hivecommons/rationguard 172, hivecommons/pluk 155, hivecommons/promptargs 147, hivecommons/hivecommons.github.io 145, hivecommons/hotshot 122, hivecommons/homebrew-hive 78, hivecommons/crux 7. +- Merged in window by author kind — mediated: agent 857, bot 70, human 528; unmediated: agent 1186, bot 103, human 935. +- First mediated merge in the evidence: 2026-10-09. +- Hub `GET /api/hub/evidence` was not consulted (it requires an admin session); numbers are this spoke's evidence only. + +## Methodology + +- **Source.** Only the per-PR evidence bundles the spoke already writes under `/data/evidence` (schema v1), folded per PR with `pkg/evidence.FoldPRs`. No new telemetry. +- **Cohorts.** A PR is *mediated* if any head carries a Hive verdict or a posted Hive review; otherwise *unmediated*. Backfilled history is unmediated by construction, so the window must start no earlier than the first mediated merge or the comparison is confounded by time. +- **Revert / change-failure.** Denominator is merged PRs observed at least 14 days after merge. A revert is a later PR the evidence linker tagged `reverted_by`; change-failure adds `followed_up_by` fixes. +- **Time-to-merge.** Lower median of merge time minus open time, over merged PRs whose bundle recorded an open time. +- **CI failure at head.** Share of merged heads with a captured `failure` status among heads with a captured success/failure. The unmediated cohort rarely has a captured status (Hive only records CI for heads it reviews), so its cell reports coverage and is not comparable when coverage is low. +- **Changes requested.** Share of mediated merged PRs with at least one `changes_requested` verdict on any head before the merge — findings the author had a chance to address pre-merge. Whether the later head fixed the finding, or a human overrode it, is not verified here. +- **Cost.** `/api/cost` estimated spend divided by its hive-attributed merged-PR count over the same window; model prices are estimates and some models are unpriced, so treat it as an upper-bound order of magnitude. +- **Intervals.** 95% Wilson score; n < 10 is never reported. +- **Regenerate.** `src/scripts/outcome-benchmark.sh --month 2026-10` (see `docs/benchmarks/README.md`). diff --git a/src/docs/cncf-reference-architecture.md b/src/docs/cncf-reference-architecture.md index dd9f76c5d0..131a128966 100644 --- a/src/docs/cncf-reference-architecture.md +++ b/src/docs/cncf-reference-architecture.md @@ -132,7 +132,7 @@ Two ideas define the architecture: judgment calls (read code, reason about a fix, write a PR). 2. **Autonomy is a dial, enforced in depth.** An **AI-native Capability Maturity - Model (ACMM)** with six levels (advisory → fully autonomous) is the single + Model (ACMM)** with six levels (advisory → autonomous) is the single control a human turns. Each level maps deterministically to a per-agent permission mode, and that mode is enforced at three independent layers: CLI tool denial, least-privilege scoped GitHub App tokens, and a network-level diff --git a/src/docs/contributor-relay.md b/src/docs/contributor-relay.md index 53f7d57de9..137dbf44db 100644 --- a/src/docs/contributor-relay.md +++ b/src/docs/contributor-relay.md @@ -350,7 +350,7 @@ Important environment variables: | `HIVE_COMMONS_STRATEGY` | `ranked` | Multi-hive routing strategy for The Commons. `ranked` solicits the first subscribed hive and falls through only when it has no work; `spread` rotates with rank weights and periodic mixing; `neediest` scores each hub's `/api/contribute/status` by `actionable_items` with a small boost for idle contributor capacity (`total_registered - active_contributors`) when those fields are present. The relay evaluates this only when it is ready for another task, never while a lease is in flight. | | `HIVE_COMMONS_SPREAD_MIX_EVERY` | `7` | In `spread`, every N completed/failed tasks the next hive is chosen randomly from the weighted rank cycle instead of by cursor. Set `0` to disable mixing. | | `HIVE_COMMONS_NEEDIEST_REFRESH_MS` | `60000` | In `neediest`, how often to refresh each subscribed hub's `/api/contribute/status` cache. Set `0` to rely on auth-time/manual refreshes only. | -| `AGENT_BACKEND` | `claude` | CLI/backend to run (`claude`, `copilot`, `goose`, `bob`, `codex`, `pi`, `aider`, `litellm`, `agy`, `opencode`, `kilo`, `muse`, `omp`, `openhands`, depending on image support and credentials). `omp` is interactive-only: Hive starts normal `omp --model ` in the prepared tmux cwd and passes no fabricated permission flags. It has no verified local confinement mechanism, so local mode refuses it without `HIVE_OMP_DANGEROUSLY_RUN_UNCONFINED=1`; container mode is the supported boundary. `agy` has the same confinement limit. `opencode`, `kilo`, `muse`, and `openhands` only run headless (`CONTRIBUTOR_MODE=headless`) — hive has no interactive-tmux wiring for them. | +| `AGENT_BACKEND` | `claude` | CLI/backend to run (`claude`, `copilot`, `goose`, `bob`, `codex`, `pi`, `aider`, `litellm`, `agy`, `opencode`, `kilo`, `muse`, `omp`, `openhands`, `omnigent`, depending on image support and credentials). `omp` is interactive-only: Hive starts normal `omp --model ` in the prepared tmux cwd and passes no fabricated permission flags. It has no verified local confinement mechanism, so local mode refuses it without `HIVE_OMP_DANGEROUSLY_RUN_UNCONFINED=1`; container mode is the supported boundary. `agy` and `omnigent` have the same confinement limit; `omnigent` is launched as `omnigent run --server local [--model ]` (`run` opens the agent REPL — bare `omnigent` only starts a background server — and `--server local` keeps the session off any remote server configured by `omnigent login`). `opencode`, `kilo`, `muse`, and `openhands` only run headless (`CONTRIBUTOR_MODE=headless`) — hive has no interactive-tmux wiring for them. | | `AGENT_MODEL` | unset (backend default) | Optional model override passed to the contributor agent (e.g. `claude-sonnet-4-6`, `gpt-4o`, `gemini-2.5-pro`). Declared to the hive when the relay connects. | | `AGENT_REASONING_EFFORT` | unset | Reasoning effort override. Consumed by `codex` (`-c model_reasoning_effort`), by `agy` (`--effort low\|medium\|high`, required whenever a model is set, else agy ignores the model), by `muse` (`--reasoning-effort none\|minimal\|low\|medium\|high\|xhigh\|max\|ultra`, applied with or without a model; a value outside that set is dropped rather than passed, because muse exits 2 on it), and by `claude` (`--effort low\|medium\|high\|xhigh\|max`, applied with or without a model; a value outside that set is dropped the same way, and unset leaves Claude Code at its own default - [#8377](https://github.com/hivecommons/hive/issues/8377)). Ignored by other backends, including inference routes such as `litellm` that drive the claude binary. | | `HIVE_CONTRIBUTOR_KNOWLEDGE_EXPORT_MAX_FACTS` | `80` | Maximum entries in the `/api/knowledge/export` response when it is fetched with a contributor registration token. Owner/API-token exports remain full; contributor startup context receives a bounded summary with a truncation marker and should fetch more specific entries on demand with `hive knowledge` or the Hive MCP knowledge tool. | @@ -387,6 +387,7 @@ Important environment variables: | `HIVE_KILO_DANGEROUSLY_RUN_UNCONFINED` | unset | **Required** for `just contribute-hive kilo local` to launch at all. kilo's `--auto` is an unattended auto-approve flag, not a boundary; kilo has no verified sandbox, filesystem allowlist, or command deny-list hive can wire. | | `HIVE_OPENHANDS_DANGEROUSLY_RUN_UNCONFINED` | unset | **Required** for `just contribute-hive openhands local` to launch at all. The bare `openhands` CLI runs the agent on the host with no sandbox; `--always-approve` is approval, not a boundary, and its Docker sandbox is reachable only through `openhands serve`, not the headless one-shot path. | | `HIVE_OMP_DANGEROUSLY_RUN_UNCONFINED` | unset | **Required** for `just contribute-hive omp local` to launch at all. OMP has no sandbox, filesystem allowlist, or command deny-list Hive can wire; local mode refuses to launch without this. | +| `HIVE_OMNIGENT_DANGEROUSLY_RUN_UNCONFINED` | unset | **Required** for `just contribute-hive omnigent local` to launch at all. Omnigent has no OS sandbox wired by Hive on the local CLI path; local mode refuses to launch without this. | ### Where each backend reads its instructions @@ -419,6 +420,7 @@ cannot serve it, the file stays absent and the relay reports a `~`-relative | `openhands` | `AGENTS.md`, `CLAUDE.md` | | `muse` | `AGENTS.md`, `CLAUDE.md` | | `omp` | `AGENTS.md`, `CLAUDE.md` | +| `omnigent` | `AGENTS.md`, `CLAUDE.md` | | anything else | `CLAUDE.md` only — the `*` fallback | A backend that reads neither `CLAUDE.md` nor one of the names above falls into @@ -467,7 +469,7 @@ OS-enforced — Seatbelt/bubblewrap/ProcessContainer depending on platform), gated on the installed CLI actually supporting the flag; opencode gets a command-name deny-list via its own `permission.bash` config (a floor, not a filesystem boundary — opencode has no OS sandbox); goose, agy, bob, pi, aider, -kilo, omp, and openhands have no confinement mechanism this repo can wire at all, and local +kilo, omp, openhands, and omnigent have no confinement mechanism this repo can wire at all, and local mode for them **refuses to launch** unless the operator sets that backend's own `HIVE__DANGEROUSLY_RUN_UNCONFINED=1`. See [sandbox-isolation.md](sandbox-isolation.md)'s per-backend confinement matrix @@ -524,6 +526,7 @@ The relay speaks to whatever backend you set up — pass it to `contribute-setup | `muse` | Muse Code (`curl -fsSL https://dev.meta.ai/install.sh | bash`). Headless-only: `muse exec ""` is its documented non-interactive sub-command. Auth is `META_API_KEY` (which muse says always takes priority) or `~/.config/muse/auth.json` written by `muse login` / `muse auth set --api-key-stdin`. Set `AGENT_MODEL` to a catalog id from `GET https://api.meta.ai/v1/models`, queried **from the machine that will run muse** — the catalog is caller-dependent (a workstation and an AWS container saw different model sets for the same key on 2026-09-08), and an id the caller cannot see fails at task time. **muse brings its own OS sandbox** (bubblewrap/seccomp on Linux, seatbelt on macOS), on by default — hive narrows it rather than refusing local launch. Installed in both images, pinned by version and per-arch SHA-256 from muse's release manifest. | | `omp` | Oh My Pi — **interactive-only**: Hive starts ordinary `omp --model ` in the prepared tmux cwd, with no fabricated permission flags; there is no headless wiring for it. Sign in once on the host (run `omp`, complete its provider setup, quit): container mode stages an allowlist of `~/.omp/agent` — only the selected provider's credential rows — into the container ([#7678](https://github.com/hivecommons/hive/issues/7678)), and the contributor image ships a pinned, checksummed `omp` ([#7661](https://github.com/hivecommons/hive/issues/7661)) so `just contribute-hive omp` runs without an `omp` on the host. No verified local confinement mechanism, so local mode refuses without `HIVE_OMP_DANGEROUSLY_RUN_UNCONFINED=1`; container mode is the supported boundary. Full setup notes: [docs/backend-setup.md](../../docs/backend-setup.md) | | `openhands` | OpenHands CLI (`uv tool install openhands --python 3.12`; PyPI `openhands`, MIT). Headless-only: `openhands --headless -t "" --always-approve --override-with-envs`. Auth is `LLM_API_KEY` (plus `LLM_BASE_URL` for an OpenAI-compatible gateway) or the `~/.openhands/settings.json` an interactive first run writes; `AGENT_MODEL=provider/model` (any litellm id) is forwarded as `LLM_MODEL` — the CLI has no `--model` flag and ignores `LLM_*` without `--override-with-envs`. No sandbox on this path, so local mode refuses without `HIVE_OPENHANDS_DANGEROUSLY_RUN_UNCONFINED=1`; not shipped in the contributor image (needs Python 3.12). Upstream has marked the CLI repo as no longer actively maintained in favour of Agent Canvas — T3 experimental. Full setup notes: [docs/backend-setup.md](../../docs/backend-setup.md) | +| `omnigent` | Omnigent CLI ([github.com/omnigent-ai/omnigent](https://github.com/omnigent-ai/omnigent), Apache-2.0). Meta-harness over Claude Code, Codex, Cursor, Pi, and custom agents. Auth uses ambient provider API keys (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`), Databricks profiles (`~/.databrickscfg`), or session tokens in `~/.omnigent/auth_tokens.json` / `~/.omnigent/config.yaml`. Launched as `omnigent run --server local`; remote-server mode is unsupported. No OS sandbox wired by Hive on the local CLI path, so local mode refuses without `HIVE_OMNIGENT_DANGEROUSLY_RUN_UNCONFINED=1`; container mode is the recommended boundary and stages only `auth_tokens.json` and `config.yaml` from `~/.omnigent`. Omnigent's internal policy engine and sandboxing compose additively, while Hive egress/API proxy remains strictly authoritative on all GitHub writes — T3 experimental. See [backend-omnigent-cli-contract.md](backend-omnigent-cli-contract.md). | ## Running multiple backends under one account @@ -1242,6 +1245,7 @@ Two admission behaviors are worth knowing when your relay seems idle: - **An issue somebody has *claimed* is not offered — and a clanker already on it is pulled off when a higher rank claims it.** The PR-claim ledger above only knows about work that has reached a pull request. Since [#8380](https://github.com/hivecommons/hive/issues/8380) the hub also keeps a **worker-claim ledger**: who is on an issue *right now*, ranked **human > agent > contributor > external**. A person claims from the terminal (`hivectl claim owner/repo#N`) or the API (`POST /api/claims/{owner}/{repo}/{number}`); a governor kick *lists* each issue it hands one of the hive's own agents — no GitHub comment, no claim, just a 30-minute hold that keeps your session and other agents off it while the kick is live — and the hub records the **agent** claim on the agent's first start signal on that issue (its own comment, label or claim request through `hive-open-issue`, or a `hive-open-pr` request naming it; [#10527](https://github.com/hivecommons/hive/issues/10527) — a comment that is an explicit *deferral* (see the [recognized forms](hive-open-issue.md#comment)) is posted but records no claim, so a parked issue is not shown in progress for two hours after its blocker lands; [#12387](https://github.com/hivecommons/hive/issues/12387)); and the relay records a **contributor** claim for your session the moment it leases you a task, releasing it when the task completes, is abandoned, is yanked or its lease expires. `selectTask` excludes every issue another holder claims exactly as it excludes leased items, and the scheduler withholds them from kicks under *In Flight*. Precedence is enforced on write: a higher rank **takes over** and the displaced holder is told to stop — for a relay session that means the hub yanks that one task (`task_revoke`, generation bumped so a late completion is fenced) and hands the session its next admissible item in the same call, leaving your other sessions alone; for an agent or an external bot it means a `🔁` comment addressed to it plus a `preempted:` label on the issue. Same rank warns unless forced; a lower rank is refused. Claims lapse on a per-kind TTL (`claims.human_ttl_s` 4h, `agent_ttl_s` 2h, `contributor_ttl_s` 30m — the relay lease length — capped by `max_ttl_s` 24h) and `claims.enabled: false` turns the whole mechanism off. Every transition is mirrored onto the issue as a comment carrying a `` marker (or `` on takeover) and a `claimed` label, so a relay that only reads GitHub can still see it was preempted. - **Claims expire.** Ledger entries live 72 hours — refreshed while the PR stays open, or anchored at the merge for a merged claim, so a settled issue is suppressed for 72h after its fix lands and nothing is stranded forever. An **open** claiming PR that goes red on a required check and stale releases the issue back to the queue; merged claims never take that valve — a fix already on `main` cannot go red. - **A repository the App cannot mint a token for is excluded, not fatal.** Since C4 the `task_assign` token is scoped to the chosen item's repository, so GitHub's `422 There is at least one repository that does not exist or is not accessible to the parent installation` means *that repo* — renamed, deleted, or removed from the installation — and nothing about the tier or the App key. It used to fail the whole selection with `token_mint_failed`, and because selection is deterministic the same doomed item was chosen on every 30 s retry: one stale entry in the project repo list (a renamed repo) wedged a hive for every contributor while `actionable_items` sat at 19 ([#7869](https://github.com/hivecommons/hive/issues/7869)). Now the claim is rolled back, the repo is excluded from selection for `mintFailureRepoCooldown` (10 min, so the fleet is not re-discovering it every retry, but the operator's fix takes effect without a restart), and the *same* call selects again from the remaining candidates (bounded at three such skips). If nothing else is admissible the reason is `repo_unmintable` rather than `no_matching_work` or `token_mint_failed`, and the hub log names the repo with a pointer at the repo list. Non-repo mint failures (5xx, bad key, an unaccepted permission) keep the `token_mint_failed` contract and exclude nothing — they would recur for any candidate. +- **An App with no installation says so.** When `github.installation_id` is `0` — the App is not installed on the hive's org, or discovery never found it — no token can be minted for any repository. The hub answers `task_unavailable` with reason `app_not_installed` and a `message` naming the org and the App install link, without calling GitHub and without excluding any repo ([#14002](https://github.com/hivecommons/hive/issues/14002)). Before this, GitHub's 404 for installation `0` was read as a repo-scope failure and contributors saw `repo_unmintable`, which points at the repo list rather than the missing installation. The relay prints the message under the reason line; the fix is the operator's (install the App on the org), never the contributor's. ## Contributor repository scope @@ -1942,17 +1946,26 @@ Per extension: `ext-exec/...` token. The Extensions tab lists currently connected peers and their declared capabilities. -External-execution items add engine-scoped tokens: `ext-exec/flue` and -`ext-exec/omp` ([#8361](https://github.com/hivecommons/hive/issues/8361)). A -run-stage item that names an external engine (`ext_exec: flue` or -`ext_exec: omp`) is offered only while the operator has enabled that engine's -binding (`runs.external.flue`, `runs.external.omp`) AND the relay declares -that engine's token in `capabilities.relay_capabilities`. A relay without the -token, or a hub with the binding off, refuses that item; it is never handed -out as ordinary local work, and declaring one token grants nothing for the -other engine. The token grants nothing by itself: the engine still receives -only a bounded context bundle, and the hub still decides acceptance of its -receipt. +External-execution items add engine-scoped tokens: `ext-exec/flue`, +`ext-exec/omp`, and `ext-exec/claude-managed-agents` +([#8361](https://github.com/hivecommons/hive/issues/8361), +[#14124](https://github.com/hivecommons/hive/issues/14124)). A run-stage item +that names an external engine (`ext_exec: flue`, `ext_exec: omp`, or +`ext_exec: claude-managed-agents`) is offered only while the operator has +enabled that engine's binding AND the relay declares that engine's token in +`capabilities.relay_capabilities`. A relay without the token, or a hub with +the binding off, refuses that item; it is never handed out as ordinary local +work, and declaring one token grants nothing for another engine. The token +grants nothing by itself: each engine receives only a bounded context bundle, +and the hub still decides acceptance of its receipt. + +**Claude Managed Agents self-hosted host.** The +`ext-exec/claude-managed-agents` token is an opt-in for the private-spoke +hosted adapter. The relay sends no vendor key, GitHub token, or dashboard +credential; Hive sends the bounded prompt/repository context only after the +normal lease authority is durable. The adapter keeps the session in advisory +`shadow` or `report-only` mode, refuses direct GitHub write paths, and records +token and list-cost metering in the spoke ledger with a hard USD 25 daily cap. **OMP workbench as a contributor host** ([#6899](https://github.com/hivecommons/hive/issues/6899)). An already-running OMP workbench does not embed the relay; it registers as a diff --git a/src/docs/contributor-token-policy.md b/src/docs/contributor-token-policy.md new file mode 100644 index 0000000000..dffd8fc487 --- /dev/null +++ b/src/docs/contributor-token-policy.md @@ -0,0 +1,72 @@ +# Contributor registration token expiry and rotation + +Registration tokens now have persisted `token_issued_at`, `token_expires_at`, +and `token_last_used_at` fields. Existing credentials receive migration-time +issuance dates on dashboard startup; restarting does not extend their lifetime. +Invite tokens are unchanged. + +Configure the dashboard in `hive.yaml` with Go durations (hours, not `d`): + +```yaml +contribute: + token_max_age: 2160h # 90 days; mandatory maximum lifetime + token_idle_max: 0s # optional inactivity cutoff; default off +``` + +`HIVE_CONTRIBUTE_TOKEN_MAX_AGE` and `HIVE_CONTRIBUTE_TOKEN_IDLE_MAX` override these +values. Invalid/negative settings fall back to 90 days / off; zero maximum age +also falls back to 90 days, rather than disabling expiry. Reducing maximum age +applies immediately; increasing it never extends an already-issued token's +stored expiry. The exact expiry instant is rejected, not accepted. Successful +authentication and pong-responsive live sessions count as activity (live-session +activity is persisted about once per minute, or more often for short idle limits). Idle limits shorter than a +heartbeat interval are not recommended. + +## Contributors + +The Management panel shows token issuance/expiry and warns when an active +contributor's token reaches 80% of maximum age. Contributors can check their own +credential with `GET /api/contribute/token`, using `Authorization: Bearer `. +Expired credentials receive HTTP 401 with recovery instructions. WebSocket +authentication sends the same instructions in `auth_failed`; existing relays +already display that reason. Connected sessions are rechecked on heartbeat, +including expiry, revocation and replacement by a newly issued token. + +Recover using GitHub proof of identity (an expired registration token cannot +rotate itself): + +```sh +hivectl contribute login # active saved hive profile +hivectl contribute login my-hive # named profile +# Equivalent existing command: +hivectl hives reissue my-hive +``` + +The command saves the new credential and regenerates `contributor.env`. Restart +the relay as instructed. The underlying recovery endpoint remains +`POST /api/contribute/reissue-token` with a GitHub bearer token. + +## Operators / incident response + +Contribute → Management → Registration tokens lists token age/expiry and offers +confirmation-gated revoke-one and revoke-all actions. These invalidate credentials +and disconnect live relays without changing contributor trust tiers. Recovery +requires each contributor's GitHub-authenticated login. Account-level revocation +(the existing Revoke control) is separate and still prevents self-recovery. + +```sh +hivectl --server https://my-hive.example contribute tokens list --output json +hivectl --server https://my-hive.example contribute tokens revoke alice +hivectl --server https://my-hive.example contribute tokens revoke --all --yes +``` + +API equivalents (hive owner/read-write authentication required): + +- `GET /api/contributors` — sanitized profiles including token timestamps and + `token_rotation_due`. +- `POST /api/contributors/{id}/token/revoke` — revoke one registration token. +- `POST /api/contributors/tokens/revoke-all` — revoke every registration token. + +Bulk revocation is deliberately not transactional across files. A persistence or +concurrency error returns 409 with the count already revoked; refresh and retry +to finish. Failed writes are never reported as successful rotations/revocations. diff --git a/src/docs/crux-cli.md b/src/docs/crux-cli.md index 86c26cfbb0..c4aa543529 100644 --- a/src/docs/crux-cli.md +++ b/src/docs/crux-cli.md @@ -4,18 +4,26 @@ ([design record](design/crux.md), RFC [#13536](https://github.com/hivecommons/hive/issues/13536)). It runs an agent from a definition and observes and controls it through `crux.Runtime` -(`src/pkg/crux/runtime.go`), with none of the governor, GitHub integration or +(`src/crux/runtime.go`), with none of the governor, GitHub integration or dashboard around it. It is the Stage 1 proof that the runtime surface is enough to drive an agent; the Stage 2 binary ships the same commands from the nested module. ## Build +Two hosts build the same CLI (`crux/cli`); they differ only in where agent +definitions come from: + ```sh -cd src && go build -o crux ./cmd/crux +cd src && go build -o crux ./cmd/crux # Hive-hosted: --config hive.yaml, pkg/config + pkg/defsrc +cd src/crux && make build # module's own: --defs , plain YAML (crux/defs) +cd src/crux && GOWORK=off go build ./cmd/crux # the same, proving the module builds alone ``` -The image also ships it at `/usr/local/bin/crux`. +The image ships the Hive-hosted one at `/usr/local/bin/crux`. The module's +binary is what a downstream checkout of `github.com/hivecommons/hive/crux` +builds; see [`src/crux/README.md`](../crux/README.md). `--version` prints the +version set with `-ldflags "-X main.version=…"` (`dev` otherwise). ## Commands @@ -30,19 +38,26 @@ Every subcommand maps onto one or two `crux.Runtime` methods and nothing else: | `crux logs --full` | `Observer.FullLogHeader`, `Observer.CaptureFullLog` | | `crux kick ` | `Observer.ResolveAgent`, `Kicker.SendKick` (`-` reads the message from stdin) | | `crux stop ` | `Observer.ResolveAgent`, `Lifecycle.Stop` | +| `crux serve [--socket P] [--listen H:P] [--token T]` | the whole surface, over HTTP+JSON ([crux-serve.md](crux-serve.md)) | Global flags go before the subcommand: | Flag | Meaning | |---|---| | `--workdir DIR` | agent work dir; default `$HIVE_WORK_DIR`, then `$HOME/.crux`. Exported to the runtime as `HIVE_WORK_DIR`. | -| `--config hive.yaml` | a hive config whose enabled `agents:` define the runtime, loaded with the same `config.Load` as `hive` itself | +| `--config PATH`, `--defs PATH` | the definition source; synonyms, default `$CRUX_DEFS`. For the Hive-hosted binary a hive config whose enabled `agents:` define the runtime, loaded with the same `config.Load` as `hive` itself; for the module's binary a directory of `*.yaml` definition files, or one file | | `--socket PATH` | control socket; default `/crux.sock` | | `--local` | never use a running `crux run`'s socket; build a fresh runtime in this process | +| `--server ADDR` | drive a `crux serve` at `ADDR` (socket path, `unix:///path`, `host:port`, `http://…`) instead of an in-process runtime; default `$CRUX_SERVER` | +| `--token T` | bearer token for `--server`; default `$CRUX_TOKEN` | +| `--version` | print the version and exit | `run` flags: `--detach` (return after `Start`, leave the agent running), `--interval D` (output poll interval, default 1s), `--no-socket`. +`logs -f` follows the agent's ring buffer; it is available in remote mode +only, where the server streams it. + Exit codes: 0 ok, 1 runtime error, 2 usage. ## Definitions @@ -92,7 +107,29 @@ crux stop fixer The socket carries one JSON request per connection (the subcommand's argv) and one JSON response (what the command wrote, and its exit code). The serving side runs the same command code the local path runs. It is a local -convenience for one work dir, not the Stage 3 `crux serve` API. +convenience for one work dir, not the `crux serve` API. + +## Remote mode + +`crux serve` is the long-lived process when something other than a +foreground `crux run` should own the agents: it builds the runtime from +`--config` and exposes it over HTTP+JSON on a unix socket (optionally TCP +with a bearer token). With `--server` or `CRUX_SERVER` set, `run`, `ps`, +`logs`, `kick` and `stop` talk to it and build nothing locally: + +```sh +crux --config hive.yaml serve & # serving on unix:///run/user/1000/crux/crux.sock +export CRUX_SERVER=/run/user/$(id -u)/crux/crux.sock +crux run fixer.yaml # parsed here, sent as JSON, output followed +crux ps +crux logs fixer -f +crux stop fixer +``` + +`serve` itself ignores `--server`/`CRUX_SERVER` and always runs locally, so a +server address left in a shell profile does not stop a new server from +starting. Routes, auth, defaults and curl examples are in +[crux-serve.md](crux-serve.md). Without a live socket, the observer commands build a fresh runtime from `--config`. That runtime knows the definitions and can read the pane of a diff --git a/src/docs/crux-publish.md b/src/docs/crux-publish.md new file mode 100644 index 0000000000..38a14e668d --- /dev/null +++ b/src/docs/crux-publish.md @@ -0,0 +1,89 @@ +# Publishing Crux: `hivecommons/crux` is a synced copy of `src/crux` + +Hive is the source of truth for the Crux agent runtime. The nested Go module +at `src/crux` (module `github.com/hivecommons/hive/crux`, RFC +[#13536](https://github.com/hivecommons/hive/issues/13536), tracker +[#14138](https://github.com/hivecommons/hive/issues/14138)) is developed, +reviewed and tested here. The public repository +[hivecommons/crux](https://github.com/hivecommons/crux) is a **published, +module-path-rewritten copy** of that module, so that + +```sh +go install github.com/hivecommons/crux/cmd/crux@latest +brew install hivecommons/hive/crux +docker run --rm ghcr.io/hivecommons/crux --help +``` + +work without a consumer pulling hive. Nobody edits Go code in +`hivecommons/crux`; its own CI refuses such pull requests and points back +here. + +## Mechanism + +| Piece | Where | What it does | +|---|---|---| +| `src/scripts/crux-publish.sh` | hive | Builds the publish tree: clears a `hivecommons/crux` checkout except its hand-maintained scaffolding, copies `src/crux` in, rewrites `github.com/hivecommons/hive/crux` → `github.com/hivecommons/crux` in `go.mod` and every import, writes `.hive-sync` (hive sha, ref, date), then proves the tree builds with `GOWORK=off go build ./... && go vet ./...`. A failing build aborts the sync; nothing is committed. | +| `.github/workflows/crux-sync.yml` | hive | Runs the script on every push to `v5` touching `src/crux/**`, on tags `crux/v*`, and on `workflow_dispatch`; commits the result on top of `hivecommons/crux` `main` (fast-forward, one commit per sync) and, for a `crux/vX.Y.Z` tag, pushes tag `vX.Y.Z` to `hivecommons/crux` at the already-synced commit whose tree matches (tag runs never move `main`). | +| `.github/workflows/ci.yml` | hivecommons/crux | Build, vet, test (`-race`), lint, and a `crux serve` smoke on linux + macOS. | +| `.github/workflows/sync-guard.yml` | hivecommons/crux | Fails any pull request from a non-sync actor that touches `*.go`, `go.mod` or `go.sum`, with a message pointing at `src/crux` here. | +| `.github/workflows/release.yml` + `.goreleaser.yaml` | hivecommons/crux | On tag `v*`: darwin/linux × amd64/arm64 binaries, checksums, cosign-signed, GitHub release, multi-arch `ghcr.io/hivecommons/crux` image. | +| `Formula/crux.rb` | hivecommons/homebrew-hive | Hand-maintained formula over the release tarballs. | + +### Why a plain copy, not `git subtree split` + +`git subtree split --prefix=src/crux` walks hive's whole history on every run +(minutes today and growing), yields commits whose imports do not compile +(the module path is rewritten *after* the split), and buys nothing: the +commit that lands in `hivecommons/crux` names the exact hive sha it came +from in its message and in `.hive-sync`. The published history is linear +and fast-forward only; `git log` there is a list of hive shas. + +### What is preserved in `hivecommons/crux` + +`.git`, `.github/`, `.goreleaser.yaml`, `Dockerfile`, `Dockerfile.goreleaser`, `.dockerignore`, +`LICENSE`, `README.md`, `.gitignore`, `.golangci.yml`. Everything else is +replaced on each sync. The module's own `src/crux/README.md` lands as +`docs/MODULE.md`; the repository `README.md` (positioning, install, quick +start) is hand-maintained there. + +### Why the nested module publishes standalone + +`src/crux/go.mod` requires nothing from the root module and the root +`go.work` is not part of the copy, so the rewritten tree is a complete module +on its own. The root module's `replace github.com/hivecommons/hive/crux => +./crux` is what lets hive consume it in-tree; the publish side never sees it. +If a change makes `src/crux` import the root module, the sync's build guard +fails and the change must be undone here (that is the Stage 2 ratchet +working as intended). + +## Credentials + +Pushing to `hivecommons/crux` needs a token with **Contents: write** there. +The workflow uses the repository secret `CRUX_SYNC_TOKEN` when set and falls +back to `TOPUP_PUSH_TOKEN` (the fine-grained PAT `v5-topup` and +`cli-pin-bump` already use). Without a usable token the run fails at the +push step with the secret to provision in its summary, and uploads the +publish tree as the `crux-publish-tree` artifact. + +## Manual sync + +Same script, same result: + +```sh +git clone https://github.com/hivecommons/crux crux-publish +src/scripts/crux-publish.sh . crux-publish +cd crux-publish && git add -A && git commit -s -m "🌱 sync: hive@$(git -C .. rev-parse --short HEAD) (src/crux)" && git push +``` + +## Releasing + +Tag hive: `git tag crux/v0.2.0 && git push origin crux/v0.2.0`. A tag +run never moves `hivecommons/crux` `main`: it rebuilds the publish tree for +the tagged hive commit, finds the commit already on `main` whose tree matches +(the push trigger synced it when the commit landed on `v5`), and pushes +`v0.2.0` there; if nothing on `main` matches it fails and tells you to sync +first. The crux release workflow does the rest. Then bump `Formula/crux.rb` +in `hivecommons/homebrew-hive` (Prow: `/lgtm` `/approve`). + +`v0.1.0` was tagged directly on `hivecommons/crux` (at the first hand-run +sync of hive `1e02ec49d331`) before the tag path was exercised. diff --git a/src/docs/crux-serve.md b/src/docs/crux-serve.md new file mode 100644 index 0000000000..bae2905f95 --- /dev/null +++ b/src/docs/crux-serve.md @@ -0,0 +1,293 @@ +# `crux serve`: the runtime over a local API + +`crux serve` exposes a `crux.Runtime` over HTTP+JSON so a process other than +the one that started an agent can run, observe, kick and stop it. It is the +Stage 3 process boundary of the Crux design record +([design/crux.md](design/crux.md), RFC +[#13536](https://github.com/hivecommons/hive/issues/13536), stage 2/3 tracker +[#14138](https://github.com/hivecommons/hive/issues/14138)): a harness other +than hive drives agents through it, and the `crux` CLI has a remote mode that +talks to it. Hive itself does not use the API yet; that is a later decision. + +The wire contract is the stdlib-only package `crux/api` +(`src/crux/api`), which a client can import without pulling in the runtime. +The server is `crux/serve`; the Go client is `api.NewClient`. + +## Start a server + +```sh +cd src && go build -o crux ./cmd/crux +crux --config /etc/hive/hive.yaml serve +# serving 3 agent definition(s) on unix:///run/user/1000/crux/crux.sock +``` + +`serve` builds the runtime from `--config` (the enabled `agents:` of a +`hive.yaml`) and listens until interrupted, stopping nothing on exit: agents +it started keep running in tmux. Without `--config` it serves an empty +runtime and every agent must arrive as a definition on `POST /v1/agents`. + +| Flag / env | Meaning | +|---|---| +| `--socket PATH`, `CRUX_SOCKET` | unix socket to serve on. Default `$XDG_RUNTIME_DIR/crux/crux.sock`, else `/crux-/crux.sock`. The directory is created `0700`, the socket `0600`, and it is never reachable at that path with wider permissions (it is bound under a temporary name, chmod'ed, then renamed). A socket somebody already answers on is refused; a stale one is replaced. | +| `--listen HOST:PORT` | also serve on TCP. **Refused without a token.** | +| `--token T`, `CRUX_TOKEN` | bearer token every request must carry. When set it gates the socket too. | +| `--hive-callback ADDR`, `CRUX_HIVE_SOCKET` (+ `CRUX_HIVE_TOKEN`) | dial hive's callback endpoint (`crux/api/callback`) for the capabilities hive owns — scoped GitHub tokens, PR creation, Linear credentials, text/canary scanners, conditions — and feed the runtime's seams from it instead of leaving them nil. Health reports it under `hive_callback`; an unreachable endpoint raises the `hive_callback_unavailable` condition. (#14246) When the runtime is a manager it also polls hive's host policy from the same endpoint; see [Host policy from hive](#host-policy-from-hive-14138-stage-4). | + +`serve` does not use the per-work-dir control socket that a foreground +`crux run` serves (`/crux.sock`); the two are different things, see +[crux-cli.md](crux-cli.md#the-runtime-is-in-process). + +### Fail closed + +- No TCP without a token: `--listen` without `--token`/`CRUX_TOKEN` is a + usage error. When the server has no token the handler accepts only + requests it can see arrived over its own unix socket; anything else + (including a `Handler()` mounted on a foreign `http.Server`) gets `401`. +- With a token, every request on every listener needs + `Authorization: Bearer `; anything else is `401` with + `WWW-Authenticate: Bearer realm="crux"`. +- Socket mode `0600`, directory `0700`. The directory must be a real + directory (not a symlink) owned by the serving uid with no group/other + bits, or `Listen` refuses to place a socket in it; a pre-existing + world-readable `/crux-` is an error, not a trust. +- Request bodies are capped at 1 MiB and unknown JSON fields are rejected. + +## The API + +All routes are under `/v1`. Bodies and responses are JSON; errors are +`{"error": "...", "code": "unauthorized|bad_request|not_found|conflict|runtime_error"}`. + +| Route | Body | Runtime method | Returns | +|---|---|---|---| +| `GET /v1/health` | | | `{"version":"v1","agents":N}` | +| `GET /v1/agents` | | `Observer.AllStatuses` | `{"processes":[Process…]}`, sorted by name | +| `POST /v1/agents` | `{"name":…, "definition":{…}}` | `Definitions.AddAgent` (when `definition` is present), `Lifecycle.Start` | `201 {"agent":…, "process":Process}`; `400` bad name, `409` the name already has a definition, `422` the runtime refused it | +| `GET /v1/agents/{name}` | | `Observer.GetStatus` | `Process` | +| `GET /v1/agents/{name}/logs?source=pane\|buffer\|full&n=N` | | `GetOutput` / `GetBufferOutput` / `FullLogHeader`+`CaptureFullLog` | `{"agent","source","lines":[…]}` or `{"header","content"}` for `full` | +| `GET /v1/agents/{name}/logs?follow=1` | | `GetBufferOutput`, polled | `text/event-stream` of `agent.log` records `{"agent","lines":[…]}` | +| `POST /v1/agents/{name}/kick` | `{"message":…}` | `Kicker.SendKick` | `{"agent","process"}` | +| `POST /v1/agents/{name}/stop` | | `Lifecycle.Stop` | `{"agent","process"}` | +| `GET /v1/events` | | watcher + `Kicker.SetKickObserver` + the `crux.Host` observers | `text/event-stream` of `Event` records | + +### Runtime families (#14138 stage 3) + +Every method of `crux.Runtime` has a route; `api.RuntimeRoutes` is the +method → route table and `crux/api/client`'s parity test fails when a method +is added to the interface without an entry. The sidecar client +(`crux/api/client.Runtime`, selected by `CRUX_MODE=sidecar`) is written +against this table. Routes marked ⚑ are new in stage 3; the rest predate +it or are the CLI routes above under another method. + +| Family | Route | Runtime method(s) | Notes | +|---|---|---|---| +| Lifecycle | `POST /v1/agents/{name}/start` ⚑ | `Start` | | +| | `POST /v1/agents/{name}/restart` ⚑ | `Restart`, `RestartWithBootstrap` (`{"bootstrap":…}`) | | +| | `POST /v1/agents/{name}/pause` ⚑ | `Pause`, `PauseBy`, `Supervisor.PauseByCause` | body selects the variant | +| | `POST /v1/agents/{name}/resume` ⚑ | `Resume` | | +| | `DELETE /v1/agents/{name}/restart-count` ⚑ | `ResetRestartCount` | | +| | `GET /v1/agents/{name}` | `IsPaused`, `Observer.ResolveAgent` | | +| Observer | `GET /v1/snapshots`, `GET /v1/agents/{name}/snapshot` ⚑ | `AllStatuses`, `GetStatus`, `GetStatusFast` | full `crux.Process` snapshot, not the wire projection | +| Kicker | `POST /v1/agents/{name}/kick/validate` ⚑ | `ValidateKick` | `409 kick_refused` | +| | `GET /v1/agents/{name}/kick` ⚑ | `KickDispatchState` | | +| | `GET /v1/agents/{name}/kicklogs[/{id}]` ⚑ | `ListKickLogs`, `ReadKickLog` | | +| Definitions | `POST /v1/definitions`, `POST /v1/definitions/reconcile` ⚑ | `AddAgent`, `ReconcileAgents` | | +| | `DELETE /v1/agents/{name}`, `PUT /v1/agents/{name}/definition` ⚑ | `RemoveAgent`, `UpdateConfig` | | +| | `PUT /v1/agents/{name}/overrides`, `PUT`/`DELETE /v1/agents/{name}/pins/{kind}` ⚑ | `SetModelOverride`, `SetBackendOverride`, `PinCLI`, `PinModel`, `UnpinCLI`, `UnpinModel` | | +| | `POST /v1/fleet/modes/{clear,sync}`, `POST /v1/fleet/inference/refresh` ⚑ | `ClearModeOverrides`, `SyncModeFiles`, `RefreshInferenceRoutes` | | +| Fleet | `GET /v1/fleet` ⚑ | `GetACMMLevel`, `SharedDevUID`, `Supervisor.ConsentWedgedAgents`, `CountAgentsWithModel` | one read | +| | `GET`/`PUT /v1/fleet/breaker`, `POST /v1/fleet/breaker/{engage,release}` ⚑ | `BreakerState`, `Restorer.RestoreBreaker`, `EngageBreaker`, `ReleaseBreaker` | | +| | `PUT /v1/fleet/acmm`, `POST /v1/fleet/relaunch-bob` ⚑ | `SetACMMLevel`, `RelaunchBobAgentsAwaitingKey` | | +| Credentials | `POST /v1/agents/{name}/login-code` ⚑ | `SubmitLoginCode` | | +| | `GET`/`PUT /v1/credentials/copilot` ⚑ | `CopilotToken`, `CopilotTokenSource`, `SetCopilotToken`, `ActivateCopilotToken` | | +| | `GET /v1/credentials/claude`, `POST /v1/credentials/claude/reload` ⚑ | `ClaudeCredentialCandidatePaths`, `ReloadClaudeToken` | | +| Resumer | `GET /v1/agents/{name}/session` ⚑ | `ResumeHandle`, `SessionID`, `SessionMissing` | | +| | `POST /v1/agents/{name}/session/kick` ⚑ | `SendResumeKick` | `409 resume_busy`, `410 resume_session_gone`, `422 resume_unsupported` map back to the `crux.ErrResume*` sentinels | +| Supervisor | `GET /v1/agents/{name}/supervision` ⚑ | `GovernorMayStart`, `AgentCredentialRefreshable`, `AgentHasValidCredential`, `AgentCapabilities`, `EffectiveBackend`, `InvocationMetadata`, `RestartTelemetry`, `StartFailureState`, `ProviderErrorBackoffRemaining`, `CrashOOMSuspected`, `AgentAuthAvailable` | one read | +| | `POST /v1/fleet/{startup-queued,restart-crashed,tokens/refresh,kicklogs/archive}` ⚑ | `MarkStartupLaunchQueued`, `CheckAndRestartCrashedAgents`, `RefreshAgentTokens`, `ArchiveAllKickLogs` | | +| | `POST /v1/agents/{name}/tokens/refresh`, `POST /v1/agents/{name}/nudge` ⚑ | `RefreshAgentTokenFor`, `SendGitHubAPIBudgetNudge` | | +| | `GET /v1/backends/{backend}/auth`, `POST /v1/audit`, `GET /v1/trajectory` ⚑ | `BackendAuthAvailable`, `RecordAudit`, `TrajectoryAgents` | | +| Restorer | `POST /v1/agents/{name}/seed` ⚑ | `SeedPauseState`, `SeedRestartTelemetry`, `SeedTurnLoss`, `SeedLastKick`, `SeedKickHistory` | one body, each field optional | +| Background | `GET /v1/health` | `StartAgentTokenRefresh`, `StartCredentialWatchdog`, `StartCopilotSessionRefresh`, `StartTranscriptActivityRefresh` | the loops run server-side; the client's calls are no-ops | +| Authorizer | `POST /v1/authorize` ⚑ | `AuthorizePROpen`, `AuthorizePushBranch`, `AuthorizeIssueOpen`, `AuthorizeMerge`, `AuthorizeReviewRequest` | synchronous; `{"action","agent","file_uid","kind","repo"}`; `403 forbidden` carries the refusal, any transport error is a refusal (fail closed) | +| Watchdog | `GET /v1/watchdog/agents`, `GET /v1/agents/{name}/watchdog`, `PUT /v1/agents/{name}/watchdog/conditions` ⚑ | `watchdog.Source` (`AgentNames`, `Observe`, `LastProduction`, `SetConditions`) | `501 not_supported` when the runtime has no watchdog source; `crux/runtime.NewWatchdogFleet` builds the fleet over these in sidecar mode | + +`Process` is the wire projection of `crux.Process`: `name`, `id`, `state`, +`backend`, `model`, `pid`, `paused` (+ reason, trigger, by), `started_at`, +`last_kick`, `restart_count`, `last_restart_reason`, `kick_refused`, +`kick_refusal_reason`, `kicks_undeliverable`, `busy_condition`, +`start_failure_class`. It is a subset by design: the full snapshot carries +engine-internal state that is neither stable nor meaningful off-process. + +`definition` on `POST /v1/agents` is the JSON form of +`crux/config.AgentConfig`, the per-agent overlay hive reads from +`agents_dir`. The server applies the same gate and defaults the YAML loader +does: `name` must match `^[a-z0-9][a-z0-9_-]*$` (≤64 chars); `id` and +`role` default to the name, `replicas` to 1, `enabled` to true unless the +JSON says otherwise. A name the runtime already knows is a `409 conflict` +rather than a silent start of the old definition; stop and re-post. Without +`definition`, `name` may also be an agent ID. Parsing the portable +`AgentDefinition` format is the caller's job, as it is the host's job for +the CLI (the CLI does it before sending). + +Agents start under the server's lifetime, not the request's: the runtime +ties an agent to the context it was started with, so a client disconnecting +does not kill what it started. + +### Events + +`GET /v1/events` is a server-sent-event stream; each record is +`event: ` then `data: `, with `: keep-alive` comments +every 15 s. `Event` is `{"time","type","agent","state","detail","data"}`. + +| Type | When | +|---|---| +| `agent.started`, `agent.stopped`, `agent.kicked` | the API performed the action | +| `agent.state` | the server's watcher (polls `AllStatuses` every second) saw an agent change state for any reason: crash, restart, pause; `state` is the new state, `detail` the last restart reason when there is one | +| `agent.kick` | the runtime's kick observer fired (`Kicker.SetKickObserver`); `detail` is the observer's event name | +| `agent.kick_outcome`, `agent.pause_transition`, `agent.relaunch`, `agent.pause_persist`, `agent.prompt`, `agent.sandbox_audit`, `runtime.audit`, `runtime.condition` | the `crux.Host` observers, relayed when the runtime is a Host; `data` carries the typed payload (`api.KickOutcome`, `api.PauseTransition`, …). A sidecar hive demuxes these into the observers its boot sequence installs (`cmd/hive`'s `sidecarHost`) | + +A subscriber that stops reading is dropped after 64 buffered events so one +slow client cannot stall the others. + +### Host policy from hive (#14138 stage 4) + +With `--hive-callback`, `serve` also reads the values an in-process hive +hands the manager through the `crux.Host` policy setters. They travel on +the hive callback endpoint (`crux/api/callback`, runtime → hive), not on +this API: + +| Callback route | Feeds | +|---|---| +| `GET /v1/policy` | a `callback.Policy` snapshot, polled every 15 s (`hivecap.WatchPolicy`): `build` → `SetBuildInfo`, `explain_mode_default` → `SetExplainModeDefaultResolver`, `kick_log_source` → `SetKickLogSourceResolver`, `gateway_backends` → `SetGatewayBackendChecker`, `dead_session_recovery_owned_elsewhere` → `SetDeadSessionRecoveryOwner`, `sandbox` (the `agent_sandbox` block) → `SetSandboxConfig`; `repo_auto_merge: true` installs the next row | +| `POST /v1/policy/repo-auto-merge` | `{"repo"}` → `{"enabled"}` for `SetRepoAutoMergeEnabledResolver`, asked per merge authorisation | + +The resolvers answer from the last snapshot (the manager reads them under +its own lock, so they never call out) and keep it through a hive outage; +the auto-merge call fails closed, refusing the merge. A value hive does not +report, or a hive without the `policy` capability (`404 not_configured`), +leaves the runtime's own default in place. + +Every `crux.Host` setter has a disposition in `api.HostSetters`, and +`TestParityHostVersusSetterTable` fails when one is added without an +entry. The remaining setters, all on the same callback endpoint: + +| Setter | Callback route (capability) | Fail-closed behaviour | +|---|---|---| +| `SetInferenceCallbacks` | `PUT`/`DELETE /v1/inference/routes/{agent}` (`inference_routes`): the manager pushes each agent's inference route into hive's proxy | route not installed; the agent runs without a per-agent route | +| `SetSandboxMutationBoundary` | `POST /v1/effects`, `POST /v1/effects/{id}/end` (`effects`): two-phase claim/end over hive's journaling boundary | mutation refused | +| `SetAppAuth` | `POST /v1/tokens` with `{agent, tier}` and no repos (installation-wide, what `AppAuth.ScopedToken(tier)` mints in-process); `hivecap.AppTokenWriter` writes the token in place (`O_TRUNC`, `0600`) to the agent's existing cache file and nowhere else, never logging it. `HIVE_PROXY_INJECT_GH_AUTH` is refused in sidecar mode (`ErrProxyInjectionUnsupported`) rather than writing the real token | cache file untouched; the agent keeps its last token or runs read-only | +| `SetAgentMint` | `POST /v1/tokens/mint` (`agent_mint`); a hive with mint disabled answers `404 not_configured`, which the writer maps to the in-process "mint off" contract (`""`, no error) | no mint token | +| `SetBobAPIKeyResolver`, `SetBobKeySourceResolver` | `GET /v1/bob/credential` (`bob`); cached 1 min runtime-side, a failed refresh serves the stale value for at most 2 min | empty key and source | +| `SetSpanStarter` | — in-process-only: a span is a Go value wrapping a call in one address space. The sidecar's manager starts spans on the OpenTelemetry global tracer (a no-op without an exporter in the runtime's process); trajectory events do not depend on it | n/a | + +The proxy seams are carried the same way: `hivecap.Set.{Accounting, +Usage, Canary}` come from `GET /v1/github/rate-limit`, `POST /v1/usage` +and `POST /v1/scan/canary`, so a proxy built on them refuses writes while +hive is away and resumes on the next successful call. No setter is served +from a `runtime:` config key: every value is already resolved hive-side +from the live config, so a reload reaches the runtime on its next poll. + +### Supervision and failure semantics (#14138 stage 4) + +In sidecar mode hive decides who keeps `crux serve` alive: + +| Env | Values | Meaning | +|---|---|---| +| `CRUX_SUPERVISE` | `child` (default), `external` | `child`: hive spawns `crux serve` itself (`crux/runtime.Supervisor`), waits for `/v1/health`, and restarts it with exponential backoff (1 s → 1 min, reset after 5 min healthy). `external`: systemd / a pod sidecar runs it; hive only reconnects. | +| `CRUX_BINARY` | path or name | the executable `child` runs; default `crux` on `$PATH`. A missing binary is logged loudly and hive behaves as `external`. | + +The child inherits hive's environment plus `CRUX_TOKEN`; its listen flags +are derived from `CRUX_SERVER` (`unix://` → `--socket`, host:port → +`--listen`). `/v1/health` now carries `started_at` and `uptime_seconds` +next to `hive_callback`. + +When the socket is gone, whichever way it is supervised: + +| Surface | Behaviour | +|---|---| +| governor loop | keeps ticking; nothing in hive blocks on the socket | +| writes with an error return (kick, stop, resume, …) | **refused** with `crux.ErrRuntimeUnavailable` (wraps `api.ErrUnreachable`). Kicks are not queued or replayed: the governor re-derives what is due on its next tick, so a replay buffer would only duplicate work after a long outage | +| observer reads | answer zero / unknown (unchanged from stage 3) | +| fleet report | `crux_runtime_unavailable` (component `agent-runtime`, severity high) raised on the first failed probe (every 15 s, or any failed call) and cleared on the first success; the dashboard renders it like every other runtime condition | +| hive `/api/health` | `crux_runtime: {mode, reachable, last_error, since, supervise, child{running, ready, pid, starts, last_exit}}`; `status` becomes `degraded` while unreachable | +| client `Status()` | `{reachable, last_error, since, failed_calls, last_success}` on `api.Client`/`client.Runtime`; a caller that cancelled its own context is not counted, an expired deadline is | + +In-process mode is untouched: there is no socket, the condition never +fires, and `/api/health` carries no `crux_runtime` member. + +## Drive it with curl + +```sh +S=/run/user/$(id -u)/crux/crux.sock +curl -s --unix-socket $S http://crux/v1/health +curl -s --unix-socket $S http://crux/v1/agents | jq '.processes[] | {name, state}' + +# run from a definition (an agents_dir-style AgentConfig document) +curl -s --unix-socket $S -X POST http://crux/v1/agents \ + -H 'Content-Type: application/json' \ + -d '{"name":"fixer","definition":{"backend":"claude","model":"claude-sonnet-4-6","role":"fixer"}}' + +curl -s --unix-socket $S 'http://crux/v1/agents/fixer/logs?source=buffer&n=50' | jq -r '.lines[]' +curl -s -N --unix-socket $S 'http://crux/v1/agents/fixer/logs?follow=1' # streams until Ctrl-C +curl -s --unix-socket $S -X POST http://crux/v1/agents/fixer/kick \ + -H 'Content-Type: application/json' -d '{"message":"look at the failing build on main"}' +curl -s -N --unix-socket $S http://crux/v1/events # in another terminal +curl -s --unix-socket $S -X POST http://crux/v1/agents/fixer/stop +``` + +Over TCP with a token: + +```sh +CRUX_TOKEN=$(openssl rand -hex 16) crux --config hive.yaml serve --listen 127.0.0.1:7777 +curl -s -H "Authorization: Bearer $CRUX_TOKEN" http://127.0.0.1:7777/v1/agents +``` + +The hostname in a unix-socket URL is ignored; `crux` is a convention. + +## Drive it with the CLI + +`--server ADDR` (or `CRUX_SERVER`) switches `run`, `ps`, `logs`, `kick` and +`stop` to the API; `--token` (or `CRUX_TOKEN`) carries the bearer token. +`ADDR` is a socket path, `unix:///path`, `host:port` or `http(s)://host:port`. + +```sh +export CRUX_SERVER=/run/user/$(id -u)/crux/crux.sock +crux run examples/agents/customized-agent.yaml # definition parsed here, sent as JSON; follows output; Ctrl-C stops +crux run --detach fixer # a name already defined on the server +crux ps +crux logs fixer -n 50 --buffer +crux logs fixer -f # follow the ring buffer over SSE +crux kick fixer "look at the failing build on main" +crux stop fixer +``` + +In remote mode nothing is built in this process: no runtime, no control +socket, no `--config` (the server's definitions are what exist). `serve` +itself cannot be run remotely. A `401` is reported with a hint to set the +token. + +## Drive it from Go + +```go +import "github.com/hivecommons/hive/crux/api" + +c, err := api.NewClient(os.Getenv("CRUX_SERVER"), os.Getenv("CRUX_TOKEN")) +procs, err := c.List(ctx) +resp, err := c.Run(ctx, api.RunRequest{Name: "fixer", Definition: defJSON}) +err = c.FollowLogs(ctx, "fixer", func(ch api.LogChunk) error { …; return nil }) +err = c.Events(ctx, func(ev api.Event) error { …; return nil }) +_, err = c.Stop(ctx, "fixer") +``` + +`crux/api` imports only the standard library, so a client binary does not +link the runtime. Errors from the server are `*api.StatusError`; +`api.IsNotFound` and `api.IsUnauthorized` classify them. +`Client.WaitReady` polls `/v1/health` for a caller that just launched +`crux serve`. + +## What this is not + +It is not tenancy, quotas or hive-as-a-client: the server trusts whoever can +reach the socket or holds the token, and hive still constructs +`*agent.Manager` in-process. Those are the rest of Stage 3 in +[design/crux.md](design/crux.md#after-the-cut). diff --git a/src/docs/dashboard-sections.md b/src/docs/dashboard-sections.md index 9c4b906487..60db6bd968 100644 --- a/src/docs/dashboard-sections.md +++ b/src/docs/dashboard-sections.md @@ -96,6 +96,7 @@ How busy Hive thinks your projects are right now, and how often it wakes its wor - The Governor picks a mode by comparing the amount of actionable work with three thresholds. Actionable work is work Hive could do now, with nothing waiting on a person. - The **actionable issues** and **actionable PRs** tiles use the same count as the Overview's Actionable now tile. The two always agree. - The **hold** list names every held item. Held means someone parked it with a hold label. +- The compact strip at the top of the Agents view also shows **running** — how many workers are mid-round right now, and the estimated spend the whole fleet has accrued in the current clock hour (`3 · $1.20 this hour`). The hour figure is the rise in the cumulative estimate since the top of the hour; `$–` means the hive has not yet taken a reading this hour. - The **cadence table** shows, for each worker, how often it runs in each mode. Continuous means it runs again as soon as it finishes. - The **budget** shows how much of the AI spending allowance for the current period has been used. - The **next kick** estimate follows the eval tick, which is longer while GitHub webhooks are trusted (`/api/status.webhooks.mode` reads `webhook`) and drops back to `governor.eval_interval_s` when they degrade (`polling`). How that decision is made is in [webhooks.md](webhooks.md#fallback-when-webhooks-degrade). @@ -335,6 +336,7 @@ One card per repository Hive looks after, listing its open issues and pull reque **How the numbers are worked out.** - The cards use the same bands as the Overview charts. See the Overview entry for each band's meaning. +- The small label above each repository name is its code hosting service. On GitHub it is the host, such as `github.com`, with `· Enterprise` on GitHub Enterprise. On GitLab or Gitea/Forgejo (`project.forge`) it names the service and its host, such as `🦊 GitLab · gitlab.example.com`. The card's links then go to that host. The badge in the section header names the same forge when every card is on it, and shows `N forges` when a `project.repo_policies[].forge` override puts cards on different forges. Each repository in `/api/status` carries this as `forge` (`github`, `gitlab` or `gitea`) and, for GitLab and Gitea, `forgeUrl`. See [Forges](integrations/forges.md). - Small pills on each item show signals, such as a hold, a failing check or no recent activity. - A `🔗` pill on an issue means Hive has verified an open or merged pull request covers it. Hover it: "closes via PR #N" means merging the PR closes the issue; "covered by PR #N (partial)" means the PR says `Refs #N` and merging it does not close the issue. The pill comes from Hive's own PR-claim evidence, so it appears on every issue labelled `🐝 covered by PR` — including ones Hive has parked in the "covered by PR" group because an outside PR already fixes them — and never depends on GitHub having a closing link. - A red pull-request pill means the sentinel flagged it for a possible security override, privilege escalation or codebase damage. Hive will not merge it until a maintainer reviews it and removes the sentinel label. Pull requests from trusted authors usually get a sentinel comment instead of the label, so they are usually not red. @@ -356,6 +358,7 @@ One card per repository Hive looks after, listing its open issues and pull reque **Settings that change it.** - The repositories Hive watches are set in **Settings → Projects**. +- `project.forge` (or a repo's `project.repo_policies[].forge` / `forge_url` override) and the `gitlab:` or `gitea:` instance URL set the host label and links. See [Forges](integrations/forges.md). - The `dashboard.issue_bands` settings change the band rules. - See [Repository card legend, issue bands, and PR bands](dashboard.md#repository-card-legend-issue-bands-and-pr-bands). @@ -617,6 +620,14 @@ More about the line: - **Updates.** The line updates with each worker update, every 10 seconds by default, so a change shows within about 15 seconds. - **Privacy.** Titles come only from the issue and pull request lists the dashboard already shows you. If it shows you no repositories' issues, the line shows no item numbers either. +**The run line.** While a worker is working, a second line under **Now:** reads like `$0.42 · 38.1K tok · 4m 12s · tests ✓ · 3 files`. The same line is pinned in the worker's detail panel header. It is built only from figures Hive already keeps: + +- **`$` and `tok`** are the estimated cost and tokens of *this* round of work: the worker's cumulative token count × public list price (the same basis as the Cost section), minus what it was when Hive started the round (the last kick). Hive remembers the starting point, so a page reload does not reset the figure. `$–` means the token collector has no figure for this worker yet. +- **Elapsed** counts from the start of the round. +- **`tests ✓` / `tests ✗` / `tests –`** is the last test-runner summary visible in the worker's terminal (`go test` `ok`/`FAIL`, pytest `N passed`/`N failed`, jest/vitest `Tests:` lines). `–` means no result line is visible. +- **`N files`** is the last `N files changed` git summary visible in the terminal; `–` means none is. +- It updates with each worker update and disappears when the round ends. The **spend (range)** row further down the card is unchanged: that is the worker's spend over the Cost section's timeframe, not this round's. + **What it is good for.** Use it to pause a worker, restart a stuck one, or change its settings. Fix any card that says it needs a login. Read the **Now:** lines to see what each worker is on without opening its log. **Example.** A card says "needs login". You click **Login** and sign in again. Another card says `Now: working — no issue or pull request yet · started 50 min ago`. That worker has changed nothing through the hive for 50 minutes, so you open its log to see what it is doing. @@ -633,7 +644,7 @@ More about the line: A progress strip per open issue and pull request showing which step of the pipeline it is on and who must act next. -Fed by `GET /api/stage-tracker`. Every issue and pull request follows the same pipeline, named once in the column header above the list: `Open → Actionable (010) → Claimed (020) → PR in progress (030/040) → CI / Review (050/060) → Merge gate (080/100) → ✓ merged`. Each entry is a title line over a thin chevron strip. Only the current step is coloured: it is filled amber and carries the step name in words; once the PR is merged the final ✓ chevron is the lit one, in green. Motion is reserved for what is running or needs a hand — the lit chevron falls into one of three buckets (`stageTrackerMotion()` in `index.html`): **static** (steady): Open, Open · on hold (any hold label wins over every other signal), Actionable (010), Claimed (020), PR in progress / Covered by PR (030/040) by a human author, Merged, and any unrecognised stage; **active** (gentle pulse): the hive is acting — 030/040 by a hive or clanker author, 050 review pending, 060 reviewed, 070 fixing, 074 adopted by hive, 080 merge gate, 095 pre-merge soak, 100 merge queue; **needs-human** (slower amber→red breathe): a person must move — 015 triage needed, 045/085 needs decision, 071 needs author, 072 returned to contributor, 090 ready for a human merge, or a `stalled` / `blocked` / `needs-human` / `needs-decision` / `needs-triage` modifier on any stage. Under `prefers-reduced-motion` nothing animates; a needs-human step keeps a steady red tint so it still stands out. Every other step is a low-contrast shape with a small grey glyph (circle, bolt, lock, branch, eye, shield, check) — passed steps a subtle neutral fill, steps ahead a faint outline. Side stages are not extra steps: `070 Fixing` (the author's move; the `needs-rebase` chip next to it says what is being fixed — the stage labeller always parks a needs-rebase PR at 070/071/072, never at CI / Review), `071 Needs author` (human-authored), `072 Returned to contributor` (clanker-authored) and `074 Adopted by hive` light *CI / Review* with their own name (an adopted PR stays on 074 through the hive's pushes going pending, green or red again, and moves on only when *Merge gate* takes it); `085 Needs human decision` (PR-only) and `090 Ready for human merge` light *Merge gate* with their own name (the 085 tooltip says what must be decided); an issue parked on a person wears `045 Needs human decision` and lights *Actionable* (the issue side) with that name — a pre-045 issue still wearing 085 is drawn the same way until the reconciler re-stages it; an actionable issue no maintainer has looked at yet wears `015 Triage needed` (carrying `needs-triage`, or held in the reporter-trust gate's `reporter_triage` bucket — never derived from who filed it) and lights *Actionable* as "Triage needed" (a human's move) — so the Actionable column reads "Actionable", "Triage needed" or "Needs human decision", and the Open column is untouched. An issue carrying a hold-class label (`hold`, `do-not-merge*`, `hive-pause/`) is never parked: it sits at *Open* as "Open · on hold" even beside `needs-human`, and the reconciler removes any 045 the hold caught it with ([#13012](https://github.com/hivecommons/hive/issues/13012)). The lit chevron ends with an ink-coloured icon of whose move it is — person (human), robot (clanker) or hexagon (hive): the claim holder on Claimed, the PR author on PR in progress and Returned to contributor, hive on CI / Review, Adopted by hive and Ready for auto merge, a human on Needs human decision and Ready for human merge. That icon is about the current step only. A **🧑 needs a person** chip marks a row Hive cannot finish on its own: the your-turn decision (the one behind the Overview's *Your turn* and the Hive Advice card) says a person must act, now or at a gate further on such as an approving review branch protection requires or a hand merge, so the chip can show while the lit step is still Hive's move. Its hover text gives the decision's verb and reason, and an issue riding one covering PR takes the PR's answer. `040 Covered by PR` shares the *PR in progress* column with 030; `060 Reviewed`, `080 Awaiting merge admission` and `100 Merge queue` all light *Merge gate* as "Ready for auto merge" (the chevron tooltip says "not yet admitted — " for 060/080 and "admitted, merging" for 100); `095 Pre-merge soak` lights the same column as "Soaking" (hive's move; tooltip "admitted, soaking — Xm left", headline "Pre-merge soak — Xm left" from the payload's `soak_remaining_seconds`, aged from the lane's last sweep), so an admitted PR inside its soak window is no longer drawn as merging; the chevron's own tooltip names the exact stage. Covering pull requests nest under their issue. An issue with no PR yet draws the whole strip like any other row, lit at Open, Actionable or Claimed with the steps ahead faint. Once exactly one PR covers it, issue and PR are a single entry: the issue's `#N title` on the left, `[author icon] #PR title` under the *PR in progress* header on the same title line, and one continuous strip — Open, Actionable and Claimed passed, the PR's stage lighting the chevron from PR in progress through CI / Review, Merge gate and ✓; the chip, tooltip and `since` are the PR's. With two or more covering PRs the issue's own strip stops at Claimed — all passed, nothing on the issue line lit. Directly beneath, tighter than the gap between entries, sits one indented sub-row per open covering PR, furthest along first: `[author icon] #N title` at the *PR in progress* column, then that PR's own strip from there on — `PR in progress → CI / Review → Merge gate → ✓` — with its own lit chevron, chip, tooltip and `since`. So an issue with two PRs, one at 030, shows "PR in progress" lit on that sub-row, not on the issue. A PR linked to an open issue in the list has no top-level row of its own; a PR with no linked issue keeps a full-width row and shows the first steps as passed. A PR whose only linked issue is already closed — a superseded PR whose body still says `Fixes #N` after an earlier PR fixed it — keeps its own row with the closed issue in the issue slot: `#N title` as a link, muted and struck through, followed by a small `closed` chip, so the pairing reads at a glance; when the title cannot be resolved the slot says `#N (closed issue)` rather than a bare number (the title comes from the snapshot, or from one cached issue read the cycle makes once per issue — see [Linked issues](dashboard.md#pr-review-and-link-signals)). The *Pull requests* filter flattens the list — every PR as a top-level row; *Issues* and *All* show the nested view. Each row is named once on the left; a covering PR appears under the *PR in progress* header as `[author icon] #N title`. A pull request with no linked issue is named once: `[PR icon] [author icon] #N title` on the left over one continuous strip (first steps passed, its own step lit) and nothing in the *PR in progress* column, which is reserved for covering PRs — so a title never appears twice in a row. Pull requests are never issues here: when a number arrives through both the issues feed and the pulls feed the PR record wins and the issue copy is dropped on the server (pairing is keyed by repository, kind and number), and the *Issues* filter never lists a PR. The author icon is the one coloured legend: hive hexagon in blue, clanker robot in purple, human silhouette in teal (the `author/*` kind, inferred from the snapshot when the label is absent; the login is in the `#N` link's tooltip). The right-hand `since` column, on the title line, is how long the item has been on its step (red when `stalled`). At most one muted chip sits to its left on the title line: the most important health modifier (`stalled`, `regressed`, `hold`, `needs-human`, `needs-decision` — shown as "needs human decision" —, `needs-triage` — shown as "triage needed" (amber) on an issue still carrying the `needs-triage` label at a stage other than 015, whose own chevron already says so —, `needs-rebase`, `blocked`, `draft`) with a `+N` for the rest. Everything else — the reason sentence, who must act, nudges sent, the repository, all flags and the last stage hops — is in the hover tooltip on the item's `#N` link (on a nested sub-row, on its `#N` PR link). Hover text in a row lives only on the `#N` link, the covering-PR link, the title and the lit chevron (its stage, and whose move it is); passed and upcoming chevrons, the kind and author icons, chips and the `since` column have none. Title lines never wrap or push the strip out of alignment: the issue title is truncated with an ellipsis before the *PR in progress* column (a clear gap always separates it from the PR entry, which starts at the column edge) and the PR title before the chips and `since`; hovering a truncated title shows the full text. Rows are sorted furthest-along first by default — a covered issue ranks by its furthest-along PR — then longest on the step first; *Sort* also offers longest waiting, newest, number and **Hive end to end first** (rows Hive can finish on its own, then rows waiting on someone else such as a PR returned to its author, a person's draft or a hold, then rows that need a person, each group furthest-along first), and the arrow beside it reverses the order, so reversed that sort puts the rows that need a person first. Every open issue and pull request of the watched repositories is listed — actionable, held, claimed, in progress, covered by a PR, filtered by a `needs-*` label, reporter-triage, draft and exempt-labelled alike; an item without a stage label or journal entry sits at *Open*, and one in a scanner bucket with no label yet is placed from the bucket (claimed → 020, in progress → 030, covered → 040). Filters work on the cached payload in the browser and persist across reloads (the page number does not): kind chips, *Repo*, a search box (`#number` / number / title substring, also matching a covering PR), *Since* (any, over 1 hour / 1 day / 1 week on the current step, or past the stage's configured dwell — stalled items always qualify), *Author* kind, *Needs attention* (anything with a health modifier or on 071 / 072 / 085 / 090), and label chips built from every label present, each with its count: a **🙋 Needs human** facet first (attention-coloured; it matches any item carrying a label of the needs-human set — the server's `needs_human_labels`, the same list as the Projects *Needs human* preset: `stage/045-needs-decision`, `stage/085-needs-decision`, `needs-decision`, `needs-direction`, `needs-spec`, `needs-signal`, `needs-human` and `needs-human/*`, the hold labels, `stage/071-needs-author`, `sentinel-alert`, `needs-triage`; its count is the number of distinct items, not the sum of label counts), then every `stage/NNN-*` label in numerical order, then all other labels alphabetically (case-insensitive). When the row would overflow, the **+N more** button opens an in-app *All labels* dialog listing every label in the same order with counts, a type-to-filter box, the same toggles the chips use and a *Clear* button; a label selected there is promoted into the visible row. Chips of one family OR, families AND; *Needs human* ANDs with the label chips. *Clear filters* shows while anything is narrowed. The list is paged — 25 / 50 / 100 rows, default 50 — with Prev / Next and "showing X–Y of N (M filtered out)" above and below; the header count "N items · M need attention" is for the filtered set, the sidebar badge for everythingthe chevron's own tooltip names the exact stage. Covering pull requests nest under their issue. An issue with no PR yet draws the whole strip like any other row, lit at Open, Actionable or Claimed with the steps ahead faint. Once exactly one PR covers it, issue and PR are a single entry: the issue's `#N title` on the left, `[author icon] #PR title` under the *PR in progress* header on the same title line, and one continuous strip — Open, Actionable and Claimed passed, the PR's stage lighting the chevron from PR in progress through CI / Review, Merge gate and ✓; the chip, tooltip and `since` are the PR's. With two or more covering PRs the issue's own strip stops at Claimed — all passed, nothing on the issue line lit. Directly beneath, tighter than the gap between entries, sits one indented sub-row per open covering PR, furthest along first: `[author icon] #N title` at the *PR in progress* column, then that PR's own strip from there on — `PR in progress → CI / Review → Merge gate → ✓` — with its own lit chevron, chip, tooltip and `since`. So an issue with two PRs, one at 030, shows "PR in progress" lit on that sub-row, not on the issue. A PR linked to an open issue in the list has no top-level row of its own; a PR with no linked issue keeps a full-width row and shows the first steps as passed. The *Pull requests* filter flattens the list — every PR as a top-level row; *Issues* and *All* show the nested view. Each row is named once on the left; a covering PR appears under the *PR in progress* header as `[author icon] #N title`. A pull request with no linked issue is named once: `[PR icon] [author icon] #N title` on the left over one continuous strip (first steps passed, its own step lit) and nothing in the *PR in progress* column, which is reserved for covering PRs — so a title never appears twice in a row. Pull requests are never issues here: when a number arrives through both the issues feed and the pulls feed the PR record wins and the issue copy is dropped on the server (pairing is keyed by repository, kind and number), and the *Issues* filter never lists a PR. The author icon is the one coloured legend: hive hexagon in blue, clanker robot in purple, human silhouette in teal (the `author/*` kind, inferred from the snapshot when the label is absent; the login is in the `#N` link's tooltip). The right-hand `since` column, on the title line, is how long the item has been on its step (red when `stalled`). At most one muted chip sits to its left on the title line: the most important health modifier (`stalled`, `regressed`, `hold`, `needs-human`, `needs-decision` — shown as "needs human decision" —, `needs-rebase`, `blocked`, `draft`) with a `+N` for the rest. Everything else — the reason sentence, who must act, nudges sent, the repository, all flags and the last stage hops — is in the hover tooltip on the item's `#N` link (on a nested sub-row, on its `#N` PR link). Hover text in a row lives only on the `#N` link, the covering-PR link, the title and the lit chevron (its stage, and whose move it is); passed and upcoming chevrons, the kind and author icons, chips and the `since` column have none. Title lines never wrap or push the strip out of alignment: the issue title is truncated with an ellipsis before the *PR in progress* column (a clear gap always separates it from the PR entry, which starts at the column edge) and the PR title before the chips and `since`; hovering a truncated title shows the full text. Rows are sorted furthest-along first by default — a covered issue ranks by its furthest-along PR — then longest on the step first; *Sort* also offers longest waiting, newest, number and **Hive end to end first** (rows Hive can finish on its own, then rows waiting on someone else such as a PR returned to its author, a person's draft or a hold, then rows that need a person, each group furthest-along first), and the arrow beside it reverses the order, so reversed that sort puts the rows that need a person first. Every open issue and pull request of the watched repositories is listed — actionable, held, claimed, in progress, covered by a PR, filtered by a `needs-*` label, reporter-triage, draft and exempt-labelled alike; an item without a stage label or journal entry sits at *Open*, and one in a scanner bucket with no label yet is placed from the bucket (claimed → 020, in progress → 030, covered → 040). Filters work on the cached payload in the browser and persist across reloads (the page number does not): kind chips, *Repo*, a search box (`#number` / number / title substring, also matching a covering PR), *Since* (any, over 1 hour / 1 day / 1 week on the current step, or past the stage's configured dwell — stalled items always qualify), *Author* kind, *Needs attention* (anything with a health modifier or on 071 / 072 / 085 / 090), and label chips built from every label present, each with its count: a **🙋 Needs human** facet first (attention-coloured; it matches any item carrying a label of the needs-human set — the server's `needs_human_labels`, the same list as the Projects *Needs human* preset: `stage/045-needs-decision`, `stage/085-needs-decision`, `needs-decision`, `needs-direction`, `needs-spec`, `needs-signal`, `needs-human` and `needs-human/*`, the hold labels, `stage/071-needs-author`, `sentinel-alert`, `needs-triage`; its count is the number of distinct items, not the sum of label counts), then every `stage/NNN-*` label in numerical order, then all other labels alphabetically (case-insensitive). When the row would overflow, the **+N more** button opens an in-app *All labels* dialog listing every label in the same order with counts, a type-to-filter box, the same toggles the chips use and a *Clear* button; a label selected there is promoted into the visible row. Chips of one family OR, families AND; *Needs human* ANDs with the label chips. *Clear filters* shows while anything is narrowed. The list is paged — 25 / 50 / 100 rows, default 50 — with Prev / Next and "showing X–Y of N (M filtered out)" above and below; the header reads "N items (I issues / P PRs) · M need attention" — N, I and P are the whole list, the same `openItems` set the Overview tile and its left-menu counter print, so N always equals the tile's **Total open issues + Total open PRs**; a "· X match filters" note follows while a filter is on. The sidebar counter beside **Hive activity monitor** is that same N — total open items, written in the same render pass as the header, so the two cannot drift apart; how many need attention is the warn-coloured header suffix and the counter's hover tooltip (counted over every item, filters or not). When covering pull requests are nested under their issue the "showing X–Y of R" line counts rows and adds "(P nested under their issues)" so R never reads as a different total. The small monospace strip beside the count — `engine enforce · N anomalies` — is the central stage engine ([ADR-0025](adr/0025-central-stage-engine.md)): `engine` names the current `governor.labels.stage.engine.mode` (green `enforce`, neutral `shadow` = paused, nothing written; hover for what the mode does and the derivation count), `anomalies` counts stage moves the engine's admission table did not expect (red when non-zero). The same numbers are `/api/status.stageEngine`; the mode is changed under **Settings → Labels → Central stage engine mode** and the operator notes are in [operator-reference.md](operator-reference.md#central-stage-engine-rollout). "updated Ns ago · cache" says how old the server-built payload is: the spoke rebuilds it at most every `dashboard.stage_tracker.cache_ttl_s` (default `governor.eval_interval_s`) from its existing snapshot and serves it with an `ETag`, so the browser's 30-second poll is a 304 and no request ever reaches GitHub (`/api/gh-rate-limits` lists the caller `dashboard:stage-tracker` with nothing charged). See [Stage labels](labels-and-control-signals.md#stage-labels-stagennn--the-pipeline-state-machine). +Fed by `GET /api/stage-tracker`. Every issue and pull request follows the same pipeline, named once in the column header above the list: `Open → Actionable (010) → Claimed (020) → PR in progress (030/040) → CI / Review (050/060) → Merge gate (080/100) → ✓ merged`. Each entry is a title line over a thin chevron strip. Only the current step is coloured: it is filled amber and carries the step name in words; once the PR is merged the final ✓ chevron is the lit one, in green. Motion is reserved for what is running or needs a hand — the lit chevron falls into one of three buckets (`stageTrackerMotion()` in `index.html`): **static** (steady): Open, Open · on hold (any hold label wins over every other signal), Actionable (010), Claimed (020), PR in progress / Covered by PR (030/040) by a human author, Merged, and any unrecognised stage; **active** (gentle pulse): the hive is acting — 030/040 by a hive or clanker author, 050 review pending, 060 reviewed, 070 fixing, 074 adopted by hive, 080 merge gate, 095 pre-merge soak, 100 merge queue; **needs-human** (slower amber→red breathe): a person must move — 015 triage needed, 045/085 needs decision, 071 needs author, 072 returned to contributor, 090 ready for a human merge, or a `stalled` / `blocked` / `needs-human` / `needs-decision` / `needs-triage` modifier on any stage. Under `prefers-reduced-motion` nothing animates; a needs-human step keeps a steady red tint so it still stands out. Every other step is a low-contrast shape with a small grey glyph (circle, bolt, lock, branch, eye, shield, check) — passed steps a subtle neutral fill, steps ahead a faint outline. Side stages are not extra steps: `070 Fixing` (the author's move; the `needs-rebase` chip next to it says what is being fixed — the stage labeller always parks a needs-rebase PR at 070/071/072, never at CI / Review), `071 Needs author` (human-authored), `072 Returned to contributor` (clanker-authored) and `074 Adopted by hive` light *CI / Review* with their own name (an adopted PR stays on 074 through the hive's pushes going pending, green or red again, and moves on only when *Merge gate* takes it); `085 Needs human decision` (PR-only) and `090 Ready for human merge` light *Merge gate* with their own name (the 085 tooltip says what must be decided); an issue parked on a person wears `045 Needs human decision` and lights *Actionable* (the issue side) with that name — a pre-045 issue still wearing 085 is drawn the same way until the reconciler re-stages it; an actionable issue no maintainer has looked at yet wears `015 Triage needed` (carrying `needs-triage`, or held in the reporter-trust gate's `reporter_triage` bucket — never derived from who filed it) and lights *Actionable* as "Triage needed" (a human's move) — so the Actionable column reads "Actionable", "Triage needed" or "Needs human decision", and the Open column is untouched. An issue carrying a hold-class label (`hold`, `do-not-merge*`, `hive-pause/`) is never parked: it sits at *Open* as "Open · on hold" even beside `needs-human`, and the reconciler removes any 045 the hold caught it with ([#13012](https://github.com/hivecommons/hive/issues/13012)). The lit chevron ends with an ink-coloured icon of whose move it is — person (human), robot (clanker) or hexagon (hive): the claim holder on Claimed, the PR author on PR in progress and Returned to contributor, hive on CI / Review, Adopted by hive and Ready for auto merge, a human on Needs human decision and Ready for human merge. That icon is about the current step only. A **🧑 needs a person** chip marks a row Hive cannot finish on its own: the your-turn decision (the one behind the Overview's *Your turn* and the Hive Advice card) says a person must act, now or at a gate further on such as an approving review branch protection requires or a hand merge, so the chip can show while the lit step is still Hive's move. Its hover text gives the decision's verb and reason, and an issue riding one covering PR takes the PR's answer. `040 Covered by PR` shares the *PR in progress* column with 030; `060 Reviewed`, `080 Awaiting merge admission` and `100 Merge queue` all light *Merge gate* as "Ready for auto merge" (the chevron tooltip says "not yet admitted — " for 060/080 and "admitted, merging" for 100); `095 Pre-merge soak` lights the same column as "Soaking" (hive's move; tooltip "admitted, soaking — Xm left", headline "Pre-merge soak — Xm left" from the payload's `soak_remaining_seconds`, aged from the lane's last sweep), so an admitted PR inside its soak window is no longer drawn as merging; the chevron's own tooltip names the exact stage. Covering pull requests nest under their issue. An issue with no PR yet draws the whole strip like any other row, lit at Open, Actionable or Claimed with the steps ahead faint. Once exactly one PR covers it, issue and PR are a single entry: the issue's `#N title` on the left, `↳ [author icon] #PR title` under the *PR in progress* header on the same title line (the ↳ marks a PR drawn on its issue's row; the PR link's tooltip says so — "linked PR, shown on its issue's row" — and, when two open issues list the same PR, every row but the lowest-numbered issue adds "also under #N"), and one continuous strip — Open, Actionable and Claimed passed, the PR's stage lighting the chevron from PR in progress through CI / Review, Merge gate and ✓; the chip, tooltip and `since` are the PR's. With two or more covering PRs the issue's own strip stops at Claimed — all passed, nothing on the issue line lit. Directly beneath, tighter than the gap between entries, sits one indented sub-row per open covering PR, furthest along first: `[author icon] #N title` at the *PR in progress* column, then that PR's own strip from there on — `PR in progress → CI / Review → Merge gate → ✓` — with its own lit chevron, chip, tooltip and `since`. So an issue with two PRs, one at 030, shows "PR in progress" lit on that sub-row, not on the issue. A PR linked to an open issue in the list has no top-level row of its own; a PR with no linked issue keeps a full-width row and shows the first steps as passed. A PR whose only linked issue is already closed — a superseded PR whose body still says `Fixes #N` after an earlier PR fixed it — keeps its own row with the closed issue in the issue slot: `#N title` as a link, muted and struck through, followed by a small `closed` chip, so the pairing reads at a glance; when the title cannot be resolved the slot says `#N (closed issue)` rather than a bare number (the title comes from the snapshot, or from one cached issue read the cycle makes once per issue — see [Linked issues](dashboard.md#pr-review-and-link-signals)). The *Pull requests* filter flattens the list — every PR as a top-level row; *Issues* and *All* show the nested view. Each row is named once on the left; a covering PR appears under the *PR in progress* header as `[author icon] #N title`. A pull request with no linked issue is named once: `[PR icon] [author icon] #N title` on the left over one continuous strip (first steps passed, its own step lit) and nothing in the *PR in progress* column, which is reserved for covering PRs — so a title never appears twice in a row. Pull requests are never issues here: when a number arrives through both the issues feed and the pulls feed the PR record wins and the issue copy is dropped on the server (pairing is keyed by repository, kind and number), and the *Issues* filter never lists a PR. The author icon is the one coloured legend: hive hexagon in blue, clanker robot in purple, human silhouette in teal (the `author/*` kind, inferred from the snapshot when the label is absent; the login is in the `#N` link's tooltip). The right-hand `since` column, on the title line, is how long the item has been on its step (red when `stalled`). A plain *Open* step — no stage label yet and not on hold — also carries that age in its own words, "Open · since 3d" ("since 14h" under two days), taken from the same `stage_since` instant the column draws (falling back to the time the item was opened, which the chevron tooltip then marks "(opened)"); past `STAGE_TRACKER_OPEN_STALE_DAYS` (2) the chevron takes a subtle red tint. *Open · hold* never shows the age. An Open item carrying the `blocked` label is drawn the same way as a hold — "Open · blocked" with a ⛔ glyph in the hold's yellow — but keeps the age ("Open · blocked · since 7h"; a stale blocked item is exactly what should stand out, [#14194](https://github.com/hivecommons/hive/issues/14194)) and its chevron tooltip says why: "blocked on owner/repo#N" when the payload names the blocker (`blocked_on`), else "carries the blocked label"; when an item is both held and blocked, hold wins — it is a human instruction. At most one muted chip sits to its left on the title line: the most important health modifier (`stalled`, `regressed`, `hold`, `needs-human`, `needs-decision` — shown as "needs human decision" —, `needs-triage` — shown as "triage needed" (amber) on an issue still carrying the `needs-triage` label at a stage other than 015, whose own chevron already says so —, `needs-rebase`, `blocked`, `draft`) with a `+N` for the rest. Everything else — the reason sentence, who must act, nudges sent, the repository, all flags and the last stage hops — is in the hover tooltip on the item's `#N` link (on a nested sub-row, on its `#N` PR link). Hover text in a row lives only on the `#N` link, the covering-PR link, the title and the lit chevron (its stage, and whose move it is); passed and upcoming chevrons, the kind and author icons, chips and the `since` column have none. Title lines never wrap or push the strip out of alignment: the issue title is truncated with an ellipsis before the *PR in progress* column (a clear gap always separates it from the PR entry, which starts at the column edge) and the PR title before the chips and `since`; hovering a truncated title shows the full text. Rows are sorted furthest-along first by default — a covered issue ranks by its furthest-along PR — then longest on the step first; *Sort* also offers longest waiting, newest, number and **Hive end to end first** (rows Hive can finish on its own, then rows waiting on someone else such as a PR returned to its author, a person's draft or a hold, then rows that need a person, each group furthest-along first), and the arrow beside it reverses the order, so reversed that sort puts the rows that need a person first. Every open issue and pull request of the watched repositories is listed — actionable, held, claimed, in progress, covered by a PR, filtered by a `needs-*` label, reporter-triage, draft and exempt-labelled alike; an item without a stage label or journal entry sits at *Open*, and one in a scanner bucket with no label yet is placed from the bucket (claimed → 020, in progress → 030, covered → 040). Filters work on the cached payload in the browser and persist across reloads (the page number does not): kind chips, *Repo*, a search box (`#number` / number / title substring, also matching a covering PR), *Since* (any, over 1 hour / 1 day / 1 week on the current step, or past the stage's configured dwell — stalled items always qualify), *Author* kind, *Needs attention* (anything with a health modifier or on 071 / 072 / 085 / 090), and label chips built from every label present, each with its count: a **🙋 Needs human** facet first (attention-coloured; it matches any item carrying a label of the needs-human set — the server's `needs_human_labels`, the same list as the Projects *Needs human* preset: `stage/045-needs-decision`, `stage/085-needs-decision`, `needs-decision`, `needs-direction`, `needs-spec`, `needs-signal`, `needs-human` and `needs-human/*`, the hold labels, `stage/071-needs-author`, `sentinel-alert`, `needs-triage`; its count is the number of distinct items, not the sum of label counts), then every `stage/NNN-*` label in numerical order, then all other labels alphabetically (case-insensitive). When the row would overflow, the **+N more** button opens an in-app *All labels* dialog listing every label in the same order with counts, a type-to-filter box, the same toggles the chips use and a *Clear* button; a label selected there is promoted into the visible row. Chips of one family OR, families AND; *Needs human* ANDs with the label chips. *Clear filters* shows while anything is narrowed. The list is paged — 25 / 50 / 100 rows, default 50 — with Prev / Next and "showing X–Y of N (M filtered out)" above and below; the header count "N rows · M need attention" is for the filtered set, the **sidebar badge is the row count of the default view** (owner: the tracker counts rows, not items — a covering PR drawn on its issue's row is not a row of its own, and an issue with several PRs already has several), and both come from the same `stageTrackerCountsText()` call so they cannot disagree. When rows sit below the payload's open issues + PRs, the "showing" line's tooltip and the badge's footnote say why — "2 PRs shown on their issue's row" — rather than printing a second total.the chevron's own tooltip names the exact stage. Covering pull requests nest under their issue. An issue with no PR yet draws the whole strip like any other row, lit at Open, Actionable or Claimed with the steps ahead faint. Once exactly one PR covers it, issue and PR are a single entry: the issue's `#N title` on the left, `[author icon] #PR title` under the *PR in progress* header on the same title line, and one continuous strip — Open, Actionable and Claimed passed, the PR's stage lighting the chevron from PR in progress through CI / Review, Merge gate and ✓; the chip, tooltip and `since` are the PR's. With two or more covering PRs the issue's own strip stops at Claimed — all passed, nothing on the issue line lit. Directly beneath, tighter than the gap between entries, sits one indented sub-row per open covering PR, furthest along first: `[author icon] #N title` at the *PR in progress* column, then that PR's own strip from there on — `PR in progress → CI / Review → Merge gate → ✓` — with its own lit chevron, chip, tooltip and `since`. So an issue with two PRs, one at 030, shows "PR in progress" lit on that sub-row, not on the issue. A PR linked to an open issue in the list has no top-level row of its own; a PR with no linked issue keeps a full-width row and shows the first steps as passed. The *Pull requests* filter flattens the list — every PR as a top-level row; *Issues* and *All* show the nested view. Each row is named once on the left; a covering PR appears under the *PR in progress* header as `[author icon] #N title`. A pull request with no linked issue is named once: `[PR icon] [author icon] #N title` on the left over one continuous strip (first steps passed, its own step lit) and nothing in the *PR in progress* column, which is reserved for covering PRs — so a title never appears twice in a row. Pull requests are never issues here: when a number arrives through both the issues feed and the pulls feed the PR record wins and the issue copy is dropped on the server (pairing is keyed by repository, kind and number), and the *Issues* filter never lists a PR. The author icon is the one coloured legend: hive hexagon in blue, clanker robot in purple, human silhouette in teal (the `author/*` kind, inferred from the snapshot when the label is absent; the login is in the `#N` link's tooltip). The right-hand `since` column, on the title line, is how long the item has been on its step (red when `stalled`). At most one muted chip sits to its left on the title line: the most important health modifier (`stalled`, `regressed`, `hold`, `needs-human`, `needs-decision` — shown as "needs human decision" —, `needs-rebase`, `blocked`, `draft`) with a `+N` for the rest. A `regressed` chip is red only when the row is parked on a person (a human-gate modifier or stage 045 / 071 / 072 / 085 / 090); on a hive-driven stage with nothing waiting on a human it draws as a muted dashed `stage ↓` pill, yields the lead slot to any other modifier, keeps the row out of **Needs attention**, and its tooltip ends "the hive is handling it — no action needed; clears on the next forward step". The marker is set by a genuine backward hop (a lapsed claim 020 → 010, a 030 ↔ 020 flicker and a 040 → 010 after a partially-addressing merge are not regressions) and clears on the first forward hop; see "The regressed marker" in `labels-and-control-signals.md`. Everything else — the reason sentence, who must act, nudges sent, the repository, all flags and the last stage hops — is in the hover tooltip on the item's `#N` link (on a nested sub-row, on its `#N` PR link). Hover text in a row lives only on the `#N` link, the covering-PR link, the title and the lit chevron (its stage, and whose move it is); passed and upcoming chevrons, the kind and author icons, chips and the `since` column have none. Title lines never wrap or push the strip out of alignment: the issue title is truncated with an ellipsis before the *PR in progress* column (a clear gap always separates it from the PR entry, which starts at the column edge) and the PR title before the chips and `since`; hovering a truncated title shows the full text. Rows are sorted furthest-along first by default — a covered issue ranks by its furthest-along PR — then longest on the step first; *Sort* also offers longest waiting, newest, number and **Hive end to end first** (rows Hive can finish on its own, then rows waiting on someone else such as a PR returned to its author, a person's draft or a hold, then rows that need a person, each group furthest-along first), and the arrow beside it reverses the order, so reversed that sort puts the rows that need a person first. Every open issue and pull request of the watched repositories is listed — actionable, held, claimed, in progress, covered by a PR, filtered by a `needs-*` label, reporter-triage, draft and exempt-labelled alike; an item without a stage label or journal entry sits at *Open*, and one in a scanner bucket with no label yet is placed from the bucket (claimed → 020, in progress → 030, covered → 040). Filters work on the cached payload in the browser and persist across reloads (the page number does not): kind chips, *Repo*, a search box (`#number` / number / title substring, also matching a covering PR), *Since* (any, over 1 hour / 1 day / 1 week on the current step, or past the stage's configured dwell — stalled items always qualify), *Author* kind, *Needs attention* (anything with a health modifier or on 071 / 072 / 085 / 090), and label chips built from every label present, each with its count: a **🙋 Needs human** facet first (attention-coloured; it matches any item carrying a label of the needs-human set — the server's `needs_human_labels`, the same list as the Projects *Needs human* preset: `stage/045-needs-decision`, `stage/085-needs-decision`, `needs-decision`, `needs-direction`, `needs-spec`, `needs-signal`, `needs-human` and `needs-human/*`, the hold labels, `stage/071-needs-author`, `sentinel-alert`, `needs-triage`; its count is the number of distinct items, not the sum of label counts), then every `stage/NNN-*` label in numerical order, then all other labels alphabetically (case-insensitive). When the row would overflow, the **+N more** button opens an in-app *All labels* dialog listing every label in the same order with counts, a type-to-filter box, the same toggles the chips use and a *Clear* button; a label selected there is promoted into the visible row. Chips of one family OR, families AND; *Needs human* ANDs with the label chips. *Clear filters* shows while anything is narrowed. The list is paged — 25 / 50 / 100 rows, default 50 — with Prev / Next and "showing X–Y of N (M filtered out)" above and below; the header reads "N items (I issues / P PRs) · M need attention" — N, I and P are the whole list, the same `openItems` set the Overview tile and its left-menu counter print, so N always equals the tile's **Total open issues + Total open PRs**; a "· X match filters" note follows while a filter is on. The sidebar counter beside **Hive activity monitor** is that same N — total open items, written in the same render pass as the header, so the two cannot drift apart; how many need attention is the warn-coloured header suffix and the counter's hover tooltip (counted over every item, filters or not). When covering pull requests are nested under their issue the "showing X–Y of R" line counts rows and adds "(P nested under their issues)" so R never reads as a different total. The small monospace strip beside the count — `engine enforce · N anomalies` — is the central stage engine ([ADR-0025](adr/0025-central-stage-engine.md)): `engine` names the current `governor.labels.stage.engine.mode` (green `enforce`, neutral `shadow` = paused, nothing written; hover for what the mode does and the derivation count), `anomalies` counts stage moves the engine's admission table did not expect (red when non-zero). The same numbers are `/api/status.stageEngine`; the mode is changed under **Settings → Labels → Central stage engine mode** and the operator notes are in [operator-reference.md](operator-reference.md#central-stage-engine-rollout). "updated Ns ago · cache" says how old the server-built payload is: the spoke rebuilds it at most every `dashboard.stage_tracker.cache_ttl_s` (default `governor.eval_interval_s`) from its existing snapshot and serves it with an `ETag`, so the browser's 30-second poll is a 304 and no request ever reaches GitHub (`/api/gh-rate-limits` lists the caller `dashboard:stage-tracker` with nothing charged). See [Stage labels](labels-and-control-signals.md#stage-labels-stagennn--the-pipeline-state-machine). **Repo chip.** Each row carries a small monospace chip before the number naming the repository it belongs to (`dibs #456`, `hive #13524`); the chip is tinted per repository from a fixed palette so one repo always looks the same, the configured primary repo is shown muted, and the full `owner/repo` is in the tooltip. Linked PR sub-rows carry the same chip. Clicking a chip sets the **Repo** filter to that repository (click it again, or pick "All repos", to clear). By default the chip appears only when the list spans more than one repository; the ⚙ settings modal's **Repositories** section offers **Show repo chip: always / only when multiple repos / never** (stored under `hive.tracker.repochip`). @@ -649,9 +660,27 @@ Fed by `GET /api/stage-tracker`. Every issue and pull request follows the same p **Row gutter.** Every issue and PR row (never a column or repo-group header) carries a left gutter with a select checkbox and a ⋯ actions button that is always present and always visible: dimmed at rest (`--ham-tool-rest-opacity`), at full emphasis on the row under the pointer or holding focus, on a selected row, and on every row while anything is selected or the context menu is open — the only animation is the hover fade (`--ham-tool-fade-ms`, ≤ 120 ms), and nothing about it depends on scrolling, row position or which rows the live refresh re-created. -**Row actions (right-click, ⋯, Shift+F10).** Right-clicking a row — or a nested PR sub-row — opens an in-app context menu of the actions that fit that item's current state; the ⋯ button in the row's left gutter opens the same menu for pointer-free use, and so do **Shift+F10** and the context-menu key while the row or anything in it has focus (every row is a tab stop). The menu is positioned at the pointer, clamped to the viewport, uses `menu` / `menuitem` roles, moves focus with ↑ ↓ Home End, opens the *Decide…* group with → and closes it with ←, and closes on Esc, Tab, click-away, scroll or resize, returning focus to the row. Shift+right-click keeps the browser's own menu. Entries, each shown only when it applies: **Decide…** on a row waiting on a person — a PR wearing the *Needs human decision* pill at 085 on the Merge gate column (the primary case: e.g. two competing `dibs` PRs), an issue row whose nested PR is parked there (the menu reads *Decide PR #458…* and acts on that PR, since the issue's own stage is only mirrored from it), or an issue parked at 045. For a PR: *Approve (/lgtm + /approve)* posts those two commands as comments attributed to you — the merge gate takes it from there exactly as if you had typed them; the dashboard reviews, labels and merges nothing — *Close as superseded by…* (a picker listing the other open PRs in the repo, or type `#458` / `owner/repo#458` / a URL; then an optional comment, then the sweep's labelled close), *Won't fix…* (a comment naming you, then a not-planned close) and *Give direction…* (your text is recorded as `/hive decision ` against the head, so the author agent pushes a new head; only on PRs the hive opened). For an issue: *Approve direction* (records the same `/hive approve` a maintainer would comment: the parking labels come off, `approved-direction` goes on, the stage engine logs the un-park and a reply comment names you; nothing is merged), *Send back to the hive…* (`/hive decision `), *Close as superseded by…* and *Close as won't-fix…*. A selection mixing parked issues and parked PRs has Decide… greyed; **Hold** or **Release hold** depending on the `hold` chip (the same `…/items/{number}/hold` toggle as the repository cards, and the same permission — owner, or a signed-in user with write access to the repo); **Re-evaluate now** (the hive-wide read-only rescan behind the Overview's refresh — a rescan already running is reported, not duplicated); **Accept triage** (only on an issue the chevron calls *Triage needed* — stage 015, or the `needs-triage` chip on another stage — never on a PR; the one-click form of a maintainer accepting triage on GitHub: adds the reporter-trust triage label, `triage/accepted` by default or the first of `issue_filter.reporter_trust.untrusted_require_labels`, removes the awaiting label `needs-triage` / `reporter_trust.awaiting_label` and a Hive-written `stage/015-triage-needed`; the row re-reads *Actionable* with the 👤 gone at once, the reporter-trust gate admits the issue on its next poll and the stage engine records 015 → 010; same permission as Hold, audited as `repo_item_accept_triage`, a no-op when already accepted); **Mark needs-human…** (hidden once the chip is on; merger or owner role, applies `needs-human` and a marker comment through the review pipeline's send-to-human endpoint); **Open on GitHub**; **Copy link**. Entries you may not use are greyed with the reason in their tooltip, never hidden. Confirmation and free-text input use the dashboard's own dialogs — no browser prompts. Every action is an existing audited dashboard endpoint (the decision endpoint is `POST /api/repos/{owner}/{repo}/items/{number}/decision` and triage acceptance is `POST /api/repos/{owner}/{repo}/items/{number}/accept-triage`, see [api-reference.md](api-reference.md)); none of them bypasses the hive's governance or merges anything. +**Row actions (right-click, ⋯, Shift+F10).** Right-clicking a row — or a nested PR sub-row — opens an in-app context menu of the actions that fit that item's current state; the ⋯ button in the row's left gutter opens the same menu for pointer-free use, and so do **Shift+F10** and the context-menu key while the row or anything in it has focus (every row is a tab stop). The menu is positioned at the pointer, clamped to the viewport, uses `menu` / `menuitem` roles, moves focus with ↑ ↓ Home End, opens the *Decide…* group with → and closes it with ←, and closes on Esc, Tab, click-away, scroll or resize, returning focus to the row. Shift+right-click keeps the browser's own menu. Entries, each shown only when it applies: **Decide…** on a row waiting on a person — a PR wearing the *Needs human decision* pill at 085 on the Merge gate column (the primary case: e.g. two competing `dibs` PRs), an issue row whose nested PR is parked there (the menu reads *Decide PR #458…* and acts on that PR, since the issue's own stage is only mirrored from it), or an issue parked at 045. The first entry is always *Let hive decide* (see [Let hive decide](#let-hive-decide) below): the hive decides as the owner's delegate and records it the way a person's decision is recorded. For a PR: *Approve (/lgtm + /approve)* posts those two commands as comments attributed to you — the merge gate takes it from there exactly as if you had typed them; the dashboard reviews, labels and merges nothing — *Close as superseded by…* (a picker listing the other open PRs in the repo, or type `#458` / `owner/repo#458` / a URL; then an optional comment, then the sweep's labelled close), *Won't fix…* (a comment naming you, then a not-planned close) and *Give direction…* (your text is recorded as `/hive decision ` against the head, so the author agent pushes a new head; only on PRs the hive opened). For an issue: *Approve direction* (records the same `/hive approve` a maintainer would comment: the parking labels come off, `approved-direction` goes on, the stage engine logs the un-park and a reply comment names you; nothing is merged), *Send back to the hive…* (`/hive decision `), *Close as superseded by…* and *Close as won't-fix…*. A selection mixing parked issues and parked PRs has Decide… greyed; **Hold** or **Release hold** depending on the `hold` chip (the same `…/items/{number}/hold` toggle as the repository cards, and the same permission — owner, or a signed-in user with write access to the repo); **Re-evaluate now** (the hive-wide read-only rescan behind the Overview's refresh — a rescan already running is reported, not duplicated); **Accept triage** (only on an issue the chevron calls *Triage needed* — stage 015, or the `needs-triage` chip on another stage — never on a PR; the one-click form of a maintainer accepting triage on GitHub: adds the reporter-trust triage label, `triage/accepted` by default or the first of `issue_filter.reporter_trust.untrusted_require_labels`, removes the awaiting label `needs-triage` / `reporter_trust.awaiting_label` and a Hive-written `stage/015-triage-needed`; the row re-reads *Actionable* with the 👤 gone at once, the reporter-trust gate admits the issue on its next poll and the stage engine records 015 → 010; same permission as Hold, audited as `repo_item_accept_triage`, a no-op when already accepted); **Mark needs-human…** (hidden once the chip is on; merger or owner role, applies `needs-human` and a marker comment through the review pipeline's send-to-human endpoint); **Open on GitHub**; **Copy link**. Entries you may not use are greyed with the reason in their tooltip, never hidden. Confirmation and free-text input use the dashboard's own dialogs — no browser prompts. Every action is an existing audited dashboard endpoint (the decision endpoint is `POST /api/repos/{owner}/{repo}/items/{number}/decision` and triage acceptance is `POST /api/repos/{owner}/{repo}/items/{number}/accept-triage`, see [api-reference.md](api-reference.md)); none of them bypasses the hive's governance or merges anything. + +**Multi-select.** The checkbox in a row's left gutter selects the row; **Shift+click** selects the range from the last clicked row, **Cmd/Ctrl+click** toggles, **Cmd/Ctrl+Shift+click** adds a range to what is selected; with a row focused, **Space** toggles it (Shift+Space extends). Cmd/Ctrl- or Shift-clicking the row itself, away from its links, selects it too. A selection bar appears above the list with the count, **Clear**, and the same actions for the whole selection — *Decide…* (opens the group as a menu, *Let hive decide* first — each selected row gets its own delegated-decision job, see below), *Hold*, *Release hold*, *Re-evaluate now*, *Mark needs-human…*, *Copy links* — each enabled only when it is valid for every selected row and otherwise greyed with the first reason ("Not applicable to acme/console#12", "Sign in with write access…"). Right-clicking a selected row acts on the whole selection. Bulk actions run one request per row, in list order, under a progress toast ("Hold 2/5 — acme/console#12…"); the result toast lists each failure by row and the successes stand, after which the list and the status snapshot refresh (a hive-wide rescan is sent once per batch). Selection is keyed on the row's item (repository, kind, number), not its node, so the live re-render keeps it; a row that leaves the list — merged, closed, filtered or paged out — leaves the selection. + +### Let hive decide + +**What it is.** *Let hive decide* is the first entry under *Decide…* on a row parked at *Needs human decision* (an issue at 045, a PR at 085, or an issue whose nested PR is parked there), and under *Decide…* in the selection bar for a bulk run. It asks the hive to make the decision itself, as the owner's delegate, under the owner's standing operating rules — and to record it exactly as a human decision is recorded: a `/hive decision ` comment on the item (plus the queue approval on a PR head when the decision is to release the PR), the parking labels lifted, `triage/accepted` added where it applies, and the stage engine's un-park logged with the requesting operator named. The item leaves 085/045 and proceeds without the owner. The action is audited as `repo_item_decision_delegated` (or `repo_item_decision_delegated_failed`) and the **Decisions log** shows those rows with a *delegated* badge so delegated and human decisions can be told apart and counted. + +**How it runs.** Clicking it posts `POST /api/repos/{owner}/{repo}/items/{number}/decide-delegated` (body `{"type": "issue"|"pr", "reason", "caption", "sentinel_reason"}` — the row's park words, so the decider reads why the hive stopped) which enqueues one delegated-decision job per item (a second click while a job is running answers 409 with the running job). The page shows *Hive is deciding acme/console#12…*, polls the same URL (`GET`) until the job reports `done` or `failed`, then the result toast reads what was decided — "Hive decided: acme/console#12 approve (small-low-risk)" — and the row refreshes. A job gathers the item (title, body, labels, author, state), the PR's mergeable state and changed files, whether the author is a hive account, the author's repository permission (trusted = write or above), the sentinel label, and the row's park reason, then runs the policy and executes the resulting actions through the same GitHub helpers the manual *Decide…* entries use. Permission is the one *Hold* uses (owner, or a signed-in user with write access to the repository); the requesting user is named in the recorded decision as the person who delegated it. + +**The policy.** The rules live in one place — `src/pkg/delegatedecision/policy.go` (`PolicyText`, the owner's words verbatim, including the contributor-adequacy rail adopted in #13998; the Go rules beside it implement them) — so the dashboard, docs and any future LLM-backed decider share the same text. In short: be decisive and, except for the safety rails, never ask the owner a question back; pick the option that keeps work moving and is reversible. **PRs:** prefer an adequate contributor PR over a hive duplicate (a mergeable, current competitor prompts a human comparison of issue coverage, checks and review approval; keep the hive PR open and parked until adequacy is confirmed, rather than closing it solely on an issue reference; an unmergeable contributor PR is superseded with credit); approve small, necessary, low-risk changes (CI pins, read-only RBAC adds, warning text, workflow hygiene) once the sentinel finding is checked and explained; a sentinel alert on a trusted author's PR is approved with a one-line "reviewed, intended" rationale; a reporter-trust hold is cleared when the fixing PR is sound; a contributor who has gone quiet is superseded now, not after a deadline (#13975). **Issues:** dedupe aggressively (`Duplicate of #N`, cross-link overlaps); secrets / PAT / org-permission items are owner-only — `needs-owner` + `hold` with one comment stating the exact owner action, and no more failure issues for that root cause; flaky tests are quarantined with a retry and a deflake task rather than blocking; CI hygiene gate proposals (actionlint, duplication ratchets, link-checker allowlists) are approved as ratchets (baseline; fail only on increases; warning-only briefly, then enforce); design proposals with concrete acceptance criteria are accepted as proposed with explicit defaults; a "review backlog" item is never auto-closed when its PR closes unmerged (drop the relation, comment, re-triage). Each decision names the rule that fired (`dedupe`, `owner-only`, `flaky-test`, `ci-ratchet`, `design-proposal`, `review-backlog`, `default-approve`, `prefer-contributor`, `contributor-offline`, `sentinel-trusted`, `small-low-risk`, `reporter-trust`, `fix-then-merge`, `default-release`, `hive-policy`, `sentinel-untrusted`, `rail-hold`) in the comment and in the job's `decision.rule`. + +**Safety rails** (`ApplyRails`, applied after the policy and before anything is written): the delegate never merges a PR — it only records the approval the auto-merge engine already acts on per repository settings (a merger/owner requester's approval is written as the queue line the sweep parses; anyone else's as `/lgtm` + `/approve` comments attributed to them); it does not touch an item wearing `hold` (refused, nothing written — release the hold first if you want it decided); it never grants org membership, secrets or permissions (those items get `needs-owner` + `hold` and the one owner-action comment, and keep their park); and a question that is a policy or scope change to the hive itself (settings, governance docs, security posture — the `governance` label or those words in the park reason) is left parked: the delegate posts "hive recommends: …; owner to confirm" and keeps 085. A sentinel-flagged PR whose author lacks write access is likewise a recommendation only. Because the sentinel sweep re-applies its label when a hive account removes it, a trusted-author sentinel approval posts the "reviewed, intended" comment and leaves the final queueing to the person who clears the label — the job's `notes` say so. + +**Audit marker.** Every comment the delegate writes ends with `` (and a visible "Decided by … as the owner's delegate under the owner-delegate policy (rule `…`)" line), so a delegated decision can always be told from a human one on GitHub, in the Decisions log, and in any later count. + +**Setting.** `decisions.delegate.enabled` (default **true** — the same default as the other "let the hive do it" switches); `decisions.delegate.repos..enabled` (or the bare repository name) overrides per repository. `decisions.delegate.llm.enabled` (default **false**) adds the LLM step described below. When it is off the menu entry stays visible but the endpoint answers 403 naming the setting. See [operator-reference.md](operator-reference.md). -**Multi-select.** The checkbox in a row's left gutter selects the row; **Shift+click** selects the range from the last clicked row, **Cmd/Ctrl+click** toggles, **Cmd/Ctrl+Shift+click** adds a range to what is selected; with a row focused, **Space** toggles it (Shift+Space extends). Cmd/Ctrl- or Shift-clicking the row itself, away from its links, selects it too. A selection bar appears above the list with the count, **Clear**, and the same actions for the whole selection — *Decide…* (opens the group as a menu), *Hold*, *Release hold*, *Re-evaluate now*, *Mark needs-human…*, *Copy links* — each enabled only when it is valid for every selected row and otherwise greyed with the first reason ("Not applicable to acme/console#12", "Sign in with write access…"). Right-clicking a selected row acts on the whole selection. Bulk actions run one request per row, in list order, under a progress toast ("Hold 2/5 — acme/console#12…"); the result toast lists each failure by row and the successes stand, after which the list and the status snapshot refresh (a hive-wide rescan is sent once per batch). Selection is keyed on the row's item (repository, kind, number), not its node, so the live re-render keeps it; a row that leaves the list — merged, closed, filtered or paged out — leaves the selection. +**What this release decides mechanically.** The decider is rules-first: the cases above that are mechanically decidable (dedupe, owner-only, flaky, CI ratchet, design proposal, review backlog, contributor-offline, sentinel-trusted, small-low-risk, reporter-trust, hive-policy) are decided from the item itself; anything else falls to the reversible default (approve direction on an issue; release a green hive PR; ask for the fix first when the head is red or conflicting). Competing-PR detection and a real adopt-supersede trigger are tracked in a follow-up issue linked from the PR that added this action. + +**LLM step for the non-mechanical cases.** With `decisions.delegate.llm.enabled: true` (default **false**), an item the rules leave at the reversible default (rule `default-approve` or `default-release`) is shown to one *owner-delegate decider* turn: the item, its comment thread, sentinel findings, linked issues/PRs, the repository's auto-merge setting, the author's association and reporter-trust state, its labels, and `PolicyText` verbatim. The model answers with a structured `{verdict, text, rationale, actions[]}`; the reply is rejected unless every verdict, action kind and label is in the delegate's closed set and fits the item (no `approve-review` on an issue, no action on a park-keeping verdict, …), and unless the actions carry the verdict out: an acting verdict must include its recording action (`comment-decision`, or `close-as-duplicate` for the close verdicts, plus `approve-review` / `adopt-supersede` where the verdict names them), `labels-remove` can never name `stage/085-needs-decision` (only the recording action lifts the park, with the comment and the marker), and `close-duplicate` closes the newer item in favour of an older one. An accepted reply is recorded under rule `llm-delegate` and goes through the same safety rails and executor as a rule decision; a model error or rejected reply keeps the reversible default and the job's `notes` say why. Rules-first stays authoritative: every case a rule decides, and every item wearing `hold`, never reaches the model. The call uses the governor reviewer endpoint (`governor.trajectory`, falling back to `governor.litellm`); `decisions.delegate.llm.model` overrides the model. ## Agent activity @@ -719,3 +748,21 @@ This section exists on the v6 line only. **When it appears.** On v6 only, always. **Settings that change it.** Which workers can take staged work changes what appears. See [Runs](runs.md). + +## Platform + +Which code hosting service Hive is connected to and which optional platform services are switched on. + +This section exists on the v6 line only. On v5 the same facts are in the Diagnostics section. + +**What it tells you.** It shows the code hosting service, such as GitHub, GitLab or Gitea, and how many repositories Hive watches. It also shows whether optional services, such as the token minting service, are on. + +**How the numbers are worked out.** The repository count is the number of repositories Hive watches. The other tiles show on or off. + +**What it is good for.** Check it after setup to confirm Hive is connected where you expect. + +**Example.** It shows `GitHub · 3 repos` and "Mint on". Hive watches three GitHub repositories and the token service runs. + +**When it appears.** On v6 only, when the hive reports platform data. + +**Settings that change it.** Your code hosting settings and optional service settings change it. The code hosting service is `project.forge`; a self-managed GitLab or a Gitea/Forgejo instance also shows its URL. See [Forges](integrations/forges.md). diff --git a/src/docs/dashboard.md b/src/docs/dashboard.md index bb8faed1fc..d8ad02cf73 100644 --- a/src/docs/dashboard.md +++ b/src/docs/dashboard.md @@ -208,6 +208,20 @@ autocomplete and argument hints. Current commands are `/help`, `/clear`, `/search`, `/history`, `/retry`, `/edit`, `/who`, `/agents`, `/beads`, `/prs`, `/governor`, `/knowledge`, `/jam`, and `/spek`. +`/prs` answers locally from the status snapshot (the same cached rows +`/api/prs` serves, so it costs no GitHub call): the open pull requests grouped +by repo, one row each — `owner/repo#N — title (@author · stage labels · age · +✅/❌/⏳ CI · approved / changes requested · needs decision · hold · mergeable)` +— under the aggregate count, capped at 25 rows with an "… and N more" footer. +The remainder of the command narrows the list and every filter is a +conjunct: `waiting on review` / `needs review` (not a draft, no approving +review, no outstanding change request), `needs decision` (`stage/085-needs-decision` or +`stage/045-needs-decision`), `hold` (held set or a hold label), `failing ci`, +`mine` (the signed-in viewer) / `by ` / `@login`, `repo ` or +a bare `owner/repo` token (the short repo name also works), and `stale` +(untouched for 7 days) / `older than 3d` (`h`, `d`, `w`). Words the parser does +not recognise are reported and ignored, and the list is still printed. + The prompt behaves like a small terminal: `Enter` sends, `Shift+Enter` inserts a newline, `Tab` completes slash commands, Up/Down cycle prior commands while preserving the draft, `?` opens the shortcut overlay, `Esc` closes the overlay @@ -227,6 +241,66 @@ is collapsible with copy buttons on fenced code blocks. There is no server-side per-user chat persistence yet because dashboard user settings are not a general-purpose cross-browser preferences API. +Agent questions are answered from the hive's own agent state (`/api/status` +→ `agents[]`), not by dumping the roster (#14409). Bare `/agents` (or "list +agents") prints the roster, with the block reason inline for every `BLOCKED` +agent. `/agents stuck|blocked|paused|running|idle|failed` filters it: blocked +agents show WHY (`blockedReason`: rate-limit cap, inference auth, policy +stand-down, crash loop, failed starts…) and since when (`blockedSince`); +paused agents show who paused them and why. `/agents [status|why|last| +history]`, `/agents status `, or plain "scanner status" / "status of +scanner" return only that agent's card — state, block reason + since, current +task, last completed turn and its outcome, last kick, last PR/issue acted on, +restart count; an unknown name answers `No agent named X; agents: …`. Natural +language such as "which agent completed last?", "what is scanner doing?", +"why is reviewer-2 blocked?", "when did scanner last run?" and "which agents +are idle?" goes through the same table-driven matcher. An agent question the +matcher does not recognise is handed to the configured chat responder with +the agent state attached as context; without a responder the chat says what +it does understand instead of printing the roster. + +### How a message is answered + +The server routes every message (`pkg/dashboard/api_chat_router.go`) into +one of three paths, and never discards a message's arguments: + +- **Bare command** — `/agents`, `/prs`, `/beads`, `/status` with no + arguments (or a one-word keyword such as `agents` or `prs?`) prints the + deterministic summary: the agent roster, per-repo open PR counts, bead + counts per store, the hive health line. +- **Deterministic argument handlers** — the `/agents` matcher above (and + the `/prs` filters, #14408) answer the argument vocabularies they + recognise without a model turn. +- **Grounded answer** — any other command with arguments or plain-words + question (`/agents what should the hive do next`, `what's waiting on my + decision`, `which PRs have failing CI`, `what version are we on`) is answered + by one read-only model turn over a compact snapshot of the hive state the + dashboard already holds in memory: agents (state, paused/blocked reason, + awaiting-CI, last kick and outcome, last completed action, last error), + open PRs (stage labels, review state, requested reviewers, CI and failing + checks, mergeability, merge verdict, age), actionable issues (stage, + priority, assignees, age), items parked for a decision, governor mode and + next kick, system alerts, release/commit and the last landed upgrade. The + snapshot is scoped to the command's dataset (`/prs …` carries no agents), + capped at 40 items per list and 24 KiB overall. The model is instructed to + answer from the snapshot only, to say *"I don't have that data"* rather + than guess, to cite the agents and `owner/repo#N` items it used, and that + chat cannot take actions. The turn runs on the governor reviewer endpoint + (`governor.trajectory`, falling back to `governor.litellm`) with a 25 s + timeout and at most two turns in flight per dashboard; each attempt writes + one `chat_grounded_answer` audit line (scope, question, snapshot size, + outcome). Nothing on this path can mutate hive state. +- **Fallback** — when grounding cannot run (setting off, no reviewer + endpoint, timeout, backend error, too many in flight) a scoped question + gets the command's usage naming the reason; it is never answered with the + bare dump. Unscoped free text falls through to the configured + `ChatResponder` when one exists, else the "guide agent unavailable" + message. + +**Setting.** `chat.grounded_answers.enabled` (default **true**) gates the +grounded path hive-wide; `chat.grounded_answers.model` overrides the reviewer +model for the grounding turn. See [operator-reference.md](operator-reference.md). + Markdown is rendered from escaped input only: bold, inline code, code fences, links, GitHub-style `@mentions`, and `#issue` / `owner/repo#N` references are decorated after escaping. Repeated or long `Status heartbeat` output is folded diff --git a/src/docs/design/README.md b/src/docs/design/README.md index 73d38b5e27..0ebad56897 100644 --- a/src/docs/design/README.md +++ b/src/docs/design/README.md @@ -56,6 +56,8 @@ that status is the thing to check before treating a page as current behaviour: `local`, so an assistant-mediated action is indistinguishable from a dashboard one. - [Crux: the agent runtime boundary](crux.md) — **Stage 1 shipped (v5); Stage 2 blocked on `pkg/config`.** RFC [#13536](https://github.com/hivecommons/hive/issues/13536): names the agent runtime inside Hive (`pkg/agent` and its ring) as Crux, lists what stays in Hive, records the staged plan (in-repo boundary → nested module → `crux serve`) and the naming decision. Inbound and outbound ratchets in `src/internal/testutil/crux_*_ratchet_test.go`, the tmux funnel, `crux.Runtime`, Crux-owned value types, CI path filters and the `crux` CLI are live; the nested module waits on an agent-runtime config leaf (the doc lists the measured outbound edges and prerequisites). +- [Crux: generic tool policy for the MITM proxy](crux-tool-policy.md) — **Design only (proposed).** #14228, child of #14138: a shared operation vocabulary with per-forge adapters, a `relay()` verdict, deny-by-default for unrecognised operations, and the proxy's evaluation pipeline written down once. It also covers migrating behind a parity test and treating #14211 as the first non-GitHub adapter. + - [PR follow-up session resume](pr-follow-up-resume.md) - **shipped (v5, default off).** #9583: ties each agent-opened PR to the CLI session that authored it (a `pkg/turn` envelope per PR) and feeds CI failures, changes-requested reviews, new review-bot threads and comments from people with write access back into that session without `/clear` while it is still live. When the session is gone (a restart or the next regular kick), the follow-up takes the fix-before-new path and the agent's next kick carries a compact handoff note of the PR's original reasoning. Pointers are pruned on merge or close; a live-review soak is the remaining follow-up. - [Achievement system 2.0](achievements-2.md) — **shipped (v5).** Spektacular-ready spec for #8832: teamwork-first Solo/Dual/Fireteam/Raid tiers, local-model achievements, the “local always wins” mastery tier, dark-pattern guardrails, anti-gaming rules, data sources, and shipped dossier/leaderboard rollout. @@ -216,7 +218,7 @@ that status is the thing to check before treating a page as current behaviour: whether Copilot token usage can be captured per request with repo context and timestamps at the proxy. Its answer is a recommendation **against** building phase 4 as scoped: per-request usage is available and already captured - (`src/pkg/proxy/github_proxy.go:2662`), but per-request repo context is not + (`src/crux/proxy/github_proxy.go:2662`), but per-request repo context is not available at the proxy in any form, and the nearest substitute — a "last `api.github.com` repo seen" from `ExtractRepo` — is an inference from read traffic, weaker than the audited `repo=` events phase 3 already joins against. @@ -262,7 +264,7 @@ that status is the thing to check before treating a page as current behaviour: benign work on an assigned third-party repo ran that repo's own test suite, which reached the operator's real bootloader via `rpm-ostree kargs`, stopped only by polkit. Maps the default tmux launch path - (`src/pkg/agent/manager.go`, `bin/agent-launch.sh`) against the three + (`src/crux/agent/manager.go`, `bin/agent-launch.sh`) against the three deployment modes (hub pod, containerized `contribute-hive`, `contribute-hive ... local`) and establishes precisely what #4938's merged host-state denylist covers — the reported command family, on the `Bash` tool surface, by bare diff --git a/src/docs/design/advisor-lane.md b/src/docs/design/advisor-lane.md index df09937f7d..1f0a352a80 100644 --- a/src/docs/design/advisor-lane.md +++ b/src/docs/design/advisor-lane.md @@ -96,7 +96,7 @@ from `.github/hooks/`, which would mean committing hive's hook into the target r that path is ruled out and the hive-owned hooks directory is the one used. Hub-launched agents already get a per-agent home directory -(`/data/home/agents/`, `src/pkg/agent/interactive_home.go`), so the Copilot and Codex +(`/data/home/agents/`, `src/crux/agent/interactive_home.go`), so the Copilot and Codex hook files have a place to live that hive controls without touching anything the operator owns. ### Roles diff --git a/src/docs/design/agent-host-confinement.md b/src/docs/design/agent-host-confinement.md index e074aab8de..d99b2fdedc 100644 --- a/src/docs/design/agent-host-confinement.md +++ b/src/docs/design/agent-host-confinement.md @@ -63,7 +63,7 @@ ask is done; see [Doc status](#doc-status-already-corrected-by-4938) below. The claude-family permission flags are `--dangerously-skip-permissions --permission-mode bypassPermissions` unconditionally -(`config/backends.conf:69`, mirrored at `src/pkg/agent/manager_launch.go:861`, base +(`config/backends.conf:69`, mirrored at `src/crux/agent/manager_launch.go:861`, base string `--model %s --dangerously-skip-permissions%s`). Full tool-permission bypass is not gated by anything added in #4938 — it is the unconditional base. @@ -75,10 +75,10 @@ Workspace "confinement" is a `cd`, not a boundary: directory only; nothing stops the CLI process — running as the contributor's own uid — from reading or writing any path that uid can reach. - Hub-pod agents: `launchInTmux` starts the session with `new-session -d -s - session -c dir` (`src/pkg/agent/manager_tmux.go:165`), then, when the agent has a + session -c dir` (`src/crux/agent/manager_tmux.go:165`), then, when the agent has a UID assigned, runs the tmux command through `su-exec`/an exec-user spec - (`agentExecUserSpec`, `src/pkg/agent/manager_tmux.go:178-186`, - `src/pkg/agent/manager_tmux.go:207-212`) so the CLI runs as `hive-` or a + (`agentExecUserSpec`, `src/crux/agent/manager_tmux.go:178-186`, + `src/crux/agent/manager_tmux.go:207-212`) so the CLI runs as `hive-` or a synthetic `uid:gid`, not as the same uid as every other agent. This is **process-identity separation between agents on the same pod**, not filesystem confinement to the workspace — there is no mount namespace, no @@ -106,16 +106,16 @@ Bash(rpm-ostree:*), Bash(bootc:*), Bash(ostree:*), Bash(grubby:*), Bash(bootctl:*), Bash(efibootmgr:*) ``` -(`src/pkg/agent/manager_pause.go:176-182`, shell mirror at +(`src/crux/agent/manager_pause.go:176-182`, shell mirror at `config/backends.conf:86`, parity enforced by -`src/pkg/agent/host_state_deny_test.go:88-95` `TestShellAndGoDenyListsAgree`, +`src/crux/agent/host_state_deny_test.go:88-95` `TestShellAndGoDenyListsAgree`, which sources `config/backends.conf` and diffs the two lists byte-for-byte). Applied on both launch paths: relay (`config/backends.conf:88-100` -`claude_family_perm_flag`) and hub-pod (`src/pkg/agent/manager_launch.go:865`, `base + +`claude_family_perm_flag`) and hub-pod (`src/crux/agent/manager_launch.go:865`, `base + claudeGitHubWriteDenyFlags + claudeHostStateDenyFlags()`). An escape hatch, `HIVE_CLAUDE_DANGEROUSLY_ALLOW_HOST_STATE`, drops the deny fragment entirely on -either path when set truthy (`src/pkg/agent/manager_launch.go:668-673`, +either path when set truthy (`src/crux/agent/manager_launch.go:668-673`, `config/backends.conf:92-99`). The PR body documents that this was verified against a real CLI (Claude Code @@ -133,9 +133,9 @@ command's leading token, not a boundary enforced by the OS.** It has the shape of every denylist: it stops the literal named binaries when invoked as a bare command word through the `Bash` tool, and nothing establishes that it stops equivalent effects reached another way. None of the following are covered by -anything in `src/pkg/agent/manager_launch.go:628-684` or +anything in `src/crux/agent/manager_launch.go:628-684` or `config/backends.conf:41-77`, and no test in -`src/pkg/agent/host_state_deny_test.go` exercises them: +`src/crux/agent/host_state_deny_test.go` exercises them: - **Absolute or relative path invocation.** `Bash(rpm-ostree:*)` matches the leading token `rpm-ostree`; whether Claude Code's matcher also catches @@ -183,11 +183,11 @@ title names. The exposure is not uniform. Three shapes exist in this repo today: -1. **Hub-hosted pod agents (containerized hive deployment).** `src/pkg/agent/manager.go` +1. **Hub-hosted pod agents (containerized hive deployment).** `src/crux/agent/manager.go` is the manager for this mode; the K8s manifest at `src/deploy/k8s/deployment.yaml` defines the pod's own security context. Agents on this path get per-agent uid separation via `agentExecUserSpec`/`su-exec` - (`src/pkg/agent/manager_tmux.go:178-186`) but no filesystem namespace scoping — + (`src/crux/agent/manager_tmux.go:178-186`) but no filesystem namespace scoping — "the host" an agent can reach here is the container's filesystem view, bounded by whatever the pod's volumes and security context expose. This is the mode where "the operator's host" is a fungible pod, not a workstation; @@ -221,17 +221,17 @@ boundary with confidence. ## The opt-in Podman sandbox, and why it's off by default -`src/pkg/sandbox/sandbox.go` and `src/pkg/agent/sandbox_executor.go` implement +`src/crux/sandbox/sandbox.go` and `src/crux/agent/sandbox_executor.go` implement a working rootless-Podman path: `PodmanArgs` builds `run --rm --userns=keep-id -v ::Z --network=` with an explicit env allowlist that -filters credential-shaped names (`src/pkg/sandbox/sandbox.go:123-146`, +filters credential-shaped names (`src/crux/sandbox/sandbox.go:123-146`, `isCredentialName` at `:150-152`). This is real workspace confinement: the container only has the one bind-mounted workspace directory, not the host filesystem. It is double-gated off: `AgentConfig.SandboxEnabled` requires both the global `agent_sandbox.enabled` bool and a per-agent `sandbox.enabled` pointer to be -true (`src/pkg/crux/config/agent_runtime.go:524-529`); both default to their zero value +true (`src/crux/config/agent_runtime.go:524-529`); both default to their zero value (`false`/`nil`), so an agent is on the sandbox path only if an operator finds and sets both knobs. `sandbox-isolation.md` confirms this is deliberate: "Sandbox execution is opt-in and the tmux path remains unchanged for all @@ -316,17 +316,17 @@ estimated from reading the code, not measured. without necessarily restricting file writes elsewhere — and is normally layered with, not instead of, a namespace/container boundary; the existing Podman path does not currently set an explicit seccomp profile beyond - Podman's own default (`src/pkg/sandbox/sandbox.go:123-146` sets no + Podman's own default (`src/crux/sandbox/sandbox.go:123-146` sets no `--security-opt seccomp=`). Cost: profile authorship and maintenance as agent tool use evolves; a too-strict profile silently breaks legitimate git/build tooling in ways that are hard to diagnose from an agent transcript. - **Dedicated low-privilege UID with no polkit/sudo rights, for the `contribute-hive local` / native case specifically.** Hive already has a precedent for per-agent UID separation on the hub-pod path - (`src/pkg/agent/uidmap.go`, `agentExecUserSpec` at - `src/pkg/agent/manager_tmux.go:178-186`), but that map is scoped to hub-pod + (`src/crux/agent/uidmap.go`, `agentExecUserSpec` at + `src/crux/agent/manager_tmux.go:178-186`), but that map is scoped to hub-pod agents inside an already-containerized deployment (`UIDMapPath = - "/var/run/hive/uid-map.json"`, `src/pkg/agent/uidmap.go:17`) — nothing + "/var/run/hive/uid-map.json"`, `src/crux/agent/uidmap.go:17`) — nothing currently wires an equivalent mechanism into `contribute-hive local`, which is the mode that actually reached the operator's bootloader. Applying the same idea there (a `hive-agent` system user with no polkit rule granting it @@ -364,9 +364,9 @@ containerized and, especially, `local` mode), with the current denylist (#4938) kept as the floor for hosts where the sandbox genuinely cannot run.** The sandbox is already built, already does real workspace confinement via a bind mount and `--userns=keep-id` -(`src/pkg/sandbox/sandbox.go:123-146`), and is exactly the shape the Codex +(`src/crux/sandbox/sandbox.go:123-146`), and is exactly the shape the Codex comparison in the original issue argues for. The double gate -(`src/pkg/crux/config/agent_runtime.go:524-529`) is a configuration default, not a +(`src/crux/config/agent_runtime.go:524-529`) is a configuration default, not a missing capability — closing it is materially cheaper than any option in category B that requires new code (bwrap/seccomp/VM), and it directly answers the mode where the incident actually happened (`local`, no container, no @@ -410,7 +410,7 @@ overstated claim in the shipped text. - Whether Claude Code's `Bash(cmd:*)` deny pattern matches absolute-path invocation (`/usr/bin/rpm-ostree`) or only the bare command word. Not tested - in `src/pkg/agent/host_state_deny_test.go`; the PR's own verification table + in `src/crux/agent/host_state_deny_test.go`; the PR's own verification table tested the bare-command case only. - Whether a shell function/alias/wrapper that ultimately execs a denied binary is caught by the same `Bash` tool-call parsing, or bypasses it. diff --git a/src/docs/design/agent-state-inventory.md b/src/docs/design/agent-state-inventory.md index 69baef26ef..9b621fa4cf 100644 --- a/src/docs/design/agent-state-inventory.md +++ b/src/docs/design/agent-state-inventory.md @@ -33,7 +33,7 @@ backend CLI that hive cannot externalize without headless turns. A hosted agent is not a process hive controls turn-by-turn. It is a backend CLI (`claude`, `codex`, `copilot`, `agy`, `goose`, `bob`, …) launched into a detached tmux session that the spoke's agent manager owns -(`src/pkg/agent/manager.go`, `Manager` / `AgentProcess`; sessions are named +(`src/crux/agent/manager.go`, `Manager` / `AgentProcess`; sessions are named `hive-` — `manager.go:846`). The manager's contract with the agent is deliberately thin: @@ -180,7 +180,7 @@ key generations (`/data/saas/hub-generations.json`). | # | State | Where (v5) | On a spoke (dashboard) pod roll | Verdict | |---|-------|------------|--------------------------------|---------| -| 27 | Completed-task ledger + per-task cooldowns + PR URLs | `completedTasks` / `completedTaskCooldown` / `completedTaskPRURL` (`contribute_ws.go:353`), persisted to `/data/contributors/completed-tasks.json` | Survives; was moved to the PVC ledger through the #3792 → #3989 lineage after re-offer loops | **fixed** | +| 27 | Completed-task ledger + per-task cooldowns + PR URLs | `completedTasks` / `completedTaskCooldown` / `completedTaskPRURL` (`contribute_ws.go:965`), persisted to `/data/contributors/completed-tasks.json` | Survives; was moved to the PVC ledger through the #3792 → #3989 lineage after re-offer loops | **fixed** | | 28 | Failure cooldowns + quarantine counters | `failedTasks` / `consecutiveFailures` (`:373`, `:381`), persisted to `/data/contributors/failed-tasks.json` | Survives (#2435 livelock fix, made durable with the ledger) | **fixed** | | 29 | No-PR completion streaks | `noPRStreaks` (`:394`), `/data/contributors/no-pr-streaks.json` | Survives (#3980; geometric backoff must outlive the completion entry it escalates) | **fixed** | | 30 | `no_work_needed` verdicts | `noWorkVerdicts` (`:416`), `/data/contributors/no-work-verdicts.json` | Survives (#3987/#3997) — "persisted in the same PVC-backed ledger dir as the cooldowns so a pod restart does not forget the verdict" | **fixed** | diff --git a/src/docs/design/agent-turn-handoff.md b/src/docs/design/agent-turn-handoff.md index 69f735b14d..4dec361125 100644 --- a/src/docs/design/agent-turn-handoff.md +++ b/src/docs/design/agent-turn-handoff.md @@ -77,7 +77,7 @@ than as a stale paragraph nobody re-reads. `logscrub` on the way out. This closes residual item 3 from the stage-2 report on the issue, which listed atomic persistence as assumed rather than done. - **The motivating problem now has an instrument.** `TurnLoss` - (`src/pkg/agent/turn_loss.go:96`), recorded by `noteTurnInterruptedLocked` + (`src/crux/agent/turn_loss.go:96`), recorded by `noteTurnInterruptedLocked` (`:117`) through the single teardown funnel `tearDownTurnLocked` (`:185`). What steps 1 and 2 explicitly did **not** settle is concurrency: the stage-2 @@ -284,7 +284,7 @@ the whole decision reduces to: **does resuming mid-turn context save enough to justify hive owning the conversation?** **And the instrument cannot answer it yet.** `TurnLoss` -(`src/pkg/agent/turn_loss.go:96`) is honest about what it measures: `UpperBound` +(`src/crux/agent/turn_loss.go:96`) is honest about what it measures: `UpperBound` is explicitly the most a restart *could* have cost, `Producing` is the threshold-free count of interruptions that certainly hit a working agent. Both are collected on the **tmux path** — the path where, by §4, hive cannot own the diff --git a/src/docs/design/agent-turn-model.md b/src/docs/design/agent-turn-model.md index 5cd2b5bf16..93bb6bc43e 100644 --- a/src/docs/design/agent-turn-model.md +++ b/src/docs/design/agent-turn-model.md @@ -48,35 +48,35 @@ plainly — that is the main thing a spike is for. The backend CLI is launched **once per tmux session**, as an interactive foreground process typed into a shell pane, and it stays there across many turns. `launchInTmux` creates a detached shell session -(`src/pkg/agent/manager_tmux.go:172`, `newSessionCommands`, which issues a plain -`tmux new-session -d` at `src/pkg/agent/manager_tmux.go:201`) and then *types the CLI +(`src/crux/agent/manager_tmux.go:172`, `newSessionCommands`, which issues a plain +`tmux new-session -d` at `src/crux/agent/manager_tmux.go:201`) and then *types the CLI invocation into that shell*: ```go -m.tmuxSendLiteralForAgent(agent, fullCmd) // src/pkg/agent/manager_tmux.go:553 +m.tmuxSendLiteralForAgent(agent, fullCmd) // src/crux/agent/manager_tmux.go:553 time.Sleep(textToEnterDelay) -m.tmuxSendEntersForAgent(agent) // src/pkg/agent/manager_tmux.go:558 +m.tmuxSendEntersForAgent(agent) // src/crux/agent/manager_tmux.go:558 ``` The composed command line is per-backend and is built as a string — `claude --model … --dangerously-skip-permissions` -(`src/pkg/agent/manager_launch.go:849`), `copilot … --allow-all` -(`src/pkg/agent/manager_launch.go:859`), and so on. +(`src/crux/agent/manager_launch.go:849`), `copilot … --allow-all` +(`src/crux/agent/manager_launch.go:859`), and so on. Nothing in the kick path ever spawns a process. `deliverKickLocked` -(`src/pkg/agent/manager_kick.go:242`) only manipulates the existing pane. +(`src/crux/agent/manager_kick.go:242`) only manipulates the existing pane. ### 1.2 A turn starts with simulated keystrokes -`SendKick` (`src/pkg/agent/manager_kick.go:102`) is the entry point; it delegates to -`deliverKickLocked` (`src/pkg/agent/manager_kick.go:242`), which: +`SendKick` (`src/crux/agent/manager_kick.go:102`) is the entry point; it delegates to +`deliverKickLocked` (`src/crux/agent/manager_kick.go:242`), which: - clears stale input with `C-c` then `C-u` (Codex uses only `C-u`; Goose skips clearing) - (`src/pkg/agent/manager_kick.go:285-296`), + (`src/crux/agent/manager_kick.go:285-296`), - types the prompt literally, chunked, via - `tmuxSendLiteralForAgent` (`src/pkg/agent/manager_kick.go:332`, implementation at - `src/pkg/agent/manager_tmux.go:553` → `tmux send-keys -l`), -- submits with `tmuxSendEntersForAgent` (`src/pkg/agent/manager_kick.go:363`). + `tmuxSendLiteralForAgent` (`src/crux/agent/manager_kick.go:332`, implementation at + `src/crux/agent/manager_tmux.go:553` → `tmux send-keys -l`), +- submits with `tmuxSendEntersForAgent` (`src/crux/agent/manager_kick.go:363`). Pane writes return errors to the kick caller. A failed clear, prompt chunk, or Enter stops delivery, increments the existing undeliverable counter, and leaves @@ -97,15 +97,15 @@ turns, so there is no exit to wait for. Turn completion is **inferred from pane text**, by two independent mechanisms: **The readiness gate before the next kick.** -`waitForInputPromptForAgent` (`src/pkg/agent/manager_tmux.go:496`) polls +`waitForInputPromptForAgent` (`src/crux/agent/manager_tmux.go:496`) polls `captureTmuxPaneForAgent` and returns once `paneShowsInputPrompt` -(`src/pkg/agent/manager_pane_signals.go:413`) matches a per-backend marker string. `SendKick` -calls it at `src/pkg/agent/manager_kick.go:178`. This gate is the only thing that +(`src/crux/agent/manager_pane_signals.go:413`) matches a per-backend marker string. `SendKick` +calls it at `src/crux/agent/manager_kick.go:178`. This gate is the only thing that stops hive typing a new prompt on top of an in-flight response. The gate is bounded by `inputPromptTimeout` (120s), which is why operator kicks from the dashboard do **not** use `SendKick` directly. `SendKickAsync` -(`src/pkg/agent/kick_async.go`) keeps the fast, deterministic preconditions +(`src/crux/agent/kick_async.go`) keeps the fast, deterministic preconditions synchronous and runs this gate plus the typing on a background goroutine, so the HTTP handler cannot outlive an ingress idle timeout and report a succeeding kick as a 504 failure (kubestellar/hive#5325). Delivery is deduplicated per agent, so @@ -113,21 +113,21 @@ a retried click cannot type the prompt twice. The governor's tick still calls `SendKick` synchronously — it has no proxy in front of it. **The pane poller.** `pollTmuxOutputForAgent` -(`src/pkg/agent/manager_poll.go:13`) runs a `3 * time.Second` ticker -(`src/pkg/agent/manager_poll.go:15`) for the agent's whole lifetime, diffing +(`src/crux/agent/manager_poll.go:13`) runs a `3 * time.Second` ticker +(`src/crux/agent/manager_poll.go:15`) for the agent's whole lifetime, diffing captured pane content to maintain `agent.LastPaneChange` -(`src/pkg/agent/manager_poll.go:102`) — hive's only evidence that a running, +(`src/crux/agent/manager_poll.go:102`) — hive's only evidence that a running, authenticated CLI is actually doing something. Stall recovery is likewise textual: `nudgeIfKickStalled` -(`src/pkg/agent/manager_kick.go:907`) compares `paneContentHash(pane)` against -`agent.lastInferKickPane` (`src/pkg/agent/manager_kick.go:883`), and -`paneShowsActiveWork` (`src/pkg/agent/manager_kick.go:823`) matches a spinner or a +(`src/crux/agent/manager_kick.go:907`) compares `paneContentHash(pane)` against +`agent.lastInferKickPane` (`src/crux/agent/manager_kick.go:883`), and +`paneShowsActiveWork` (`src/crux/agent/manager_kick.go:823`) matches a spinner or a token-counter marker. ### 1.4 Scheduling: one shared ticker, not a per-turn continuation -**Correction to the task framing: `src/pkg/agent/scheduler.go` does not exist.** +**Correction to the task framing: `src/crux/agent/scheduler.go` does not exist.** Kick *timing* lives in `pkg/governor`; kick *text* is built in `pkg/scheduler`. - `Governor.agentsDueForKick` (`src/pkg/governor/governor.go:887`) compares @@ -158,17 +158,17 @@ CLI subprocess. | State | Where | Citation | |---|---|---| -| Pause flag (one bool per agent) | `/data/hive.yaml` via `AgentConfig.Paused` | `src/pkg/crux/config/agent_config.go:223`; writer `SetAgentPausedAndSave` `src/pkg/config/save.go:64` | +| Pause flag (one bool per agent) | `/data/hive.yaml` via `AgentConfig.Paused` | `src/crux/config/agent_config.go:219`; writer `SetAgentPausedAndSave` `src/pkg/config/save.go:64` | | Pause provenance (`PausedAt`, `PausedReason`, `PausedTrigger`, `PausedBy`), CLI/model pins, model/backend overrides, restart count, `LastKick`, truncated kick history | `/data/hive-state.json` via `snapshot.AgentState` | `src/pkg/snapshot/state.go:82-117`; path `src/cmd/hive/spokewire.go:3` | -| Watchdog failure count, crash-loop latch, backoff deadline, healthy-since, conditions | same file, `snapshot.PersistedState.Watchdog` | `src/pkg/snapshot/state.go:48`; `watchdog.PersistedAgent` `src/pkg/watchdog/reconciler.go:295` | +| Watchdog failure count, crash-loop latch, backoff deadline, healthy-since, conditions | same file, `snapshot.PersistedState.Watchdog` | `src/pkg/snapshot/state.go:48`; `watchdog.PersistedAgent` `src/crux/watchdog/reconciler.go:216` | | Fleet-breaker engagement + held set | same file, `BreakerState` | `src/pkg/snapshot/state.go:56` | | Governor budget/spend/eval history, cadence overrides, ACMM level | same file | `src/pkg/snapshot/state.go:16-42` | -| **Full text of every delivered prompt** | `/data/prompt-history.jsonl` (lumberjack-rotated JSONL) | `src/pkg/dashboard/prompt_history.go:45`; writer `Server.RecordPrompt` `src/pkg/dashboard/prompt_history.go:353`, wired at `src/cmd/hive/main.go:3253` | -| **Rendered terminal scrollback, per kick** | `/data/logs/kicks//-.log` | `src/pkg/agent/kick_logs.go:43` (`defaultKickLogDir`); writer `archiveKickLogLocked` `src/pkg/agent/kick_logs.go:180` | -| Token-usage summary | `/data/token-summary.json` | `src/pkg/tokens/collector.go:194`, `:112-116` | -| Structured audit trail | `/data/audit.jsonl`, reloaded into a ring at boot | `src/pkg/dashboard/audit.go:22`, `loadFromDisk` `:94` | -| Agent-name → UID allocation | durable source `/data/.hive/uid-map.json`, runtime copy `/var/run/hive/uid-map.json` | `PersistedUIDMapPath` / `UIDMapPath` `src/pkg/agent/uidmap.go`; load `src/pkg/agent/manager.go` | -| Backend CLI's own session/credential files | the CLI's own `HOME` / `CODEX_HOME`, rooted at `/data/home` | per-agent `CODEX_HOME` `src/pkg/agent/manager_env.go:510`, helper `src/pkg/agent/manager_homes.go:56`; per-agent HOME `src/pkg/agent/interactive_home.go:29`; shared `.claude` bridged by symlink `interactive_home.go:67` | +| **Full text of every delivered prompt** | `/data/prompt-history.jsonl` (lumberjack-rotated JSONL) | `src/pkg/dashboard/prompt_history.go:24`; writer `Server.RecordPrompt` `src/pkg/dashboard/prompt_history.go:361`, wired at `src/cmd/hive/main.go:4035` | +| **Rendered terminal scrollback, per kick** | `/data/logs/kicks//-.log` | `src/crux/agent/kick_logs.go:22` (`defaultKickLogDir`); writer `archiveKickLogLocked` `src/crux/agent/kick_logs.go:178` | +| Token-usage summary | `/data/token-summary.json` | `src/pkg/tokens/collector.go:202`, `:608-611` | +| Structured audit trail | `/data/audit.jsonl`, reloaded into a ring at boot | `src/pkg/dashboard/audit.go:23`, `loadFromDisk` `:189` | +| Agent-name → UID allocation | durable source `/data/.hive/uid-map.json`, runtime copy `/var/run/hive/uid-map.json` | `PersistedUIDMapPath` / `UIDMapPath` `src/crux/agent/uidmap.go`; load `src/crux/agent/manager.go` | +| Backend CLI's own session/credential files | the CLI's own `HOME` / `CODEX_HOME`, rooted at `/data/home` | per-agent `CODEX_HOME` `src/crux/agent/manager_env.go:510`, helper `src/crux/agent/manager_homes.go:56`; per-agent HOME `src/crux/agent/interactive_home.go:29`; shared `.claude` bridged by symlink `interactive_home.go:67` | Two caveats on that table: @@ -184,33 +184,33 @@ Two caveats on that table: ### 2.2 In-process — lost on restart All of the following are fields of `AgentProcess` -(`src/pkg/agent/manager.go:208`) or goroutines owned by the manager. None is +(`src/crux/agent/manager.go:208`) or goroutines owned by the manager. None is written anywhere. | State | Field / mechanism | Citation | |---|---|---| -| Live output ring buffer | `OutputBuffer *RingBuffer` — pure in-memory circular slice, no persistence path | `src/pkg/agent/manager.go:252`; `src/pkg/agent/ringbuffer.go` | -| Last captured pane content | `lastPaneCapture []string`, guarded by `paneMu` | `src/pkg/agent/manager.go:253` | -| Activity clock | `LastPaneChange` (guarded by `paneMu`) | `src/pkg/agent/manager.go:366` | -| Most recent kick text | `LastKickMessage` — replaced on every kick; the durable copy is truncated (`KickRecord.Snippet`) or lives in prompt-history | `src/pkg/agent/manager.go:259`; the prompt-history store is contrasted with the 120-character snippet at `src/pkg/dashboard/prompt_history.go:186-188` | -| tmux session/socket identity | `tmuxSession`, `tmuxSocket` | `src/pkg/agent/manager.go:266-267` | -| Per-launch cancel func and launch generation | `cancel context.CancelFunc`, `launchGen int` | `src/pkg/agent/manager.go:268`, `:404` | -| Launch-serialization flag | `launching bool` (guarded by `m.mu`) | `src/pkg/agent/manager.go:278` | -| One-shot bootstrap override | `BootstrapOverride` | `src/pkg/agent/manager.go:283` | -| Token-restart backoff ladder | `lastTokenRestart`, `tokenRestartAttempts`, `tokenRestartGaveUp` | `src/pkg/agent/manager.go:303`, `:322`, `:325` | -| Login / quota observations | `NeedsLogin`, `QuotaExhausted` | `src/pkg/agent/manager.go:344`, `:351` | -| Consent-screen watcher timers | `consentSeenAt`, `lastConsentDismiss` | `src/pkg/agent/manager.go:367-368` | -| Stall-watchdog per-kick state | `lastInferKickAt`, `lastInferKickPane`, `stallNudgeSent`, `lastInferKickMarks`, `actionNudgeSent` | `src/pkg/agent/manager.go:369-372`, `:405`, `:424` | -| Transient-API-error nudge cooldown | `lastTransientNudge`, `transientNudgesThisKick` | `src/pkg/agent/manager.go:388-389` | -| Un-archived-scrollback flag | `kickLogPending` (guarded by `m.mu`) | `src/pkg/agent/manager.go:417` | -| Sandbox / bob-key latches, last launch banner | `sandboxResumeAfterCancel`, `awaitingBobKey`, `lastLaunchFailureBanner` | `src/pkg/agent/manager.go:435`, `:444`, `:471` | -| Poller goroutines themselves | `go m.pollTmuxOutputForAgent(agent, agentCtx)` and siblings, tied to a per-launch context | `src/pkg/agent/manager_launch.go:246`, `:249`, `:256` | -| Blocked-action thrash windows | `Manager.thrash map[string]*thrashState` under its own `thrashMu` — deliberately *not* `m.mu`, to avoid re-entrancy from the output-capture goroutines | `src/pkg/agent/manager.go:567-571`; `thrashState` `src/pkg/agent/manager_thrash.go:65`; trip logic `recordBlockedAndCheck` `src/pkg/agent/manager_thrash.go:104` | +| Live output ring buffer | `OutputBuffer *RingBuffer` — pure in-memory circular slice, no persistence path | `src/crux/agent/manager.go:252`; `src/crux/agent/ringbuffer.go` | +| Last captured pane content | `lastPaneCapture []string`, guarded by `paneMu` | `src/crux/agent/manager.go:253` | +| Activity clock | `LastPaneChange` (guarded by `paneMu`) | `src/crux/agent/manager.go:366` | +| Most recent kick text | `LastKickMessage` — replaced on every kick; the durable copy is truncated (`KickRecord.Snippet`) or lives in prompt-history | `src/crux/agent/manager.go:259`; the prompt-history store is contrasted with the 120-character snippet at `src/pkg/dashboard/prompt_history.go:186-188` | +| tmux session/socket identity | `tmuxSession`, `tmuxSocket` | `src/crux/agent/manager.go:266-267` | +| Per-launch cancel func and launch generation | `cancel context.CancelFunc`, `launchGen int` | `src/crux/agent/manager.go:268`, `:404` | +| Launch-serialization flag | `launching bool` (guarded by `m.mu`) | `src/crux/agent/manager.go:278` | +| One-shot bootstrap override | `BootstrapOverride` | `src/crux/agent/manager.go:283` | +| Token-restart backoff ladder | `lastTokenRestart`, `tokenRestartAttempts`, `tokenRestartGaveUp` | `src/crux/agent/manager.go:303`, `:322`, `:325` | +| Login / quota observations | `NeedsLogin`, `QuotaExhausted` | `src/crux/agent/manager.go:344`, `:351` | +| Consent-screen watcher timers | `consentSeenAt`, `lastConsentDismiss` | `src/crux/agent/manager.go:367-368` | +| Stall-watchdog per-kick state | `lastInferKickAt`, `lastInferKickPane`, `stallNudgeSent`, `lastInferKickMarks`, `actionNudgeSent` | `src/crux/agent/manager.go:369-372`, `:405`, `:424` | +| Transient-API-error nudge cooldown | `lastTransientNudge`, `transientNudgesThisKick` | `src/crux/agent/manager.go:388-389` | +| Un-archived-scrollback flag | `kickLogPending` (guarded by `m.mu`) | `src/crux/agent/manager.go:417` | +| Sandbox / bob-key latches, last launch banner | `sandboxResumeAfterCancel`, `awaitingBobKey`, `lastLaunchFailureBanner` | `src/crux/agent/manager.go:435`, `:444`, `:471` | +| Poller goroutines themselves | `go m.pollTmuxOutputForAgent(agent, agentCtx)` and siblings, tied to a per-launch context | `src/crux/agent/manager_launch.go:246`, `:249`, `:256` | +| Blocked-action thrash windows | `Manager.thrash map[string]*thrashState` under its own `thrashMu` — deliberately *not* `m.mu`, to avoid re-entrancy from the output-capture goroutines | `src/crux/agent/manager.go:567-571`; `thrashState` `src/crux/agent/manager_thrash.go:65`; trip logic `recordBlockedAndCheck` `src/crux/agent/manager_thrash.go:104` | Note the asymmetry: several counters that exist precisely to *stop a runaway loop* (`tokenRestartAttempts`, `transientNudgesThisKick`, `stallNudgeSent`) are in-process only. A restart resets each of them to zero. The watchdog's own -backoff ladder is persisted (`src/pkg/watchdog/reconciler.go:374`) specifically +backoff ladder is persisted (`src/crux/watchdog/reconciler.go:374`) specifically so that it doesn't have this property; the manager-side nudge counters were not given the same treatment. This is an observation about current behaviour, not a bug report — see Open questions. @@ -218,7 +218,7 @@ bug report — see Open questions. ### 2.3 tmux state, and the one accidental survival path tmux is a separate process tree with its own server per agent socket -(`src/pkg/agent/manager_tmux.go:118` builds `tmux -L `). What tmux holds — +(`src/crux/agent/manager_tmux.go:118` builds `tmux -L `). What tmux holds — the running CLI process and its scrollback — is therefore outside hive's memory and *can* outlive a hive process restart. @@ -233,31 +233,31 @@ if !agent.forceRelaunch && m.tmuxPaneHasCLIForAgent(agent) { return nil } ``` -(`src/pkg/agent/manager_launch.go:235-281`) +(`src/crux/agent/manager_launch.go:235-281`) Three things about this are worth stating precisely: - **It is one of two reattach points, and both are early returns rather than a recovery routine.** The other is `ensureTmuxSession` - (`src/pkg/agent/manager_tmux.go:285`), whose first act is + (`src/crux/agent/manager_tmux.go:285`), whose first act is `if m.tmuxSessionExistsForAgent(agent) { return nil }` - (`src/pkg/agent/manager_tmux.go:286-288`) — a surviving session is reused, not + (`src/crux/agent/manager_tmux.go:286-288`) — a surviving session is reused, not recreated. Boot reaches both: `main` unconditionally calls `agentMgr.Start(ctx, name)` for every enabled agent (`src/cmd/hive/main.go:3674`) and the reuse-vs-relaunch decision is taken inside. There is no `Adopt`, `Reattach`, or `RecoverAgents` function; searching for one finds only `RestoreBreaker` - (`src/pkg/agent/manager_pause.go:382`), which restores control metadata and is + (`src/crux/agent/manager_pause.go:382`), which restores control metadata and is explicitly documented as *not* touching agent state: "a boot restore must never change agent state, only reattach the breaker" - (`src/pkg/agent/manager_pause.go:379-380`). + (`src/crux/agent/manager_pause.go:379-380`). Because reattachment is emergent from two independent early returns rather than an explicit path, nothing in the codebase names it, tests it end to end, or reports whether it happened. The only trace is a log line - (`src/pkg/agent/manager_launch.go:236`). + (`src/crux/agent/manager_launch.go:236`). - **The adoption test is a screen-scrape.** `tmuxPaneHasCLIForAgent` - (`src/pkg/agent/manager_tmux.go:441`) is one line: `paneHasCLIMarker( + (`src/crux/agent/manager_tmux.go:441`) is one line: `paneHasCLIMarker( m.captureVisiblePaneForAgent(agent))`. Hive decides whether an agent's conversation is still alive by pattern-matching rendered terminal output. - **It does not survive the deployment's actual restart mode.** In hosted @@ -265,7 +265,7 @@ Three things about this are worth stating precisely: destroys every tmux server in the container. The `kick_logs.go` header states this outright: graceful shutdown must archive scrollback because "a pod roll or hive image upgrade … destroys every tmux server in the container" - (`src/pkg/agent/kick_logs.go:19-21`, `:334-336`). + (`src/crux/agent/kick_logs.go:19-21`, `:334-336`). So the reattach path helps only when the hive **process** restarts inside a surviving **container**. For the case the RFC names — spoke rolls — it does not @@ -275,33 +275,33 @@ apply. ## 3. What a restart actually does to a turn -`Restart` (`src/pkg/agent/manager_restart.go:722`) does not resume anything. In order: +`Restart` (`src/crux/agent/manager_restart.go:722`) does not resume anything. In order: 1. `C-c` into the pane and cancel the launch context - (`src/pkg/agent/manager_restart.go:838-843`). + (`src/crux/agent/manager_restart.go:838-843`). 2. Archive the scrollback *before* it is destroyed — `tearDownTurnLocked(agent, "restart")` - (`src/pkg/agent/manager_restart.go:849`), with the comment that `kill-session` + (`src/crux/agent/manager_restart.go:849`), with the comment that `kill-session` destroys the only record of the previous run - (`src/pkg/agent/manager_restart.go:845-848`). + (`src/crux/agent/manager_restart.go:845-848`). 3. Reap the CLI process and any UID-owned helpers - (`src/pkg/agent/manager_restart.go:866-876`). -4. `tmux kill-session` (`src/pkg/agent/manager_restart.go:878`). + (`src/crux/agent/manager_restart.go:866-876`). +4. `tmux kill-session` (`src/crux/agent/manager_restart.go:878`). The replacement CLI starts with **no prompt at all** in the default case. -`buildBootstrapPrompt` (`src/pkg/agent/manager_env.go:19`) unconditionally returns `""` -(`src/pkg/agent/manager_env.go:27`), with the reasoning recorded inline: the +`buildBootstrapPrompt` (`src/crux/agent/manager_env.go:19`) unconditionally returns `""` +(`src/crux/agent/manager_env.go:27`), with the reasoning recorded inline: the governor's first eval cycle kicks all due agents with fully substituted templates, whereas sending a boot prompt here leaked unsubstituted `${ISSUE_LIST}` -placeholders to the agent (`src/pkg/agent/manager_env.go:20-25`). +placeholders to the agent (`src/crux/agent/manager_env.go:20-25`). So a restarted agent sits idle at its input prompt until the shared governor ticker (§1.4) next finds it due. Only the explicit override paths supply text -directly: `RestartWithBootstrap` (`src/pkg/agent/manager_restart.go:419`) sets +directly: `RestartWithBootstrap` (`src/crux/agent/manager_restart.go:419`) sets `BootstrapOverride`, and `RestartThenSendKick` -(`src/pkg/agent/manager_restart.go:505`) is documented as restarting "with a clean +(`src/crux/agent/manager_restart.go:505`) is documented as restarting "with a clean slate". Whichever path runs, delivery is gated on the new pane reaching its -input marker (`deliverStartupKick`, `src/pkg/agent/manager_kick.go:480`). +input marker (`deliverStartupKick`, `src/crux/agent/manager_kick.go:480`). **Restart is a reset, not a resume.** Whatever the agent had established in its conversation — plan, working context, partial tool results — is gone, and the @@ -314,32 +314,32 @@ that a restart happened or that earlier work exists. `src/pkg/watchdog/` observes liveness and, in acting modes, restarts. -Its input is an `Observation` (`src/pkg/watchdog/classify.go:47`) whose +Its input is an `Observation` (`src/crux/watchdog/classify.go:47`) whose substantive fields are `Pane string` — the visible pane, deliberately excluding scrollback so "stale markers in scroll history must not vouch for a dead CLI" -(`src/pkg/watchdog/classify.go:52-53`) — plus boolean verdicts derived from that +(`src/crux/watchdog/classify.go:52-53`) — plus boolean verdicts derived from that pane and a `LastChange` timestamp from the poller. It classifies each pane into one `PaneClass` -(`src/pkg/watchdog/classify.go:13-31`): `ready`, `shell-prompt`, +(`src/crux/watchdog/classify.go:13-31`): `ready`, `shell-prompt`, `auth-required`, `stuck-overlay`, `no-output`, `no-session`, `unknown`. -`Dead()` (`src/pkg/watchdog/classify.go:37`) returns true for `shell-prompt`, +`Dead()` (`src/crux/watchdog/classify.go:37`) returns true for `shell-prompt`, `stuck-overlay`, `no-output`, and `no-session`. `auth-required` is deliberately excluded, because restarting into a dead credential produces a restart loop -(`src/pkg/watchdog/classify.go:34-36`). The pattern tables it matches against are +(`src/crux/watchdog/classify.go:34-36`). The pattern tables it matches against are literal CLI UI chrome — `"Login expired"`, `"Please run /login"` -(`src/pkg/watchdog/classify.go:99-104`), `"[Next]"`, `"Choose an accent"` -(`src/pkg/watchdog/classify.go:110-119`). +(`src/crux/watchdog/classify.go:99-104`), `"[Next]"`, `"Choose an accent"` +(`src/crux/watchdog/classify.go:110-119`). -Its action is `r.fleet.Restart(ctx, name)` (`src/pkg/watchdog/reconciler.go:661`, +Its action is `r.fleet.Restart(ctx, name)` (`src/crux/watchdog/reconciler.go:661`, via `planRestartLocked` at `:576` and `restartDetached` at `:657`) — i.e. exactly the destructive path in §3. **What the watchdog does about lost in-process state: nothing, and by design it cannot.** It restores its *own* ladder across restarts -(`Snapshot`/`Restore`, `src/pkg/watchdog/reconciler.go:374` and `:408`), so it +(`Snapshot`/`Restore`, `src/crux/watchdog/reconciler.go:374` and `:408`), so it "neither forgets a crash-loop nor re-runs a backoff ladder from the top" -(`src/pkg/watchdog/reconciler.go:401-402`). It has no mechanism to restore the +(`src/crux/watchdog/reconciler.go:401-402`). It has no mechanism to restore the agent's conversation, because none exists to call. From the watchdog's point of view a restarted agent is a fresh agent that happens to keep its name and its failure count. @@ -357,16 +357,16 @@ RFC. ### 5.1 Hive does not own the conversation, and cannot read it Every supported backend is an opaque interactive subprocess. Hive composes a -command line as a string (`src/pkg/agent/manager_launch.go:849`, -`src/pkg/agent/manager_launch.go:859`, `:888`), +command line as a string (`src/crux/agent/manager_launch.go:849`, +`src/crux/agent/manager_launch.go:859`, `:888`), types it into a shell, and from then on interacts only through keystrokes in and rendered characters out. Hive sets `HOME` and a per-agent `CODEX_HOME` -(`src/pkg/agent/manager_env.go:510`, helper at `src/pkg/agent/manager_homes.go:56`) so +(`src/crux/agent/manager_env.go:510`, helper at `src/crux/agent/manager_homes.go:56`) so that each agent's CLI writes its session files somewhere hive controls the *location* of. That is location control, not format control: nothing in -`src/pkg/agent/` parses, writes, or migrates a backend session file. +`src/crux/agent/` parses, writes, or migrates a backend session file. Notably, **no launch path passes a resume/continue/session-id flag** to any backend. The composed command lines carry model and permission flags only. There @@ -394,9 +394,9 @@ It is easy to mistake either for one; neither is. asked to do?"; the store's own comment still contrasts itself with the 120-character snippet (`src/pkg/dashboard/prompt_history.go:186-188`). No model response, tool call, or tool result is recorded. - `/data/logs/kicks/…` holds **rendered terminal text**, captured with - `tmux capture-pane -p -J` (`src/pkg/agent/terminal.go:109-111`). It is a + `tmux capture-pane -p -J` (`src/crux/agent/terminal.go:109-111`). It is a human debugging artifact, retained to 10 files / 64 MiB per agent - (`src/pkg/agent/kick_logs.go:28`, `:39`). It contains ANSI-rendered, + (`src/crux/agent/kick_logs.go:28`, `:39`). It contains ANSI-rendered, width-wrapped, spinner-animated output with no message boundaries and no role structure. It is not machine-replayable into any backend. @@ -410,11 +410,11 @@ Whatever shape a handoff envelope takes, hive has a working pattern for it: a `.tmp` + rename (`src/pkg/snapshot/state.go:179-190`), aged out at seven days (`src/pkg/snapshot/state.go:14`), restored at boot by `restoreAgentRuntimeState` (`src/cmd/hive/state_restore.go`). The watchdog's -`PersistedAgent` (`src/pkg/watchdog/reconciler.go:216`) shows how a subsystem +`PersistedAgent` (`src/crux/watchdog/reconciler.go:216`) shows how a subsystem contributes a durable subset of its in-memory record to that envelope without persisting the whole thing, including the judgement call about *which* fields are safe to restore under which mode -(`src/pkg/watchdog/reconciler.go:401-407`). +(`src/crux/watchdog/reconciler.go:401-407`). If step 2 externalizes manager-side state, this is the mechanism to extend rather than a new one to invent. @@ -423,10 +423,10 @@ rather than a new one to invent. This is the load-bearing constraint behind all of the above and is worth isolating. Turn completion (§1.3), liveness classification (§4), stall -detection (`src/pkg/agent/manager_kick.go:907`), auth failure -(`src/pkg/watchdog/classify.go:99-104`), quota exhaustion -(`src/pkg/agent/manager.go:351`), and even whether a surviving CLI can be -adopted (`src/pkg/agent/manager_tmux.go:441`) are all decided by matching substrings +detection (`src/crux/agent/manager_kick.go:907`), auth failure +(`src/crux/watchdog/classify.go:99-104`), quota exhaustion +(`src/crux/agent/manager.go:351`), and even whether a surviving CLI can be +adopted (`src/crux/agent/manager_tmux.go:441`) are all decided by matching substrings against rendered terminal output. A structured turn return value — the RFC's testability and subagent-sync @@ -547,11 +547,11 @@ Things this spike did not establish, and what would settle each. Settling it requires per-backend version-pinned testing against the actual binaries, since an undocumented on-disk format is not a contract. 2. **Should the manager-side nudge and restart counters join the persisted - envelope?** `tokenRestartAttempts` (`src/pkg/agent/manager.go:322`), - `transientNudgesThisKick` (`src/pkg/agent/manager.go:389`), and - `stallNudgeSent` (`src/pkg/agent/manager.go:372`) are loop-breakers that + envelope?** `tokenRestartAttempts` (`src/crux/agent/manager.go:322`), + `transientNudgesThisKick` (`src/crux/agent/manager.go:389`), and + `stallNudgeSent` (`src/crux/agent/manager.go:372`) are loop-breakers that reset to zero on restart, while the watchdog's equivalent ladder is - deliberately persisted (`src/pkg/watchdog/reconciler.go:401-402`). Whether the + deliberately persisted (`src/crux/watchdog/reconciler.go:401-402`). Whether the difference is intentional (these counters are per-launch by design, and a fresh launch legitimately deserves a fresh budget) or an oversight is not determinable from the code alone and was not raised as a defect by this @@ -559,20 +559,20 @@ Things this spike did not establish, and what would settle each. 3. **What is the actual observed restart rate, and how much work is lost per restart?** ~~The RFC's motivation is "frequent spoke rolls", but this spike found no metric that quantifies work discarded by a restart.~~ **An - instrument now exists** (`src/pkg/agent/turn_loss.go`); the question is + instrument now exists** (`src/crux/agent/turn_loss.go`); the question is awaiting fleet data rather than awaiting a way to collect it. See [Measuring the motivation](#measuring-the-motivation-open-question-3) below. 4. **Is `buildBootstrapPrompt`'s path-building dead code, or a staging post?** *Since answered: dead code.* The function used to assemble a candidate policy-file list and then discard it by returning `""`; the path construction has since been deleted, and the function is now an explained - `return ""` stub (`src/pkg/agent/manager_env.go:19-27`, `return ""` at - `src/pkg/agent/manager_env.go:27`), with the removal recorded in its comment. + `return ""` stub (`src/crux/agent/manager_env.go:19-27`, `return ""` at + `src/crux/agent/manager_env.go:27`), with the removal recorded in its comment. 5. **Are there non-tmux agent execution paths with different properties?** - `src/pkg/agent/sandbox_executor.go` runs a different shape of execution - (`src/pkg/agent/sandbox_executor.go:405` composes its own command line) and + `src/crux/agent/sandbox_executor.go` runs a different shape of execution + (`src/crux/agent/sandbox_executor.go:405` composes its own command line) and inference backends are handled separately - (`IsInferenceBackend`, `src/pkg/agent/manager_routing.go:23`). This spike scoped + (`IsInferenceBackend`, `src/crux/agent/manager_routing.go:23`). This spike scoped itself to the tmux CLI path, which is the fleet's normal mode; the sandbox and inference paths were not mapped and may not share these constraints. @@ -585,7 +585,7 @@ The RFC argues from "frequent spoke rolls discard agent work". That premise was unsized: `RestartCount` (`src/pkg/snapshot/state.go`) counts restarts and says nothing about what any of them cost. Step 4's feasibility call depends on the difference, and the measurement is cheap next to a prototype — so it is now -instrumented, in `src/pkg/agent/turn_loss.go`. +instrumented, in `src/crux/agent/turn_loss.go`. **What is counted.** A turn is interrupted when hive tears down an agent's CLI and tmux session while a delivered kick's output has not yet been rotated diff --git a/src/docs/design/copilot-cost-capture.md b/src/docs/design/copilot-cost-capture.md index 165f8bf0ea..c53639191f 100644 --- a/src/docs/design/copilot-cost-capture.md +++ b/src/docs/design/copilot-cost-capture.md @@ -39,25 +39,25 @@ already shipped. The path, end to end: - The proxy MITMs the Copilot completion hosts. `IsCopilotAPIHost` - (`src/pkg/proxy/rules.go:112`) matches `githubcopilot.com` and any + (`src/crux/proxy/rules.go:112`) matches `githubcopilot.com` and any `.githubcopilot.com` subdomain, covering the enterprise per-account hosts. Two call seams dispatch into the sniff path — the explicit-CONNECT seam - (`src/pkg/proxy/github_proxy.go:877`) and the iptables-transparent seam + (`src/crux/proxy/github_proxy.go:877`) and the iptables-transparent seam (`:461`), both guarded identically by `p.tokenSink != nil && agentName != ""`. -- `proxyCopilotHTTP` (`src/pkg/proxy/github_proxy.go:2559`) runs a keep-alive +- `proxyCopilotHTTP` (`src/crux/proxy/github_proxy.go:2559`) runs a keep-alive HTTP request/response loop over the terminated TLS connection. It identifies completion requests via `isCopilotCompletionsPath` - (`src/pkg/proxy/copilot_usage.go:40`, suffix `/chat/completions`), buffers the + (`src/crux/proxy/copilot_usage.go:40`, suffix `/chat/completions`), buffers the request body, and reads the model id out of it (`:2260`). - For streaming requests it rewrites the body to carry `stream_options.include_usage=true` (`ensureStreamUsageRequested`, - `src/pkg/proxy/copilot_usage.go:63`; call site `github_proxy.go:2589`). + `src/crux/proxy/copilot_usage.go:63`; call site `github_proxy.go:2589`). Without that hint the OpenAI-compatible endpoint omits the terminal usage chunk and streamed completions are uncountable. Note this is the one place the proxy alters an agent's request payload. - `forwardCopilotResponseWithUsage` (`github_proxy.go:2642`) buffers the response, extracts usage via `extractCopilotUsage` - (`src/pkg/proxy/copilot_usage.go:136` — SSE terminal-chunk path at `:104`, + (`src/crux/proxy/copilot_usage.go:136` — SSE terminal-chunk path at `:104`, non-streaming JSON path via `extractOpenAIUsage`), resolves the concrete model when the request pinned the `auto` sentinel (`extractCopilotResponseModel`, `:147`), and records it at `github_proxy.go:2662`. @@ -123,14 +123,14 @@ recoverable from adjacent traffic. This is the finding that decides the page. ### 2a. Not in the Copilot request The structs the proxy parses out of a completion request are `model`, `stream`, -and `stream_options` (`copilotUsageRequest`, `src/pkg/proxy/copilot_usage.go:32`). +and `stream_options` (`copilotUsageRequest`, `src/crux/proxy/copilot_usage.go:32`). No repo, workspace, or path field is read, and no request header is inspected for one — `proxyCopilotHTTP` forwards headers verbatim and branches solely on `req.URL.Path` (`github_proxy.go:2245`). A grep for repo-bearing Copilot editor headers (`Editor-Version`, `Copilot-Integration-Id`, `X-GitHub-*`) across `src/pkg/proxy/` returns nothing, and the sniff-test fixtures send only `Host`, `Content-Type`, `Content-Length`, and `Connection` -(`src/pkg/proxy/copilot_sniff_test.go:155`, `:253`, `:450`, `:604`, `:682`, +(`src/crux/proxy/copilot_sniff_test.go:155`, `:253`, `:450`, `:604`, `:682`, `:750`). Whether the live Copilot CLI sends a repo hint in a header or in the `messages` @@ -140,7 +140,7 @@ repository** — the proxy neither parses nor logs it. See ### 2b. Not in per-agent proxy state -`GitHubProxy` (`src/pkg/proxy/github_proxy.go:152-207`) holds exactly one +`GitHubProxy` (`src/crux/proxy/github_proxy.go:152-207`) holds exactly one repo-shaped field: `allowedRepos map[string]bool` (`:158`), built once at construction from the hive's configured repo list (`NewGitHubProxy`, `:247`, loop at `:270-272`). It is a hive-wide allow-set, identical for every agent — @@ -150,14 +150,14 @@ per-agent "current repo" or "last repo seen" map anywhere in the struct. `agentName` itself comes from a UID lookup per connection: `LookupUIDByLocalPort` then `p.uidMap.LookupByUID(uid)` (`github_proxy.go:461` region; `UIDMap.LookupByUID` at -`src/pkg/agent/uidmap.go:80`). `UIDMap` maps UID ↔ agent name and nothing else +`src/crux/agent/uidmap.go:80`). `UIDMap` maps UID ↔ agent name and nothing else (`uidmap.go:24`, methods at `:43`, `:60`, `:80`, `:113`, `:129`). It carries no repo dimension, and the lookup is per-connection with no cross-connection state. ### 2c. Not recoverable from adjacent traffic The proxy *can* extract a repo from a URL path — `ExtractRepo` -(`src/pkg/proxy/rules.go:290`) matches `^/repos/([^/]+/[^/]+)` (`:279`) and +(`src/crux/proxy/rules.go:290`) matches `^/repos/([^/]+/[^/]+)` (`:279`) and `^/([^/]+/[^/]+)\.git/` (`:281`). Two facts kill this as a phase-4 source: 1. **It has exactly one non-test caller**, `RepoFilterAllowed` @@ -179,7 +179,7 @@ The proxy *can* extract a repo from a URL path — `ExtractRepo` `github_proxy.go:882-885` repeats it at the CONNECT seam. Such a connection falls to `tunnelDirect` (`github_proxy.go:887`) or the raw relay (`:484`), so the proxy sees only SNI `github.com` (`extractSNI`, `:660`) and - never a `/owner/repo.git/info/refs` path. The `gitPathPrefix` branch of + never a `/owner/repo.git/info/refs` path. The `gitPathRepo` branch of `ExtractRepo` therefore cannot fire on real clone traffic, and the proxy cannot learn the repo the agent cloned. diff --git a/src/docs/design/crux-tool-policy.md b/src/docs/design/crux-tool-policy.md new file mode 100644 index 0000000000..e2733ae340 --- /dev/null +++ b/src/docs/design/crux-tool-policy.md @@ -0,0 +1,432 @@ +# Crux: a generic tool policy for the MITM proxy (#14228) + +Status: **Proposed (design only).** Spec for #14228, a child of the Crux +stage 2/3 tracker #14138. No code changes with this record. Implementation +is cut into the child issues listed under +[Exit criteria](#exit-criteria-and-implementation-children). The parent +record is [crux.md](crux.md). Its +[Credentials and hive-provided capabilities](crux.md#credentials-and-hive-provided-capabilities) +section sets the engine/hive split this record relies on. + +Scope: the request-level policy that `crux/proxy` applies to agent traffic +it intercepts. That means forge APIs (GitHub REST and GraphQL today, and +GitLab, Gitea and Forgejo next: #14211, #14116), Linear's GraphQL API, git +smart-HTTP, and the classification of inference endpoints. Out of scope: the +tool-approval desk (`pkg/toolapprove`, [tool-approval-desk.md](tool-approval-desk.md)), +which decides approval-shaped requests before an agent acts, and +`pkg/pushbroker`, which pushes for the agent with a hive-minted token. Both +sit next to this policy. Neither one is part of it. + +Line references are to `src/crux/proxy/` on v5 at `6350270f1` and are given +as orientation only. The identifiers are the stable reference. + +## The problem: three classifiers and an implicit order + +The proxy's policy is GitHub-shaped. It is split across three classifiers +with three different postures: + +| Mechanism | File | Keyed on | Posture on unknown | +|---|---|---|---| +| `ProxyRule` table (`rules`) + `denyRules` | `rules.go` | `(method, path regexp)` on any inspected host, `MinMode` + optional `Capability` OR-grant | **deny** (`AllowedByModeCaps` falls through to `false`) | +| `GraphQLAllowedCaps` | `rules.go` | substring match of mutation names (`graphQLMergeMutationRe`, `graphQLPRWriteMutationRe`) | **permit at `ModeIssuesOnly`** (the `default:` branch, line ~691) | +| `LinearGraphQLAllowed` / `linearMutations` | `linear_rules.go` | top-level GraphQL operation name on the single `POST /graphql` | **deny** | + +Several gates compose with these classifiers but are not part of them: +write-surface enforcement (`WriteSurfaceEnforceRefusal`, +`WriteSurfaceGraphQLRefusal` in `write_surface_enforce.go`), repo pause / +agent repo scope / repo auto-merge-off (`RepoPauseRefusal`, +`AgentRepoScopeRefusal`, `RepoAutoMergeRefusal`, `GraphQLAutoMergeRefusal`), +per-repo autonomy levels (`autonomyModeForRepo`, `autonomyGraphQLMode` in +`autonomy_levels.go`), the allowed-repo filter (`RepoFilterAllowed`), the +issue-shape gate (`enforceIssueShapeFor`), canary egress +(`inspectCanaryEgress`) and the per-agent write budget +(`githubBudgetRefusal` in `agent_github_budget.go`). + +The only place the order of these gates is written down is the body of +`proxyHTTPHost` in `github_proxy.go`. At `6350270f1` the order is: + +| Step | Where | Deny code (`denial_message.go`) | +|---|---|---| +| internal caller (`internalCallerName`) skips the agent gates | throughout | n/a | +| repo paused | `p.repoPauseRefusal`, line ~1385 | `repo_paused` | +| agent repo scope | `p.agentRepoScopeRefusal`, line ~1389 | `agent_repo_scope` | +| repo auto-merge off (REST) | `p.autoMergeRefusal`, line ~1393 | `repo_auto_merge_disabled` | +| write-surface (REST, git) | `WriteSurfaceEnforceRefusal`, line ~1399 | `write_surface_enforce` | +| Linear classifier | `p.enforceLinear`, line ~1426 | `linear_policy` | +| GraphQL: auto-merge-off, write-surface, classifier | lines ~1439–1464 | `repo_auto_merge_disabled`, `write_surface_enforce`, `graphql_mutation` / `graphql_denied` | +| REST mode (+capability, per-repo autonomy). Hard deny is checked *inside* it and only re-surfaced as the reason | `AllowedByModeCaps` / `DeniedMessage`, lines ~1472–1489 | `mode_denied`, `hard_deny` | +| allowed-repo filter | `RepoFilterAllowed`, line ~1490 | `repo_not_in_config` | +| issue shape | `p.enforceIssueShapeFor`, line ~1503 | `issue_shape` | +| canary egress | `p.inspectCanaryEgress`, line ~1517 | `canary_egress` | +| write counters, then the one-line 403 (`ProxyDenialBody`, `X-Hive-Proxy-Reason`) | lines ~1530–1590 | | +| write budget, as a 429 | `p.githubBudgetRefusal`, line ~1598 | (429, not 403) | +| credential injection, then forward | `rewriteGitHubAuth` / `injectLinearCredential`, line ~1622 | | + +Three facts follow, and the decisions below rest on them: + +1. **A forge is keyed on its wire shape, so a second forge means + duplicating policy.** #14211 adds GitLab and Gitea rows to the shared, + host-blind `rules` slice and copies the hard-deny prose per forge. Its + contract test (`ForgeRulesContract`) already invents a shared operation + name (`CreateChangeRequestHardDeny`, `CommentIssue`, …) to keep the + copies in step. That operation vocabulary is real. It is just not in the + code. +2. **The same operation gets different verdicts depending on its wire + shape.** REST `PUT …/pulls/{n}/merge` and `POST …/pulls` are hard-denied + for every mode (`denyRules`), and the comment above `denyRules` says the + GraphQL merge path matches. It does not. `GraphQLAllowedCaps` returns + `mode >= ModeIssuesPRsMerge` for `mergePullRequest` (line ~677), and + `createPullRequest` is gated at `ModeIssuesAndPRs`. Neither is a relay. +3. **Hard deny is not a first stage.** It runs inside the REST mode check, + after pause, scope, auto-merge and write-surface. A paused repo returns + `repo_paused` for a PR-create, not `hard_deny`. #12127 made reason codes + machine-readable, so this order is now observable behaviour. + +## Decisions + +Hive filed this issue, so the engineering questions are decided here. Each +answer below names the alternative it rejects. The rule used throughout is +to pick the simplest option that is additive and backward-compatible. + +### 1. Policy model: shared operations, per-forge adapters + +**Decision:** the unit of policy is an **operation** from one shared +vocabulary (`pr.create`, `pr.merge`, `issue.comment`, `label.add`, …). Each +forge has an **adapter** that maps a request's wire shape (method + path, +GraphQL field names, git service) to zero or more operations plus the +target repo. One shared **evaluator** applies one policy table to those +operations. *Rationale:* one ACMM ladder holds across every forge, so a +relay or tier change is made once and every forge picks it up. #14211's +`ForgeRulesContract` already shows the vocabulary is needed. *Rejected:* +a `(method, path)` table per forge behind a shared evaluator. Each table is +simple to audit for its own host, but the ladder gets copied N times, and +`linear_rules.go`'s header explains why `(method, path)` cannot express a +single-endpoint GraphQL API at all. + +The vocabulary starts as exactly the set of distinct things today's GitHub +and Linear tables gate. Defaults equal today's verdicts: + +| Operation | GitHub REST / git (today's rule) | GitHub GraphQL (today) | Default policy | +|---|---|---|---| +| `read` | `GET`/`HEAD`/`OPTIONS` `.*` | non-mutation document | `ModeAdvisory` | +| `auth.login` | `POST /login/device/code`, `/login/oauth/access_token` | — | `ModeAdvisory`, write-surface exempt | +| `git.fetch` | `POST …\.git/git-upload-pack` | — | `ModeAdvisory` | +| `git.push` | `POST …\.git/git-receive-pack`; `GET …/info/refs?service=git-receive-pack` (`gitServiceIsReceivePack`, write-surface only) | — | `ModeIssuesAndPRs` | +| `ref.write` | `POST …/git/refs`, `POST …/git/commits`, `DELETE …/git/refs/…` | `createCommitOnBranch`, `createRef`, `updateRef`, `deleteRef` | `ModeIssuesAndPRs` | +| `branch.protect` | — | `createBranchProtectionRule`, `updateBranchProtectionRule` | `ModeIssuesAndPRs` | +| `issue.create` | `POST …/issues` | (default bucket) | `ModeIssuesOnly` | +| `issue.edit` | `PATCH …/issues/{n}` | (default bucket) | `ModeIssuesOnly` | +| `issue.comment` | `POST …/issues/{n}/comments` | conversational fields (`converseMutations`) | `ModeIssuesOnly` or `converse` | +| `label.add` | `POST …/issues/{n}/labels` | (default bucket) | `ModeIssuesOnly` | +| `pr.create` | `POST …/pulls` | `createPullRequest` | **`relay(hive-open-pr)`** | +| `pr.edit` | `PATCH …/pulls/{n}` | `markPullRequestReadyForReview`, `convertPullRequestToDraft` | `ModeIssuesAndPRs` | +| `pr.review` | `POST …/pulls/{n}/reviews` | `addPullRequestReview`, `submitPullRequestReview` | `ModeIssuesAndPRs` or `converse` | +| `pr.update_branch` | `PUT …/pulls/{n}/update-branch` | — | `ModeIssuesPRsMerge` | +| `pr.merge` | `PUT …/pulls/{n}/merge` | `mergePullRequest`, `mergeBranch`, `enablePullRequestAutoMerge` | **`relay(hive-merge)`** | +| `agent.session` | — (Linear: `agentActivityCreate`, `agentSessionUpdate`) | — | `ModeAdvisory` or `converse` | +| `attachment.link` | — (Linear: `attachment*`) | — | `ModeIssuesAndPRs` | +| `inference.call` | inference hosts (see question 6) | — | allow (mode-independent) | +| `unknown` | anything an adapter cannot classify | unrecognised mutation | **deny** (see question 2) | + +A request whose GraphQL document selects several top-level fields maps to +several operations. The request is allowed only if **every** operation is +allowed, and the verdict that gets reported is the strictest one (relay +beats deny-by-mode, which beats allow). This is the same "the whole +document, not one substring" rule that `isConversationOnlyMutation` +already applies for `converse`. + +The adapters stay Go code. GitHub's GraphQL and Linear's operation parsing +(`topLevelMutationFields`, `linearOperationFields`) are parsers with body +limits (`graphQLBodyLimit`, `linearBodyLimit`, `linearMaxOperations`), not +regex rows. + +### 2. Default posture for an unrecognised operation: deny + +**Decision:** one rule for every adapter. A request the adapter cannot map +to a known operation becomes `unknown`, and `unknown` is **denied** with +reason code `mode_denied` (REST) or `graphql_mutation` / `linear_policy` +(GraphQL hosts), so today's codes do not change. GitHub GraphQL's +permissive `default:` is **a bug to close, not a documented exception**. +It is closed with a ratchet rather than a flag day. During migration, the +GitHub GraphQL adapter maps each unrecognised mutation field to the named +legacy operation `legacy.graphql_mutation`, whose default policy is +`ModeIssuesOnly`. That is exactly today's verdict, so the parity test +holds. A follow-up child then (a) maps the mutation names that agents +actually send, read from the `graphql_mutation` violation and write-counter +logs, onto real operations, and (b) changes `legacy.graphql_mutation` to +deny. *Rationale:* fail closed is already the posture for REST and Linear, +and "REST is the primary gate" does not hold for GraphQL-only operations. +*Rejected:* keeping the permissive default as a documented GitHub +exception. It is the one place where an unknown write gets through, and +every new GraphQL forge would inherit the question. + +The same ratchet closes the divergence noted in fact 2. Under the parity +phase, GraphQL `pr.merge` and `pr.create` keep today's mode-gated verdicts +through two named entries in the evaluator's `legacyExceptions` list. The +child that empties that list makes them `relay`, which matches what the +`denyRules` comment already claims. The list can only shrink, and a test +pins its contents. + +### 3. Hard deny becomes a `relay()` verdict + +**Decision:** the evaluator returns one of three verdicts: `allow`, +`deny(code, reason)`, or `relay(command)`. `relay` is a deny that names the +sanctioned path. The 403 body is rendered from one shared template per +relay command (what was refused, the relay invocation, what not to use). +The reason code is still `hard_deny`. A forge adapter can add a short list +of forge-specific "do not use" hints (for GitHub: `gh pr create`, the +GitHub MCP `create_pull_request`). It does not write its own prose. +*Rationale:* this removes the per-forge message copies seen in #14211 +("use the request-file path…") and keeps the two GitHub messages in +`denyRules` byte-identical. The compiled-in relay templates reproduce them, +and the parity test asserts that. *Rejected:* a plain `deny` with a +per-rule message string (today's `denyRule.Msg`). That copies the relay +contract into every adapter, and the copies drift. + +### 4. Where the tables are declared: engine vocabulary and adapters in Go, hive-shaped policy as config with compiled-in defaults + +**Decision:** the policy has three parts, and each lives in a different +place: + +| Part | Shape | Lives in | Why | +|---|---|---|---| +| Operation vocabulary (`Operation` constants) | engine | `crux/proxy`, Go | a stable contract between adapters and policy | +| Adapters (wire shape → operations) | engine | `crux/proxy`, Go, one file per forge | parsers with body limits, unit-tested, reviewed as code | +| Policy table (operation → `MinMode`, `Capability`, `relay` command and its usage line) | **hive-shaped** | supplied by the host. Compiled-in default = today's tables | relay names (`hive-open-pr`, `hive-merge`) and ladder levels are hive policy | + +The host installs a table in-process with `(*GitHubProxy).SetToolPolicy(PolicyTable)`. +When stage 3 lands, the same table arrives as config through the `crux serve` +API, as the stage-2 capability table in crux.md already says for +`SetRepoPausedFunc` and the other setters. If nothing is installed, the +compiled-in `DefaultPolicyTable` applies. It reproduces `rules`, `denyRules`, +`GraphQLAllowedCaps` and `linearMutations` exactly, so hives that configure +nothing see no change. Any table that is installed is validated against +the vocabulary: an unknown operation name is a load error, and a missing +operation falls back to the default row, never to allow. *Rationale:* this +follows the stage-2 rule (hive policy answers on the hive side, engine +mechanism in the engine) without forcing every host to ship a table. +*Rejected:* (a) a data file of regex routes shipped with the runtime. It +moves classification out of reviewed, tested code and still cannot express +GraphQL. (b) Keeping everything compiled in. That leaves `hive-open-pr` and +`hive-merge` hard-coded inside an engine that other consumers are meant to +use. + +### 5. Composition order: one documented pipeline + +**Decision:** every adapter runs through the single pipeline below. Its +**gate order matches the order `proxyHTTPHost` applies today**, so reason +codes do not change. Classification moves to the front, and hard deny +becomes the `relay` verdict of the policy stage, which is where it already +takes effect. + +```text + agent request (CONNECT / transparent TLS / plain) + │ + ┌────────────────────────────▼────────────────────────────┐ + 0 │ host → adapter RegisterForgeHost table (Q7) │── no adapter ─▶ opaque tunnel (as today) + └────────────────────────────┬────────────────────────────┘ + ┌────────────────────────────▼────────────────────────────┐ + 1 │ classify adapter: wire → []Operation, repo │── unparseable / over limit ─▶ unknown + └────────────────────────────┬────────────────────────────┘ + │ internal caller (hive control plane)? ── yes ─▶ skip 2–5, go to 6 + ┌────────────────────────────▼────────────────────────────┐ + 2 │ run-state gates repo paused → agent repo scope → │── deny repo_paused / agent_repo_scope / + │ repo auto-merge off │ repo_auto_merge_disabled + └────────────────────────────┬────────────────────────────┘ + ┌────────────────────────────▼────────────────────────────┐ + 3 │ write-surface lane under write_surface.enforce, │── deny write_surface_enforce (+ audit) + │ op is a write (not auth.login) │ + └────────────────────────────┬────────────────────────────┘ + ┌────────────────────────────▼────────────────────────────┐ + 4 │ policy per op: relay? → mode(+per-repo │── relay(cmd) → hard_deny + │ autonomy) OR capability; unknown │── deny mode_denied / graphql_* / linear_policy + │ → deny; strictest op wins │ + └────────────────────────────┬────────────────────────────┘ + ┌────────────────────────────▼────────────────────────────┐ + 5 │ allowed-repo filter RepoFilterAllowed │── deny repo_not_in_config + └────────────────────────────┬────────────────────────────┘ + ┌────────────────────────────▼────────────────────────────┐ + 6 │ body gates issue shape (agents only) → │── deny issue_shape / canary_egress + │ canary egress scan │ + └────────────────────────────┬────────────────────────────┘ + │ write counters; first deny above → one-line 403 + X-Hive-Proxy-Reason + ┌────────────────────────────▼────────────────────────────┐ + 7 │ write budget per-agent window (agents only) │── 429 + └────────────────────────────┬────────────────────────────┘ + ┌────────────────────────────▼────────────────────────────┐ + 8 │ credential injection adapter-owned (GitHub, Linear) │ + └────────────────────────────┬────────────────────────────┘ + ▼ + forward +``` + +Each stage gets the classified operations and the target repo, not the raw +`(method, path)`. So the run-state gates and the write-surface gate are +written once and work for every forge, including GraphQL hosts. Today they +are duplicated across the REST and GraphQL branches. *Rationale:* moving +gates is observable through reason codes and the write-surface audit +entry. Documenting the current order and routing it through one function +gives a single pipeline without any behaviour change. *Rejected:* the order +listed in the issue (hard deny → write-surface → repo scope/paused → mode). +It would make a PR-create on a paused repo report `hard_deny` instead of +`repo_paused`, and would replace the write-surface audit entry for an +enforced lane's PR-create with a relay denial. Both are behaviour changes +with no safety gain, because every path still ends in deny. + +### 6. git smart-HTTP and inference are in scope as small adapters + +**Decision:** "tool policy" covers everything the proxy inspects, not only +forge JSON APIs. git smart-HTTP gets one forge-neutral sub-adapter +(`gitsmart`) that every forge adapter delegates to for `…\.git/…` and +`info/refs` paths. It produces `git.fetch` or `git.push`, using +`gitServiceIsReceivePack` and the `gitUploadPackPath` / `isGitReceivePack` +matchers. Inference hosts (`IsCopilotAPIHost`, `SetInferenceRoute` routes) +get an `inference` adapter that produces `inference.call`. Its default +policy is allow at every mode, so the only new behaviour is that inference +requests use the same verdict type and denial format. Inference routing, +translation, entitlement and `inference_budget.go` stay where they are and +are not part of the pipeline stages. *Rationale:* git push is already a +write in the write-surface gate and the mode table. Having a separate model +for it is how GraphQL ended up with its own posture. *Rejected:* limiting +the policy to forge APIs. That leaves `git-receive-pack` in a third +classifier, and every new forge would have to re-derive push handling. + +### 7. Host identification: host → adapter mapping from config + +**Decision:** `githubHosts` is generalised to a mutex-guarded +`forgeHosts map[string]AdapterID`, populated by +`RegisterForgeHost(host string, adapter AdapterID)`. `RegisterGitHubHost(h)` +remains as a wrapper for `RegisterForgeHost(h, AdapterGitHub)`, and +`IsGitHubHost` / `IsLinearHost` / `NeedsInspection` become lookups in the +same map. Defaults: `api.github.com`, `github.com` → `github`, and +`api.linear.app` → `linear`. Each adapter also declares which of its hosts +need MITM (today `NeedsMITM`, `hostNeedsMITM`, `hostNeedsMITMFor`), so +adding a host does not quietly leave it in an opaque tunnel (the trap that +`NeedsInspection`'s comment describes). The host gets these mappings from +the same config that supplies forge URLs. Under stage 3 they arrive with +the policy table. *Rationale:* this is deterministic, auditable and +additive, and it is how GHE hosts are added today. *Rejected:* (a) probing +the host (`/api/v4/version`, `/api/v1/version`). It adds latency and a +time-of-check/time-of-use gap, and an attacker-reachable endpoint would +choose the policy. (b) Using the agent's declared `forge`. That is +agent-influenced input selecting the agent's own policy. + +### 8. Testing contract every adapter ships + +**Decision:** an adapter is merged only with all of the following, each a +table test parameterised over the registered adapters: + +1. **Completeness.** Every operation in the vocabulary has at least one + wire example in the adapter's fixture table, or appears in the adapter's + explicit `Unsupported` list with a reason. A new operation with no row + for an existing adapter fails the build. This generalises #14211's + `ForgeRulesContract`. +2. **Classification round-trip.** Each fixture classifies to the operation + it claims. Near-miss fixtures (for example GitLab + `POST /projects/:id/labels`, which *creates a project label* and is not + `label.add`) classify to the right operation or to `unknown`. +3. **Relay coverage.** For every adapter and every mode, each operation the + default table marks `relay` (`pr.create`, `pr.merge`) is refused with + `hard_deny` and the shared relay message. +4. **Fail closed.** An unparseable body, an over-limit body, or an + unmatched route is denied at every mode. +5. **Denial message.** The `denial_message_test.go` assertions (one line, + ends `reason=`, code in `X-Hive-Proxy-Reason`, what/why/instead + preserved) run for one refused request per adapter. +6. **Parity (GitHub and Linear, migration only).** For a fixed corpus of + `(mode, caps, method, path, body)` built from every row of `rules`, + `denyRules`, `GraphQLAllowedCaps`'s buckets and `linearMutations`, plus + the existing `rules_test.go` / `linear_rules_test.go` cases, the + evaluator returns the same allow/deny decision **and the same reason + code** as the legacy functions. The test is deleted together with the + legacy path. +7. **Ratchet.** `legacyExceptions` and the + `legacy.graphql_mutation` mapping are pinned exactly, and the only + allowed change is to shrink them. + +*Rationale:* (1)–(5) are the properties #14211 and the Linear gate each +tested ad hoc. (6) is what makes the migration a pure refactor. *Rejected:* +tests for each adapter only. Those cannot catch an operation missing from +one forge, which is the failure #14211's contract was written to prevent. + +### 9. Migration: evaluator first behind parity, then new forges, then delete legacy + +**Decision:** + +1. **Evaluator + GitHub adapter, no behaviour change.** Add the + vocabulary, `Verdict`, `PolicyTable` with `DefaultPolicyTable`, the + evaluator, and the `github` and `gitsmart` adapters. `proxyHTTPHost` + calls the evaluator. `AllowedByMode`, `AllowedByModeCaps`, + `DeniedMessage`, `GraphQLAllowed` and `GraphQLAllowedCaps` stay as thin + wrappers over it, because existing tests and `pkg/apiproxy` call them. + The parity test gates this step. +2. **Linear onto the evaluator.** Add a `linear` adapter. Its posture is + already deny, so parity is direct. +3. **New forges only on the generic model.** Add `RegisterForgeHost` + (question 7) and the GitLab and Gitea/Forgejo adapters. No non-GitHub + row is ever added to the host-blind `rules` slice. +4. **Close the GraphQL exceptions** (question 2): map observed mutations, + empty `legacyExceptions`, and make `legacy.graphql_mutation` deny. +5. **Hive-side policy table** through `SetToolPolicy` in-process, then + through the stage-3 API. +6. **Delete the legacy path:** `rules`, `denyRules`, the classifier bodies + in `GraphQLAllowedCaps` and `LinearGraphQLAllowed`, and the parity test. + +*Rationale:* GitHub moves first, but only as a verified refactor. Otherwise +the generic evaluator would govern only forges that have no production +traffic, and the GitHub table would stay the de facto spec. *Rejected:* +(a) using the generic evaluator only for new forges and moving GitHub "in +a later ratchet". That leaves two evaluators with different postures, which +is the problem this record is meant to fix. (b) A single PR that both moves +GitHub and changes verdicts. Parity cannot be reviewed if the expected +results change in the same diff. + +**#14211 is the first non-GitHub adapter, not a stop-gap.** Its GitLab and +Gitea routes, its hard-deny set (MR/PR create and merge for every mode) and +its `ForgeRulesContract` are the content of the `gitlab` and `gitea` +adapters (step 3) and the seed of the completeness test (question 8.1). It +should be rebased onto the evaluator rather than merged into the shared +`rules` slice, for three reasons. The slice is host-blind, so GitLab and +Gitea rows would also match requests to GitHub hosts. Widening the GitHub +rows with `(?:/api/v1)?` changes GitHub coverage with no GitHub reason. And +its hard-deny prose ("use the request-file path") becomes the shared +`relay(hive-open-pr)` / `relay(hive-merge)` messages. Its fixtures carry +over unchanged. + +## Exit criteria and implementation children + +This record answers questions 1–9 of #14228 and draws the pipeline once +(above). It also records the decision on #14211 (first adapter, rebased). +#14228 stays open until the following children are filed **as children of +#14228**, not of #14138: + +1. `crux/proxy`: operation vocabulary, `Verdict` (`allow` / `deny` / + `relay`), `PolicyTable` + `DefaultPolicyTable`, evaluator, and the + GitHub + gitsmart adapters behind a parity test (no behaviour change) +2. `crux/proxy`: Linear adapter on the evaluator (parity with + `linearMutations`) +3. `crux/proxy`: `RegisterForgeHost(host, adapter)` and adapter-declared + MITM hosts. `RegisterGitHubHost` becomes a wrapper +4. `crux/proxy`: GitLab and Gitea/Forgejo adapters (rebase of #14211 onto + the evaluator) with the adapter contract tests +5. `crux/proxy`: close GitHub GraphQL's permissive default and the + `legacyExceptions` (GraphQL `pr.merge` / `pr.create` → relay) by ratchet +6. `crux/proxy`: inference adapter (`inference.call`) on the shared verdict + and denial format +7. `crux/proxy` + `cmd/hive`: `SetToolPolicy` and the hive-side policy + table (relay commands, ladder levels). Stage-3 API transport follows + with #14138's sidecar work +8. `crux/proxy`: delete the legacy `rules` / `denyRules` / classifier + bodies and the parity test + +## References + +- Spec issue: #14228. Tracker: #14138. Parent record: [crux.md](crux.md) +- Forge pressure: #14211 (GitLab/Gitea rule table), #14116 (Forgejo smoke) +- Code: `src/crux/proxy/rules.go`, `linear_rules.go`, + `write_surface_enforce.go`, `agent_github_budget.go`, + `autonomy_levels.go`, `github_proxy.go` (`proxyHTTPHost`), + `denial_message.go`, `denial_message_test.go` +- Related decisions: #4492 (capabilities as OR-grants, Linear gate), #9772 + (write-surface enforcement), #12127 (one-line denials with reason codes), + [tool-approval-desk.md](tool-approval-desk.md) diff --git a/src/docs/design/crux.md b/src/docs/design/crux.md index f39ea0b5cf..a251568209 100644 --- a/src/docs/design/crux.md +++ b/src/docs/design/crux.md @@ -1,14 +1,27 @@ -# Crux: the agent runtime boundary (#13536) +# Crux: the agent runtime boundary (#13536, continued in #14138) -Status: **Stage 1 shipped (v5); Stage 2 blocked on `pkg/config`.** The +Status: **Stage 1 shipped (v5); Stage 2 step 4 shipped: the ring is the +nested Go module `github.com/hivecommons/hive/crux` at `src/crux/` +(#13950), joined to the root module by `src/go.work`; `crux.Process` is +its own type and `crux/types` is folded into the root (#13982, #13996); +the first cut of Stage 3's `crux serve` has landed (#13959).** The boundary is named, the inbound and outbound architecture ratchets and the tmux funnel are live, `crux.Runtime` exists, the dashboard is on it, Crux -owns its value types and the hub, spoke, dashboard and `cmd/hive` reference -them as `crux.`, CI path filters select Crux-only shards for Crux-only -PRs, and the `crux` CLI ships. Stage 2 (nested module) is measured but not -started: the ring's outbound closure is about 55 Hive packages, almost all -through `pkg/config`; see [Outbound edges](#outbound-edges). Stage 3 is -design only. +owns its value types and the hub, spoke, dashboard and `cmd/hive` +reference them as `crux.`, CI path filters select Crux-only shards +for Crux-only PRs, and the `crux` CLI ships. The inbound ratchet is +empty: `cmd/hive` holds `crux.Runtime` and constructs it through +`crux/runtime.New` (#14138 stage 1, see +[Stage 1: the Runtime seam](#stage-1-the-runtime-seam)), so no package in +the root module imports `crux/agent` any more. The ring's outbound closure, about 55 Hive +packages when this record began, is now zero, in test code too: the +outbound baseline is empty (the ring's tests use their own +`crux/internal/testutil` copy of `Eventually*` and `SkipUnlessRequired`); +see [Outbound edges](#outbound-edges). What the move did and how to work +in the two-module tree is under +[Step 4: the module cut](#step-4-the-module-cut); what is left is under +[After the cut](#after-the-cut) and tracked in #14138 (#13536 was the +Stage 1 RFC and is closed). ## What Crux is @@ -30,15 +43,21 @@ The name is "the point everything turns on". Hive turns on Crux. |---|---| | Runtime core | `pkg/agent` (manager, kick files, engine adapters: aider, bob, claude, codex, copilot, gemini, goose) | | Agent definition | `pkg/agentmode`, `pkg/agentparse`, `pkg/agentsmd`, `pkg/skillreg` | -| Engines and gateway | `pkg/claude`, `pkg/inference`, `pkg/openrouter` | -| Policy at the tool boundary | `pkg/proxy` (MITM), `pkg/toolapprove` | +| Engines and gateway | `pkg/claude`, `pkg/inference`, `pkg/openrouter`, `pkg/watsonx` | +| Policy at the tool boundary | `pkg/proxy` (MITM), `pkg/toolapprove`, `pkg/pushbroker` | | Isolation | `pkg/sandbox`, `pkg/kubejob` | | Observation | `pkg/watchdog`, `pkg/trajectory`, `pkg/logscrub` | -| Surface and CLI | `pkg/crux`, `pkg/crux/types`, `pkg/crux/cli`, `cmd/crux` | +| Stdlib-only leaves | `pkg/crux/config`, `pkg/crux/effects`, `pkg/crux/gitidentity`, `pkg/crux/jev`, `pkg/crux/jsonextract`, `pkg/crux/outputschema` | +| Surface and CLI | `pkg/crux`, `pkg/crux/cli` | The same list, as directories, is `CRUX_DIRS` in `.github/scripts/crux-scope.sh` (CI path filter) and `cruxModulePrefixes` in `src/internal/testutil/crux_outbound_ratchet_test.go`; the two must agree. +`cmd/crux`, the Hive-hosted `crux` binary's `main`, is a host in the root +module like `cmd/hive` (Stage 2, prerequisite 3): it holds the Hive-backed +`cli.DefinitionLoader` and reaches the runtime only through `pkg/crux`. The +module ships its own binary too, `crux/cmd/crux`, wired to `crux/defs` +(#14138; see [Definition loaders](#definition-loaders)). The RFC's claim that Crux's outbound coupling is "one `pkg/github` symbol and two `pkg/config` fields" was wrong. Measured on v5 at `437843c` (Oct 23, @@ -96,7 +115,7 @@ two-repo changes per day, which is why Stage 1 happens in this repo. `HasSession`); `pkg/tui` attaches through it and the ratchet's tmux baseline is zero everywhere else. Operations are added as call sites move behind the funnel, not ahead of them. -3. `crux.Runtime` (shipped): `src/pkg/crux/runtime.go` is the exported +3. `crux.Runtime` (shipped): `src/crux/runtime.go` is the exported runtime surface, grouped as `Lifecycle`, `Observer`, `Kicker`, `Definitions`, `Fleet` and `Credentials`; every method is annotated with the consumer that needs it and `*agent.Manager` satisfies it directly, so @@ -116,7 +135,8 @@ two-repo changes per day, which is why Stage 1 happens in this repo. Consumers never see the leaf. It is a separate package only because `pkg/crux` still aliases `agent.AgentProcess` as `crux.Process` and so imports `pkg/agent`, which cannot import it back; it folds into - `pkg/crux` once `Process` is its own type (Stage 2). The governor-side + `pkg/crux` once `Process` is its own type (Stage 2); it did in #13963, + and the definitions now live in the `crux` root package. The governor-side audit vocabulary (lease stages, commit trailers, artifact links) that `pkg/agent` used to re-export from `pkg/agentaudit` is no longer re-exported; consumers reference `agentaudit` directly. @@ -124,13 +144,15 @@ two-repo changes per day, which is why Stage 1 happens in this repo. Ratchet counts over that sequence: `pkg/dashboard` 68 → 51 (onto `crux.Runtime`) → 39 (provider status) → 19 (audit eviction); `pkg/hub` 18 → 1 (`AgentAvailableAtACMMLevel`); `pkg/hub/spoke` 15 → 5 - → 3; `cmd/hive` 37 → 33 → 22; `pkg/loginscan` 7 → 4. What remains in - every consumer is the manager and its constructors (`Manager`, - `NewManager`, `ProjectContext`, `NewRingBuffer`), agent-mode and - capability parsing, UID maps, Copilot-token helpers and pane + → 3; `cmd/hive` 37 → 33 → 22; `pkg/loginscan` 7 → 4. What remained in + every consumer at that point was the manager and its constructors + (`Manager`, `NewManager`, `ProjectContext`, `NewRingBuffer`), agent-mode + and capability parsing, UID maps, Copilot-token helpers and pane classification: implementation the test-only constructors and - `cmd/hive`'s wiring need, not observation. `pkg/crux` and everything - under it is inside the boundary for the ratchet's purposes. + `cmd/hive`'s wiring need, not observation. Stage 2 then took every row + but `cmd/hive` to zero and `cmd/hive` to 12; the sequence is under + [After the cut](#after-the-cut). `pkg/crux` and everything under it is + inside the boundary for the ratchet's purposes. 4. Agent configuration as its own struct and file (shipped): `AgentRuntimeConfig` and `AgentRuntimeGovernorConfig` in `src/pkg/config/agent_config.go`, embedded inline in `Config` and @@ -156,11 +178,14 @@ own tags (`crux/vX.Y.Z`), own coverage floor and CI lane; same repo, same PRs, same v5/v6 branches. The `crux` binary that this stage was going to ship already exists (item 6), so the stage is the module cut alone. -Not started. It was blocked until the outbound ratchet's `pkg/config` rows -were gone, which prerequisite 1 delivered; the remaining prerequisites are -the other outbound rows. A nested module whose packages import `pkg/config` needs the root -module to `replace` the nested one *and* the nested one to `replace` the -root: a module cycle. It builds, but nothing can consume +Prerequisites done, move not started. The move was blocked until the +outbound ratchet's `pkg/config` rows were gone, which prerequisite 1 +delivered, until `pkg/agent` and the sibling ring packages imported nothing +from Hive, which prerequisite 2 delivered, and until the proxy's edges were +cut and `cmd/crux` was scoped, which prerequisite 3 delivered. The reason +the order mattered: a nested module whose packages import `pkg/config` +needs the root module to `replace` the nested one *and* the nested one to +`replace` the root: a module cycle. It builds, but nothing can consume `github.com/hivecommons/hive/crux` without also pulling in all of Hive, so tags on it mean nothing, and every import path under `pkg/agent/...` would move for the ring's consumers and for the test shards, coverage floors and @@ -179,41 +204,774 @@ cut buys the churn without the benefit. Prerequisites, in order: boundary that loads `hive.yaml` for the CLI's `DefinitionLoader`. What moved, what was aliased and what was inverted is in [Configuration leaf](#configuration-leaf). -2. The remaining `pkg/agent` edges, measured on v5 after prerequisite 1 - (distinct `.` selectors; *test* counts only `_test.go`): - - | Edge | Non-test | Test-only | Files | Shape of the cut | - |---|---|---|---|---| - | `effects` | 1 (`Boundary`) | 0 | 3 | stdlib-only already; moves into the ring (`pkg/crux/effects` or `pkg/crux/types`), `pkg/effects` aliases | - | `gitidentity` | 2 (`AgentIdentity`, `AppLinkedAgentIdentity`) | 0 | 1 | stdlib-only already; moves in, `pkg/gitidentity` aliases | - | `outputschema` | 4 (`AgentReport`, `AgentReportFilePrefix`, `AgentReportFileSuffix`, `Validate`) | 0 | 1 | the report schema moves in; it imports `effects` (moves with it) and `convergence/proof`, which has to be inverted or left behind with the host-side validator | - | `jev` | 6 (`DefaultEndpoint`, `EndpointEnvVar`, `ModeEnvVar`, `SkillFile`, `SkillMarkdown`, `SkillRelDir`) | 5 | 2 + 1 test | constants and the skill text: a host-provided skill installer / environment seam, or the six symbols move to a stdlib leaf (`jev` itself imports `agentaudit` and `config`, so it does not move whole) | - | `fleetreport` | 8 (`Observe`, `Clear`, six `Class*` constants) | 7 | 5 + 3 test | a reporter interface on `Manager` the host injects, the same shape as `SpanStarter`; the classes become `pkg/crux/types` constants. `pkg/watchdog` has the same edge (7 symbols, incl. `RuntimeTracker`) and takes the same seam | - | `pushbroker` | 6 (`Broker`, `DefaultRemote`, `GitHubAppMinter`, `PushEnv`, `Result`, `TokenMinter`) | 2 | 3 + 3 test | moves in once its own `pkg/github` (`GitHubAppMinter`) and `logscrub` edges are inverted the way `AgentTokenCachePath` was; `effects` and `gitidentity` move first | - - 27 non-test symbols across six edges; `effects`, `gitidentity` and - `outputschema` are 7 of them and are pure moves. Order: effects + - gitidentity (one PR, aliases), outputschema, fleetreport seam (agent and - watchdog together), jev, pushbroker. -3. `pkg/proxy`'s edges to `github`, `ioscan`, `issueshape`, `tokens` and - `inferencehealth`, and `pkg/watchdog`'s to `rotation`. The proxy is both a - ring member and an inbound consumer (14 `agent.` references); - whether it stays in the ring is decided when those edges are cut, not - before. -4. Then the move: `git mv` of the ring under `src/crux/`, `src/crux/go.mod`, - `src/go.work`, one `replace` in the root module, and the shards, floors - and `crux-scope.sh` repointed in the same PR. - -**Stage 3: platform.** `crux serve` with an API (gRPC or HTTP), tenancy, auth, -quotas; Hive and hub become clients; the hub-to-runtime contract becomes -explicit and versioned; move to `hivecommons/crux`. Only when a second -consumer is real (candidates: Nous / Experiment Lab, Spektacular, Dibs, -external optimizer and formal-verification lanes, enterprise agent services). +2. **Done** (#13916, #13933, #13937). The remaining `pkg/agent` edges, plus + the sibling ring edges that fell cheaply: 27 non-test symbols across the + six `pkg/agent` edges measured after prerequisite 1, and 7 more across + `watchdog`, `toolapprove`, `trajectory` and `inference`. Each symbol was + moved, aliased or inverted by one rule: a value type or pure function + with no Hive dependency moves into a stdlib-only `pkg/crux/` leaf + and the Hive package aliases it; a service with Hive dependencies becomes + a narrow interface the host injects; a whole package that is really + runtime-side and imported only by the ring joins the ring. + + | Edge | Symbols | Decision | Where | + |---|---|---|---| + | `agent → effects` | `Boundary` (1) | moved whole | `pkg/crux/effects`; `pkg/effects` aliases every symbol (#13916) | + | `agent → gitidentity` | `AgentIdentity`, `AppLinkedAgentIdentity` (2) | moved whole | `pkg/crux/gitidentity`; the two host callers repointed, `pkg/gitidentity` removed (#13916) | + | `agent → jev` | `DefaultEndpoint`, `EndpointEnvVar`, `ModeEnvVar`, `SkillFile`, `SkillMarkdown`, `SkillRelDir` (6) | moved: environment contract + `jev-decide` skill | `pkg/crux/jev`; `pkg/jev` keeps the server and CLI and aliases the six (#13916) | + | `agent → outputschema` | `AgentReport`, `AgentReportFilePrefix`, `AgentReportFileSuffix`, `Validate` (4) | moved: schema + validator; `proof.Provenance` inverted to a type | `pkg/crux/outputschema`; the receipt handle is `pkg/crux/types.ProofProvenance`, `pkg/convergence/proof` and `pkg/outputschema` alias; report directory / path helpers stay host side (#13916) | + | `trajectory → jsonextract` | `Object` (1) | moved whole | `pkg/crux/jsonextract`; `pkg/jsonextract` aliases (#13916) | + | `agent → fleetreport`, `watchdog → fleetreport` | `Observe`, `Clear`, seven `Class*` (9 non-test across the two packages) | inverted | `pkg/crux/types.ConditionReporter` (`Observe`, `Clear`); the classes are `types.Class*` constants and `fleetreport` aliases them. `Manager.SetConditionReporter` (pane text redacted before it leaves the ring) and `watchdog.WithConditionReporter`; `cmd/hive` injects `fleetreport.DefaultReporter{}`, which delegates to the package tracker so `SetDefaultTrackerForTest` still swaps it (#13933) | + | `agent → pushbroker` | `Broker`, `DefaultRemote`, `GitHubAppMinter`, `PushEnv`, `Result`, `TokenMinter` (6) | joined the ring; its one Hive edge inverted | `pkg/pushbroker` is a ring package. Its `*github.AppAuth` field became `pushbroker.TokenScoper`; `cmd/hive`'s `appMinter` adapts and leaves a nil interface for a nil credential (#13933) | + | `inference → watsonx` | `DefaultMinter`, `ProjectIDHeader` (2) | joined the ring | `pkg/watsonx` is stdlib-only and imported only by `pkg/inference` (#13933) | + | `watchdog → rotation` | `Prober` (1) | inverted | `watchdog.UsageProber` (`Provider`, `ProbeError`); `cmd/hive` adapts each `rotation.Prober`, and the two Claude-credential probe tests moved with the adapter (#13937) | + | `toolapprove → ioscan` | `ScanInput`, `ScanOutput`, `SeverityHigh` (3) | inverted | `toolapprove.TextScanner`; `pkg/ioscan.ToolApproveScanner` implements it and `cmd/hive` and `pkg/turn` install it at init (`SetDefaultTextScanner`). A desk with no text scanner fails closed (`ErrNoTextScanner`). `ioscan` could not join the ring: `scheduler`, `github`, `config` and `dashboard` import it (#13937) | + + `pkg/agent`, `pkg/inference`, `pkg/toolapprove`, `pkg/trajectory` and + `pkg/watchdog` import nothing outside the ring in non-test code; the two + `internal/testutil` *(test)* rows are the ratchets themselves. The ring + prefix list grew by `pkg/pushbroker` and `pkg/watsonx` in + `crux_outbound_ratchet_test.go`, `crux-scope.sh` and the v2-tests + boundary comment together. +3. **Done** (#13941, #13942, and the `cmd/crux` re-scope that followed). + `pkg/proxy`'s edges and `cmd/crux`'s, measured on v5 after prerequisite 2 + as distinct non-test `.` selectors, cut by the same three + rules as prerequisite 2. The inventory in the tracker named + `github.AgentProxyToken` as the `proxy → github` edge; on inspection that + type was already behind a function field, and the real edge was the REST + rate-limit accounting (`CoreRateLimitSnapshot`, `RecordRESTRequest`, + `RecordInjectedRESTRequest`). + + | Edge | Symbols | Decision | Where | + |---|---|---|---| + | `proxy → inferencehealth` | `GatewayStatus`, `NewStore`, `Store` (3) | moved whole | `pkg/crux/inferencehealth`; `pkg/inferencehealth` aliases every symbol. Its one test that reads `fleetreport` stayed Hive side in the alias package (#13941) | + | `proxy → issueshape` | `BodyHasMisEscapedNewlines`, `UnsubstitutedTemplatePlaceholder` (2) | moved whole | `pkg/crux/issueshape`; `pkg/issueshape` aliases (#13941) | + | `proxy → ioscan` | `CanaryLeak`, `CanaryRegistry`, `NewCanaryRegistry` (3) | inverted; the leak record moved | `proxy.CanaryScanner` (`ScanEgress`), installed by `Proxy.SetCanaryScanner`, which `cmd/hive` already called with the `ioscan` registry. `CanaryLeak` is `pkg/crux/types.CanaryLeak` and `ioscan` aliases it. No scanner means no egress inspection; the host always installs one (#13942) | + | `proxy → tokens` | `BackendCopilot`, `InferenceSink`, `NewInferenceSink`, `SessionEntry` (4) | inverted | `proxy.UsageSink` (`Record`, `RecordWithCache`, `RecordCopilotUsage`), installed by `Proxy.SetTokenSink`; `*tokens.InferenceSink` satisfies it unchanged and `cmd/hive` injects it. A nil sink records nothing, matching the nil-pointer tolerance the concrete type had (#13942) | + | `proxy → github` | `CoreRateLimitSnapshot`, `RecordRESTRequest`, `RecordInjectedRESTRequest` (3) | inverted | `proxy.GitHubRESTAccounting` (`RecordRequest`, `RecordInjected`, `CoreSnapshot`), installed by `Proxy.SetGitHubRESTAccounting`; `cmd/hive/proxy_rest_accounting.go` adapts the process-global `pkg/github` store. Without it the proxy still forwards and reports no core-bucket snapshot (#13942) | + | `cmd/crux → config`, `cmd/crux → defsrc` | `AgentConfig`, `Config`, `Load`, `LoadAgentOverrides`, `PoliciesConfig`; `DefinitionKind`, `MergeAllowedFields`, `ParseDefinition` (8) | re-scoped: `cmd/crux` left the ring set | Every one of the eight is the `cli.DefinitionLoader` implementation: `hive.yaml`, Hive's per-agent defaults, the portable `AgentDefinition`. That is host knowledge, and `main` has no layer above it to be injected from, so `cmd/crux` is to the `crux` binary what `cmd/hive` is to `hive`: a host composition root in the root module, built against the ring. It left `cruxModulePrefixes` (which `crux-scope.sh` and the v2-tests boundary comment never listed it in) and the inbound ratchet keeps measuring it at zero `agent.` references, so it can reach the runtime only through `pkg/crux` | + + The proxy stayed in the ring. It is still an inbound consumer (14 + `agent.` references: `Manager`, agent-mode and capability + parsing, UID maps, the Linear credential), which is fine inside one + module and is the first question step 4 has to answer, below. +4. Then the move. Scoped under [Step 4: the module cut](#step-4-the-module-cut). + +### Step 4: the module cut + +Shipped in #13950 (after the pre-cut #13948), one PR on v5. What it did, +against the plan this section carried before it: + +1. **`pkg/proxy` moved with the ring** (the default). Its 14 + `agent.` references are now an intra-module import of + `crux/agent`; nothing was re-exported through `crux.` for it and no + consumer count changed. The inbound ratchet no longer walks `src/crux`, + so the `pkg/proxy: 14` row is gone from its baseline. `pkg/tokens`, + `pkg/ioscan` and `pkg/github` stay adapted from `cmd/hive` (the + prerequisite-3 seams); no thin root-module wiring package was needed. +2. **`src/crux/go.mod`**: module `github.com/hivecommons/hive/crux`, the + root's Go version, direct requires `cel-go`, `go-qrcode`, `otel`, + `otel/trace` and `gopkg.in/yaml.v3`; `go.sum` seeded from the root's + pins (so `cel-go` stays on the version the root uses rather than the + latest, whose rename of a package would have broken the build) and + tidied with `GOWORK=off`. +3. **The move**, with `git mv` so history follows the files: + + | before | after | + |---|---| + | `pkg/crux/*.go` | `crux/` — the module root package is `crux` | + | `pkg/crux/{cli,config,effects,gitidentity,inferencehealth,issueshape,jev,jsonextract,outputschema,types}` | `crux/` | + | `pkg/crux/testutil` | `crux/internal/testutil` | + | `pkg/{agent,agentmode,agentparse,agentsmd,claude,sandbox,kubejob,toolapprove,skillreg,inference,openrouter,watchdog,trajectory,logscrub,proxy,pushbroker,watsonx}` | `crux/` | + + 458 Go files changed import path and nothing else; the alias packages + in the root (`pkg/config`, `pkg/effects`, `pkg/jev`, `pkg/outputschema`, + `pkg/jsonextract`, `pkg/inferencehealth`, `pkg/issueshape`, `pkg/ioscan`, + `pkg/github`, `pkg/agentaudit`, `pkg/fleetreport`) kept every non-ring + caller's source unchanged. `cmd/crux` stayed in the root module as the + host composition root; the nested module ships no binary. +4. **`src/go.work`** uses `.` and `./crux`; the root `go.mod` requires the + module at the placeholder pseudo-version and carries the single + `replace github.com/hivecommons/hive/crux => ./crux`, so `GOWORK=off` + builds (the Dockerfiles, `go install`) resolve it too. `src/go.work.sum` + is committed. The nested module never requires or replaces the root. + The outbound ratchet walks `src/crux` as the module and asserts no + `github.com/hivecommons/hive/pkg/...`, `.../cmd/...` or + `.../internal/...` import; baseline empty. +5. **CI** addresses both modules from `src/` as `./... ./crux/...` (see + [Working in the two-module tree](#working-in-the-two-module-tree) for + why): the v2-tests agent slices run `./crux/agent`; the crux-only + `go list` is `./crux/... ./internal/... ./pkg/config ./pkg/rotation`; + the rest buckets, warm-build-cache, `existing-pkgs` and `no-test-pkgs` + lists add `./crux/...`; the coverage floors are keyed `crux/` + (`floor_key` gained `*/crux/*` and `*/crux` cases, so `crux/config` + can never alias `pkg/config`) with the same values the `pkg/` + keys had, and the gate prints an informational `CRUX_TOTAL` and fails a + full run whose shards produced no Crux profiles. `coverage-hourly.yml` + mirrors the floors; `v2-ci.yml` vets, `go-security-analysis.yml` + `govulncheck`s and `golangci-lint`s both modules; `setup-go`'s cache key + includes `src/crux/go.sum`. `crux-scope.sh` is `CRUX_DIRS=(src/crux)` + with `src/go.work*` and `src/crux/go.{mod,sum}` as fail-safe paths; + `affected-go-packages.sh` lists both modules and reads the root module + path with `GOWORK=off go list -m`. The Dockerfiles copy + `src/crux/go.mod`, `src/crux/go.sum` and `src/go.work` before + `go mod download` (the `replace` target has to exist). There is no + `scripts/check-go-ratchet-completeness.sh` in the tree; the ratchets + themselves assert the directories they walk exist. +6. **Tags** did not ship with the cut; see [After the cut](#after-the-cut). + +### Working in the two-module tree + +`src/` is a Go workspace. The rules that follow from that, every one of +which bit a CI script during #13950: + +- From `src/`, `./...` is the **root module only**. The Crux module is + `./crux/...` (and `./crux/`), addressable from `src/` because of + `go.work`. Build, vet and test both with one invocation: + `go build ./... ./crux/...`, `go vet ./... ./crux/...`, + `go test ./... ./crux/...`. `cd crux && go test ./...` is the module on + its own. +- `go list -m` in workspace mode prints both modules. A script that wants + the root module path says `GOWORK=off go list -m`. +- `GOWORK=off` from `src/` is the Docker view: the root builds through the + `replace`, the nested module's packages are still reachable as imports, + but `./crux/...` is not a pattern the root module can list. +- `go mod tidy` runs per module, with `GOWORK=off` in each (workspace mode + refuses to tidy). Keep the two `go.mod`s on the same Go version. +- Ring packages import each other and the standard library only; the + outbound ratchet (`go test ./internal/testutil/ -run Crux`) fails any + ring file that imports `github.com/hivecommons/hive/pkg/...` or + `.../internal/...`. Test helpers the ring needs live in + `crux/internal/testutil`; `internal/testutil` keeps the canonical copy. +- Root-module tests that read ring files by path (`internal/settingsparity` + walked `src/crux/config`; the docs citation and link checkers) name + `src/crux/...` now. `pkg/` paths in prose were left alone where + they describe history. +- Relative paths inside ring tests kept their depth (`pkg/agent` and + `crux/agent` are both two below `src/`), so `../../../bin` and + `../../../config/backends.conf` still resolve. + +### After the cut + +Still Stage 2, in this order: + +- **Tags** (plan item 6). `crux/vX.Y.Z` tags on the same commits as + Hive's, cut by the release workflow, so a consumer can pin the module. + One caveat the plan did not record: a module at `src/crux` of a + repository is only `go get`-able if its module path ends in the + directory path from the repo root, which for this tree would be + `github.com/hivecommons/hive/src/crux`, not + `github.com/hivecommons/hive/crux`. The root module has the same problem + (`github.com/hivecommons/hive` lives in `src/`) and has never been + fetched by path either, so this is not a regression; it is the thing to + settle when the first consumer outside this repository appears + (`src/` → repo root, or the Stage 3 move to `hivecommons/crux`, which + fixes it by construction). +- **`crux.Process` as its own type**, so the dashboard, hub and spoke stop + importing `crux/agent` for the test-only constructors (`NewManager`, + `ProjectContext`, `NewRingBuffer`) and the inbound baseline for + `agent.` can go to zero outside `cmd/hive`. The first slice of + #13962 moves consumer test construction and setup controls behind + `crux/cruxtest`, and spoke activity reads behind `crux.ActivityObserver`. + The second slice routes agent-mode and capability parsing (`AgentMode`, + `Mode*`, `ParseAgentMode`, `DefaultAgentMode`, + `AgentAvailableAtACMMLevel`) through `crux`. The third slice exposes the + Copilot-token helpers (`CopilotTokenSource*`, `CopilotUserTokenPath`, + `GitHubTokenLogin`) and pane classification (`SanitizePaneText`, + `RedactPaneTokens`, `PaneShowsBlockingPrompt`, `ClassifyAuthPane`) on + `crux`, so `pkg/loginscan` no longer imports `crux/agent`. The final + slice (#13962) made `crux.Process` its own struct: `crux/agent` copies + its `AgentProcess` into one when it snapshots an agent (`GetStatus`, + `GetStatusFast`, `AllStatuses` return `*crux.Process`), and + `ProjectContext`, `RingBuffer`, the Copilot-token constants, + `GitHubTokenLogin`, `DefaultAgentMode`, `AgentAvailableAtACMMLevel` and + the fatal-network pane patterns are defined in `crux` and aliased or + forwarded by `crux/agent`. The root package no longer imports + `crux/agent`; the `Runtime` assertion and `agent.AsRuntime` (formerly + `crux.FromManager`) live in `crux/agent`. The pane classifiers that share + tables with the pane poller (`SanitizePaneText`, `ClassifyAuthPane`, + `AuthPaneState`, `PaneShowsBlockingPrompt`) moved to the leaf + `crux/paneclass`, which imports `crux/agent` so the root does not have to. +- **Fold `crux/types` into the module root.** Done (#13963). It was a + sub-package so that the alias packages in the root could be written + against a stdlib-only leaf before the cut; with the ring in its own module + the leaf no longer needed to be separate from `crux`. Once the root + stopped importing `crux/agent`, the `crux → crux/agent → crux/types` + cycle was gone, and the last edge (the root imported `crux/watchdog` for + `Process.WatchdogConditions`, while `crux/watchdog` imported `crux/types` + for `ConditionReporter` and `ClassAgentCrashLoop`) was cut in the same + change: `Condition`, `ConditionType`, `ConditionStatus` and their + constants are defined in the root and aliased by `crux/watchdog`. The + former leaf's files are now root files, every importer names `crux.`, + and the root imports neither `crux/agent` nor `crux/watchdog`. +- **Inbound ratchet to one row.** After #13982 the inbound baseline was + `cmd/hive` 22, `pkg/dashboard` 3. #14103 moved the dashboard's last + three symbols into the root (`CanonicalizeCopilotModel` with its + CLI-accepted model table, `BackendRequiresInteractiveAuth`; the third, + `IsInferenceBackend`, was already `config.IsInferenceBackend` behind an + alias) and dropped `crux/agent` from the dashboard import allowlist: + dashboard 3 → 0. #14143 moved the pause vocabulary (`BreakerTrigger`, + `PauseCausation`, `PauseTransitionEvent`) into `crux/pause.go`, had + `cmd/hive` name `crux.ProjectContext` and `crux.DefaultAgentMode` + directly, and renamed a loop variable called `agent` in + `initAgentConfigDrivenSystems` that the ratchet's type-free AST walk had + counted as five symbols: cmd/hive 22 → 12. The baseline is now the single + row `cmd/hive` 12, and the twelve are classified, not pending: + `Manager`, `NewManager`, `LinearCredential`, `AgentMintIssuer`, + `StartPermissionsWatcher`, `WatchdogFleet`, `RunAgyTurn`, + `ResumeHandle`, `ResolveReasoningEffort`, `ErrResumeBusy`, + `ErrResumeSessionGone`, `PRCreator`. Each was the in-process manager + itself or a value `cmd/hive` hands it (credential resolvers, the token + minter, the PR creator, the fleet watchdog, the agy-turn subcommand). + #14138 decided the sidecar plan, and its stage 1 took the row to zero: + `cmd/hive` holds `crux.Runtime` plus `crux.Host`, the capability types + are `crux.`, construction goes through `crux/runtime.New`, and the + fleet watchdog through `crux/runtime.NewWatchdogFleet`. The baseline map + is empty and the ratchet stays armed. The `AgentProcess` method split (13 methods, + one mutex) that this record proposed is moot for the same reason: no + consumer outside `crux/agent` names `AgentProcess` any more; they hold + `*crux.Process` snapshots or build fixtures through `crux/cruxtest`. +- **`crux serve`** has shipped its first cut (#13959), the Go client and + CLI remote mode (`crux --server` / `CRUX_SERVER`, #14090) and the serve + docs ([crux-serve.md](../crux-serve.md)); the Stage 3 status block below + has the detail. What remains of this record's list, in #14138's order: + the sidecar stages after the seam (credential direction, the missing + endpoint families, supervision, the flip); `crux/vX.Y.Z` tags and the + `go get` path caveat above; a generic tool policy for the MITM proxy (the rules are + GitHub-shaped today; design record + [crux-tool-policy.md](crux-tool-policy.md), #14228); and keeping this + status section current. +- **Done: move `cmd/crux` into the nested module** (#14138). The module now + ships `crux/cmd/crux`, buildable with `GOWORK=off` from `src/crux` alone + (`make build` / `make smoke` there), with the same verbs, `--server` + remote mode and `--version`. The root `cmd/crux` remains as a thin host + wrapper: same `cli.Env`, Hive's loader. See + [Definition loaders](#definition-loaders) for the split. + +### Definition loaders + +`crux/cli` programs against `cli.DefinitionLoader` (`LoadConfig`, +`LoadDefinitionFile`); two hosts implement it and nothing else in the module +knows which one it got: + +| Host | Loader | Reads | Stays where | +|---|---|---|---| +| `crux/cmd/crux` (module) | `crux/defs.Loader` | `--defs` / `$CRUX_DEFS`: a directory of `*.yaml` or one file; each file a portable `AgentDefinition` (`kind: AgentDefinition`, merged through the same field allow-list as `pkg/defsrc`) or a bare `AgentConfig` overlay named by the file; defaults from `crux/config` (`id`, `role`, one replica, `enabled`, `clear_on_kick`) | in the module: stdlib + `yaml.v3` + `crux/config`, so the outbound ratchet stays empty | +| `cmd/crux` (root) | `hiveLoader` | `--config hive.yaml` through `pkg/config.Load` (enabled agents, `ProjectContext` from the project/github/policies blocks, well-known-agent defaults); the portable format through `pkg/defsrc` | in the root module: everything about *where* a Hive's definitions come from (repo-tree definition sources, the dashboard overlay, `agents_dir`) is host knowledge | + +The root loader borrows the module's name rule and import defaults +(`defs.ValidateName`, `defs.DefaultBackend`, `defs.DefaultRestartStrategy`), +so a definition file loads to the same agent under either binary; the only +intentional difference is the Hive-only presentation table +(`applyKnownAgentDefaults`) and the `ProjectContext`, which a bare crux has +no source for. + +**Stage 3: platform.** `crux serve` with an API, tenancy, auth, quotas; Hive +and hub become clients; the hub-to-runtime contract becomes explicit and +versioned; move to `hivecommons/crux`. The platform pieces wait until a +second consumer is real (candidates: Nous / Experiment Lab, Spektacular, +Dibs, external optimizer and formal-verification lanes, enterprise agent +services). + +*Status: the process boundary has shipped and `cmd/hive` can consume it +behind `CRUX_MODE=sidecar` (stage 3 of the sidecar plan below); in-process +remains the default until stage 4–6 prove parity.* + +- **Shipped: `crux serve` and the API** (operator page + [crux-serve.md](../crux-serve.md)). The transport question the plan left + open ("gRPC or HTTP") is settled as HTTP+JSON over a unix domain socket, + with optional TCP behind a bearer token, and server-sent events for the + two streams. The wire contract is `crux/api`, stdlib-only so a client + does not link the runtime; the server is `crux/serve`; the socket path, + env names and `ServeConfig` are in `crux/config`. The routes under `/v1` + are the CLI verbs — list, get, run from a definition, logs (pane, buffer, + full, or followed), kick, stop — plus `/v1/events`, which carries the + lifecycle actions the API performed, state transitions the server's + watcher observed (crash, restart, pause; there is no runtime hook for + these, so it polls `AllStatuses`), and the kick observer relayed through + `Kicker.SetKickObserver`. It fails closed: no TCP without a token, a + token-less server answers only requests it saw arrive on its own unix + socket, the socket is `0600` from the instant it is reachable in a + directory that must be owned by the serving uid with no group/other bits, + a token when set gates the socket too. Posting a definition applies the + YAML loader's name gate and per-agent defaults, and a name the runtime + already knows is a `409`. Agents start under the server's context, not the + request's, because the runtime ties an agent's lifetime to the context + `Start` received. +- **Shipped: a Go client and CLI remote mode** (#14090). `api.NewClient` + speaks the contract; `crux --server ADDR` (or `CRUX_SERVER`) with `--token` + (`CRUX_TOKEN`) makes `run`, `ps`, `logs`, `kick` and `stop` drive a + server instead of an in-process runtime, which answers the question the + CLI section below raised: a process other than the launcher now owns the + agent by being the one process that holds the runtime, and everyone else + is a client. The per-work-dir control socket of `crux run` stays as the + single-work-dir convenience it was. `serve` itself ignores `CRUX_SERVER` + so a stale address in a shell profile cannot stop a server from starting. +- **Shipped: every `Runtime` family over the API, and a client that is a + `Runtime`** (#14138 stage 3; #14243, #14245). `api.RuntimeRoutes` maps + each method to its route and a parity test in `crux/api/client` keeps + the interface and the table equal; `crux/runtime.New` returns the client + for `CRUX_MODE=sidecar`. Route table in + [crux-serve.md](../crux-serve.md#runtime-families-14138-stage-3). +- **Not done, in this order** (tracked in #14138). *Hive as a client*: the + sidecar branch of `cmd/hive` exists (stage 3) but is not the default; + `pkg/dashboard` still takes the in-process manager, and the failure modes + of the governor against an unreachable runtime are stage 4. *`Process` as + its own type* and *folding `crux/types`* (Stage 2 leftovers above). + *Tags* (`crux/vX.Y.Z`, above). *Tenancy, quotas, the hub contract, the + move to `hivecommons/crux`*: when the second consumer exists. + +### Hive as a client: the staged sidecar plan + +Decided in #14138 ("/hive decision A"): `cmd/hive` consumes its own +`crux serve` API, in six stages so that no stage breaks a hosted hive and +in-process stays the default until parity is proven. The stages are (1) a +`Runtime` seam with two implementations selected by config, (2) the +credential direction, recorded before coding, (3) the endpoint families +the API lacks with a parity test, (4) failure semantics and supervision, +(5) the dashboard through the client, (6) the flip. The inbound ratchet +shrinks only when a stage flips; the outbound ratchet stays empty +throughout; no stage merges without its parity test green. + +#### Stage 1: the Runtime seam + +*Status: shipped.* The inventory below is what the host calls on the +runtime, by family. Each family is an +interface in the `crux` root; `crux.Runtime` composes them, so a consumer +depends on the narrowest one that serves it. `*agent.Manager` satisfies +`Runtime` (asserted in `crux/agent`); dashboard and hub were already on +the six dashboard families and gained nothing new to call. + +| Family | Methods | Consumer today | +|---|---|---| +| `Lifecycle` | `Start`, `Stop`, `Restart`, `RestartWithBootstrap`, `Pause`, `PauseBy`, `Resume`, `IsPaused`, `ResetRestartCount` | dashboard, cmd/hive | +| `Observer` | `GetStatus`, `GetStatusFast`, `AllStatuses`, `ResolveAgent`, `GetOutput`, `GetBufferOutput`, `CaptureFullLog`, `FullLogHeader` | dashboard, cmd/hive, `crux` CLI | +| `Kicker` | `SendKick`, `SendKickAsync`, `ValidateKick`, `KickDispatchState`, `ListKickLogs`, `ReadKickLog`, `SetKickObserver` | dashboard, cmd/hive, CLI | +| `Definitions` | `AddAgent`, `RemoveAgent`, `UpdateConfig`, `ReconcileAgents`, `SetModelOverride`, `SetBackendOverride`, `PinCLI`, `PinModel`, `UnpinCLI`, `UnpinModel`, `ClearModeOverrides`, `SyncModeFiles`, `RefreshInferenceRoutes` | dashboard, cmd/hive | +| `Fleet` | `BreakerState`, `EngageBreaker`, `ReleaseBreaker`, `SetACMMLevel`, `GetACMMLevel`, `RelaunchBobAgentsAwaitingKey`, `SharedDevUID` | dashboard, cmd/hive | +| `Credentials` | `SubmitLoginCode`, `CopilotToken`, `CopilotTokenSource`, `SetCopilotToken`, `ActivateCopilotToken`, `ReloadClaudeToken`, `ClaudeCredentialCandidatePaths` | dashboard | +| `Resumer` (new) | `ResumeHandle`, `SendResumeKick`, `SessionID`, `SessionMissing`; sentinels `ErrResumeBusy`, `ErrResumeSessionGone`, `ErrResumeUnsupported`; value `ResumeHandle` | cmd/hive (PR follow-up) | +| `Supervisor` (new) | `GovernorMayStart`, `MarkStartupLaunchQueued`, `CheckAndRestartCrashedAgents`, `ConsentWedgedAgents`, `RefreshAgentTokens`, `RefreshAgentTokenFor`, `AgentCredentialRefreshable`, `AgentHasValidCredential`, `AgentCapabilities`, `EffectiveBackend`, `InvocationMetadata`, `RestartTelemetry`, `StartFailureState`, `ProviderErrorBackoffRemaining`, `PauseByCause`, `ArchiveAllKickLogs`, `CountAgentsWithModel`, `CrashOOMSuspected`, `BackendAuthAvailable`, `AgentAuthAvailable`, `SendGitHubAPIBudgetNudge`, `RecordAudit`, `TrajectoryAgents` | cmd/hive (governor, heartbeat, rotation, login scan, trajectory) | +| `Restorer` (new) | `SeedPauseState`, `SeedRestartTelemetry`, `SeedTurnLoss`, `SeedLastKick`, `SeedKickHistory`, `RestoreBreaker` | cmd/hive (snapshot restore) | +| `Background` (new) | `StartAgentTokenRefresh`, `StartCredentialWatchdog`, `StartCopilotSessionRefresh`, `StartTranscriptActivityRefresh` | cmd/hive (boot) | +| `Authorizer` (new) | `AuthorizePROpen`, `AuthorizePushBranch`, `AuthorizeIssueOpen`, `AuthorizeMerge`, `AuthorizeReviewRequest` | cmd/hive (request relays; method values handed to the watchers at boot) | + +Two things `cmd/hive` calls are **not** runtime-instance methods and are +re-exported by the new `crux/runtime` package rather than put on the +interface: `RunAgyTurn` (the `hive agy-turn` subprocess; a sidecar runs it +as a `crux` subcommand) and `StartPermissionsWatcher` (a process-lifetime +loop over the agent home directories). `WatchdogFleet`, which reads the +manager's agent table under its lock, is built by +`crux/runtime.NewWatchdogFleet(rt, hooks)` and returns `ErrNotInProcess` +for any other implementation; its API shape is a stage 3 endpoint family. + +**Hive-provided capabilities, deliberately not in `Runtime`.** The +runtime is handed these; it does not implement them, and a sidecar +runtime will receive them over a hive-side callback endpoint (stage 2 +decides the shape) or publish them as API events (stage 3). The *types* +moved to the `crux` root so a host names them without importing +`crux/agent` (`crux/agent` aliases each): `AppTokenMinter`, +`AgentMintIssuer`, `LinearCredential`, `PRCreator`, `SpanStarter`. The +*installation* of them is `crux.Host`, a separate interface only the +in-process implementation satisfies (`crux/runtime.HostOf`), grouped by +what replaces it: credential capabilities (`SetAppAuth`, `SetAgentMint`, +`SetLinearCredentialResolver`, `SetBobAPIKeyResolver`, +`SetBobKeySourceResolver`, `SetSandboxPushMinter`, `SetSandboxPRClient`) +→ callback endpoint; launch-time resolvers (`SetBuildInfo`, +`SetGatewayBackendChecker`, `SetExplainModeDefaultResolver`, +`SetRepoAutoMergeEnabledResolver`, `SetKickLogSourceResolver`, +`SetInferenceCallbacks`, `SetSandboxConfig`, `SetSandboxMutationBoundary`, +`SetDeadSessionRecoveryOwner`) → definition fields or a callback; +observers (`SetKickOutcomeObserver`, `SetPauseTransitionObserver`, +`SetRelaunchObserver`, `SetPersistPauseCallback`, +`SetRecordPromptCallback`, `SetSandboxAuditCallback`, `SetAuditSink`, +`SetConditionReporter`, `SetSpanStarter`) → `/v1/events`. +`ResolveReasoningEffort` turned out to be pure (it reads only +`config.ReasoningEffortsByBackend`), so it moved to `crux/config` with +`AgyLaunchEffort` and `AgyDefaultEffort`; `crux/agent` forwards to it. + +**Selection by config.** `crux/config.Mode` is `inprocess` (default) or +`sidecar`; `config.RuntimeConfig{Mode, Server, Token}` is read from +`CRUX_MODE`, `CRUX_SERVER`, `CRUX_TOKEN` by `RuntimeConfigFromEnv` and +validated at construction, so a typo fails boot rather than silently +running in-process. Like `ServeConfig` it is a process-level setting, not a +`hive.yaml` key; the key lands with stage 4, when an operator first has +supervision to configure. `crux/runtime.New(cfg, opts...)` returns the +in-process `*agent.Manager` as `crux.Runtime` for `inprocess` (options: +`WithDefinitions`, `WithLogger`, `WithProject`, `WithUIDMapPath`) and, +since stage 3, the `crux/api/client` runtime for `sidecar` +(`ErrModeUnavailable` is now only for a mode the build does not know). The +package is the one place outside `crux/agent` that names the manager on +the construction path; `crux/cli` and `crux/cruxtest` keep their own +constructors for now. `cmd/hive` holds the result as a local +`hostedRuntime` (`crux.Runtime` + `crux.Host`), fails boot if the selected +mode cannot provide the `Host` half, and passes the narrowest family on +(`crux.Host` to the installers, `crux.Runtime` elsewhere); its tests build +runtimes through `cruxtest.NewRuntime`. The inbound ratchet's `cmd/hive` +row went 12 → 0 and the baseline map is empty. + +**Parity test.** `crux/cli.RuntimeVerbs` maps each CLI verb (`run`, +`ps`, `logs`, `kick`, `stop`) to the `Runtime` methods it calls; +`crux/cli`'s tests assert every listed method exists on `crux.Runtime`, +every verb the dispatcher routes to a runtime is listed, and the listed +methods are exactly the `rt.` selectors in each command body +(following helpers such as `stream`). Stage 3 adds the other half, every +verb and every `Runtime` method against an API route. + +**What stage 3 needed from the API** to serve `Runtime` in full, from the +table above: the `Resumer` family (handle capture, resume kick with the +busy / gone / unsupported distinction as status codes); the `Authorizer` +gates, which sit on the relay request path and so need a synchronous +endpoint with a latency budget; the `Supervisor` reads and actions; `Restorer` as a bulk seed on connect; `Background` as +server-side loops (a no-op for the client); the `Fleet` watchdog +observe/restart and the dashboard's `Credentials` family; `/v1/events` for +the observer half of `Host`; and a `crux agy-turn` subcommand so the +`hive agy-turn` subprocess does not need the runtime linked in. + +#### Stage 3: the endpoint families and the sidecar client + +*Status: shipped in five lane-merged PRs; in-process stays the default.* + +- **Endpoints** (#14243). `crux/api/runtime.go` holds the paths, wire + types and codes for every family; `crux/serve/families.go` serves them + against any `crux.Runtime`. Resume refusals are status codes (`409 + resume_busy`, `410 resume_session_gone`, `422 resume_unsupported`); + `POST /v1/authorize` is synchronous and a transport error is a refusal; + `Restorer` is one `POST /v1/agents/{name}/seed` body with optional + fields; `Background` is the server's own loops. The watchdog's + per-agent reads became a `watchdog.Source` interface (`AgentNames`, + `Observe`, `LastProduction`, `SetConditions`) with its own routes, `501 + not_supported` when the runtime has none. When the runtime is a + `crux.Host`, the server installs the eight observers and relays them on + `/v1/events` with typed `data` (`agent.kick_outcome`, + `agent.pause_transition`, `agent.relaunch`, `agent.pause_persist`, + `agent.prompt`, `agent.sandbox_audit`, `runtime.audit`, + `runtime.condition`). `cruxtest.Recorder` fakes it all. +- **Client** (#14245). `crux/api/client.Runtime` implements the full + interface over `api.Client`; `crux/runtime.New` returns it for + `sidecar`, dialing `$CRUX_SERVER` with `$CRUX_TOKEN`. One reconnecting + `/v1/events` subscription feeds `SetKickObserver` and + `SetEventObserver`. `Authorizer` fails closed on any error; + `Background` is a no-op; `crux/runtime.NewWatchdogFleet` builds the + fleet over any `WatchdogSourcer`, so the rotation watchdog runs + unchanged against a sidecar. + **Parity test**: `TestParityInterfaceVersusRouteTable` reflects over + `crux.Runtime` and asserts every method is a key of `api.RuntimeRoutes` + and vice versa, so a method cannot be added without a route. +- **Callback wiring** (#14246, #14248). `crux serve --hive-callback` + (`$CRUX_HIVE_SOCKET`, `$CRUX_HIVE_TOKEN`) dials the hive callback + endpoint and feeds the manager's seams from `hivecap.New` (text + scanner, condition reporter, PR creator, Linear resolver, sandbox push + minter); `Client.Status()` is watched into the + `hive_callback_unavailable` condition and reported under + `hive_callback` on `/v1/health`. On the hive side, `cmd/hive` in sidecar + mode wraps the client in `sidecarHost`: the observer setters are fed + from the event stream, the capability setters are inert because the + callback carries them, and the policy setters (`SetBuildInfo`, + `SetGatewayBackendChecker`, `SetSandboxConfig`, `SetSpanStarter`, …) are + inert with a one-time log until stage 4. `bootAgents` serves + `cruxhost.New(deps)` on `callback.Listen` before the rest of the wiring + and exits if it cannot bind. `Usage` (the inference sink) stayed + `not_configured` until #14405 wired it. +- **`crux agy-turn`** (#14247). The manager types its own executable into + headless agy panes; under `crux serve` that is `crux`, so the subcommand + exists there too. + +**What stage 3 leaves for stage 4.** The policy setters above as routes +or configuration (the `hive.yaml` key for the runtime, so an operator can +set sandbox config, the gateway checker, explain-mode defaults, repo +auto-merge, dead-session recovery ownership); `SetAgentMint`, `SetAppAuth` +as callback-backed cache writers in `crux/agent` (today the sidecar's +agents have no mint issuer); the proxy move and with it the `Accounting`, +`Usage` and `Canary` seams fed from the callback; `SetSpanStarter` +(tracing across the boundary); the governor's behaviour against an +unreachable runtime (reconnect, degraded state, the +`hive_callback_unavailable` / runtime-unreachable pair of conditions); +`pkg/dashboard` on the client (stage 5); and a soak of a hosted hive in +sidecar mode before the flip (stage 6). + +**Stage 4 progress.** *Status: shipped* (#14394 policy setters, #14400 +failure semantics and supervision, #14404 credential writers, #14405 +proxy seams). Every `crux.Host` setter now has a disposition in +`api.HostSetters`, and `TestParityHostVersusSetterTable` fails when a +setter is added without one. + +| Setter | Disposition | Via | +|---|---|---| +| `SetBuildInfo`, `SetGatewayBackendChecker`, `SetExplainModeDefaultResolver`, `SetKickLogSourceResolver`, `SetSandboxConfig`, `SetDeadSessionRecoveryOwner` | callback | `GET /v1/policy` snapshot (`hivecap.WatchPolicy`, polled every 15 s) | +| `SetRepoAutoMergeEnabledResolver` | callback | `POST /v1/policy/repo-auto-merge`, fail closed (merge refused) | +| `SetInferenceCallbacks` | callback | `PUT`/`DELETE` `/v1/inference/routes/{agent}`: the sidecar manager pushes each agent's route into hive's proxy | +| `SetSandboxMutationBoundary` | callback | `POST /v1/effects`, `POST /v1/effects/{id}/end`: two-phase claim/end over hive's journaling boundary, fail closed | +| `SetAppAuth` | callback | `POST /v1/tokens` with `{agent, tier}` and no repos (installation-wide, what `AppAuth.ScopedToken(tier)` mints in-process); `hivecap.AppTokenWriter` writes the token in place to the agent's cache file | +| `SetAgentMint` | callback | `POST /v1/tokens/mint` (`agent_mint` capability); mint off is `""`, the in-process contract | +| `SetBobAPIKeyResolver`, `SetBobKeySourceResolver` | callback | `GET /v1/bob/credential` (`bob` capability), cached 1 min runtime-side | +| `SetLinearCredentialResolver`, `SetSandboxPushMinter`, `SetSandboxPRClient` | callback | `GET /v1/linear/credential`, `POST /v1/tokens`, `POST /v1/pulls` (stage 2) | +| observers (`SetKickOutcomeObserver`, `SetPauseTransitionObserver`, `SetRelaunchObserver`, `SetPersistPauseCallback`, `SetRecordPromptCallback`, `SetSandboxAuditCallback`, `SetAuditSink`, `SetConditionReporter`) | event | the `/v1/events` stream (stage 3); conditions also `POST /v1/conditions` | +| `SetSpanStarter` | in-process-only | a span is a Go value wrapping a call in one address space. The sidecar's manager starts spans on the OpenTelemetry global tracer (a no-op without an exporter in the runtime's process); trajectory events do not depend on it | + +No setter is served from a `runtime:` config key: every value is already +resolved hive-side from the live config, so a reload reaches the runtime +on its next poll, and the callback pattern was already there. + +- [x] Policy setters over `GET /v1/policy` / `POST /v1/policy/repo-auto-merge` + (#14394). See [crux-serve.md](../crux-serve.md#host-policy-from-hive-14138-stage-4). +- [x] `SetInferenceCallbacks` over the `inference_routes` family and + `SetSandboxMutationBoundary` over the `effects` family (#14394). +- [x] `SetSpanStarter` documented as in-process-only (#14394). +- [x] `SetAgentMint` / `SetAppAuth` / Bob key resolvers as callback-backed + writers (`crux/hivecap/credentials.go`, #14404). Tokens are never logged + and go only to the existing per-agent cache path; all three fail closed + (no credential) when hive is away. `HIVE_PROXY_INJECT_GH_AUTH` is + refused in sidecar mode rather than silently writing the real token: + the proxy-inject registry lives in `pkg/github` (stage 5). +- [x] The proxy seams from the callback (#14405): hive serves the + inference usage ledger (`usage` was `not_configured` since stage 3) and + a proxy on `hivecap.Set.{Accounting,Usage,Canary}` refuses writes while + hive is away and resumes when it returns (tested over a real socket). + The proxy *process* still runs in hive; the move proper is stage 5. +- [x] Failure semantics and supervision (#14400), below. + +**Failure semantics decided in stage 4** (full table in +[crux-serve.md](../crux-serve.md#supervision-and-failure-semantics-14138-stage-4)): + +- *Runtime socket gone.* The governor keeps its loop. `client.Runtime` + methods return `crux.ErrRuntimeUnavailable` (wrapping + `api.ErrUnreachable`); kicks are **refused, not queued** — a kick is a + prompt about the state of the world when it was issued, and replaying + a stale one after a long outage is worse than the governor's next + cycle re-deriving it. `cmd/hive` probes the runtime every 15 s and + raises `crux_runtime_unavailable` (fleetreport class, severity high, + key `sidecar`) on the first failure, clearing it on the first success; + `/api/health` carries `crux_runtime` and reports `degraded`. +- *Supervision.* `CRUX_SUPERVISE=child` (default) spawns and supervises + `crux serve` as a child with exponential backoff (1 s → 1 min) and a + readiness check on `/v1/health`; `external` only reconnects. A missing + binary logs loudly and behaves as `external`. +- *Health.* `/v1/health` includes `hive_callback`, `started_at` and + `uptime_seconds`; hive's `/api/health` includes `crux_runtime`. + +**What stage 4 leaves for stage 5** (the dashboard through the client): + +- `pkg/dashboard` on `client.Runtime`; render `crux_runtime_unavailable` + on the fleet card and the condition list (the class exists; the + rendering path has not been exercised in sidecar mode). +- The proxy move proper: the MITM proxy in the runtime process on the + callback seams (CA distribution, agent env), and with it + `HIVE_PROXY_INJECT_GH_AUTH` for sidecar agents (today refused). +- Kick replay revisit once the dashboard shows the condition: a bounded + queue is a one-line change in `client.Runtime` if operators want it. +- Supervisor pidfile and `SIGTERM` propagation on hive shutdown; today the + child's context is hive's and it is killed with it. +- The watchdog's rotation probe (`UsageProber`) as a callback family. +- A soak of a hosted hive in sidecar mode before the flip (stage 6). + +## Credentials and hive-provided capabilities + +Status: **Stage 2 of the #14138 sidecar plan (decision A): done.** Design +in #14197; the callback contract (`crux/api/callback`), the runtime-side +adapters (`crux/hivecap`) and the hive-side mount (`pkg/cruxhost`) in +#14222 (coverage 94.8% / 99.1% / 100%, default floors). Stage 3 wired both +ends (see "What stage 3 wires" below): `crux serve --hive-callback` and the +sidecar branch of `cmd/hive`'s `bootAgents`. + +The decision this section records: **the runtime never holds the GitHub +App private key, or any other hive-owned credential.** When hive runs the +runtime as a sidecar (`crux serve`), the runtime obtains short-lived scoped +tokens and every other hive-owned capability by calling back into hive over +a hive-side endpoint. In-process, the same capabilities are the direct +function calls `cmd/hive` already installs today. The seam interfaces the +Stage 2 inversions left behind are the contract in both modes; the sidecar +adds one implementation of each, not a second contract. + +### Inventory + +Every interface the ring defines for something the host supplies, measured +on v5 at `8bce0b0` (Oct 25, 2026) with +`grep -rn 'interface {' src/crux --include='*.go' | grep -v _test`. The +last column is the classification this stage settles: a *capability* is +something hive owns (a credential, a registry, an accounting ledger, a +forge client) that the runtime asks for; a *runtime feature* is behaviour +the runtime owns and only lets the host observe or tune. + +| Seam | Defined in | Hive implementation today | What crosses the boundary | Sensitivity | Class | +|---|---|---|---|---|---| +| `pushbroker.TokenScoper` (`ScopedTokenForRepos(ctx, tier, repos)`) | `crux/pushbroker` | `*github.AppAuth` via `cmd/hive`'s `appMinter` | tier + repo names in; an installation token out | **credential**: the token is a live GitHub write credential | capability | +| `pushbroker.TokenMinter` (`MintPushToken(ctx, repo)`) | `crux/pushbroker` | `pushbroker.GitHubAppMinter{Auth: TokenScoper}` (ring code over the seam above) | same | same | capability (derived from `TokenScoper`) | +| `agent.AppTokenMinter` (`WriteAgentToken(ctx, agent, tier, uid)`) | `crux/agent` | `*github.AppAuth` (`Manager.SetAppAuth`) | agent, tier, uid in; hive **writes the agent's token cache file** as a side effect | **credential**, plus a filesystem write into the agents' tree | capability, but mis-shaped for a process boundary (see below) | +| `agent.AgentMintIssuer` (`Enabled`, `MintAgentToken(agent, tier)`) | `crux/agent` | `mint.AgentMinter` (`cmd/hive` `buildMinter`) | agent + tier in; an OIDC/JWT out | **credential** (opt-in, additional) | capability | +| `agent.LinearCredential` + `SetLinearCredentialResolver(func() LinearCredential)` | `crux/agent`, used by `crux/proxy` | closure in `cmd/hive` over hub OAuth / `hive.yaml` | a Linear access token or API key out | **credential** | capability | +| `agent.PRCreator` (`CreatePR(ctx, repo, head, base, title, body)`) | `crux/agent` | `*github.Client` (`SetSandboxPRClient`) | PR fields in; `crux.PRResult` out; hive authors as the App bot | hive's forge identity acts; no secret crosses | capability | +| `proxy.GitHubRESTAccounting` (`RecordRESTRequest`, `RecordInjectedRESTRequest`, `CoreRateLimitSnapshot`) | `crux/proxy` | `cmd/hive`'s `githubRESTAccounting{}` over `pkg/github` package state | caller, method, path, status and the rate-limit response headers in; the shared core-bucket snapshot out | telemetry; the snapshot feeds the per-agent GitHub budget's reserve floor | capability (hive owns the ledger) | +| `proxy.UsageSink` (`Record`, `RecordCopilot`, `RecordCopilotWithCache`) | `crux/proxy` | `*tokens.InferenceSink` | per-agent inference token counts in | telemetry (billing) | capability | +| `proxy.CanaryScanner` (`Scan(agent, text, source)`) | `crux/proxy` | `*ioscan.CanaryRegistry` | a GitHub write body in; "leaked" + which agent/source out | the body may hold agent work product; the registry holds the planted canaries, which **must not** be shipped to the runtime | capability | +| `toolapprove.TextScanner` (`ScanInput`, `ScanOutput`) | `crux/toolapprove` | `ioscan.ToolApproveScanner` (`SetDefaultTextScanner` in `cmd/hive`'s `init`) | tool payload text in; a verdict with masked snippets out | payload may hold secrets the scanner is there to catch | capability (hive owns the rule set) | +| `watchdog.UsageProber` (`Provider`, `ProbeError`) | `crux/watchdog` | `cmd/hive`'s `usageProber` over `pkg/rotation` | provider name in; sanitised probe error out | hive holds the provider credentials the probe uses | capability (deferred; stays with the rotation watchdog until stage 3 moves it) | +| `crux.ConditionReporter` (`Observe`, `Clear`) | `crux` root, used by `crux/agent`, `crux/watchdog` | `fleetreport.DefaultReporter{}` | class, key, message, scrubbed diagnostics out | diagnostics are already scrubbed at the call site | capability (hive owns the fleet report) | +| `agent.SpanStarter` | `crux/agent` | `tracing.StartSpan` | a live `trace.Span` | none | **runtime feature**: not remotable, see below | +| `crux.AuditSink` (`Record(actor, action, agent, fields)`) | `crux` root, used by `crux/agent` | the dashboard's agent-audit recorder, injected by `cmd/hive` | lifecycle audit entries out | none (already scrubbed) | capability (deferred with `UsageProber`; must not block, so it becomes a batched route, not a per-call one) | +| `watchdog.Alerter`, `watchdog.AuthProbe`, `watchdog.KickPeriodReporter`, `watchdog.KickSuppressionReporter`, `trajectory.Sink` | `crux/watchdog`, `crux/trajectory` | `cmd/hive` adapters over Discord/Slack, `pkg/rotation`, `pkg/fleetreport` | alerts and observations out | none | observers; they ride the `api.Event` stream in stage 3, not callbacks | +| `Kicker.SetKickObserver`, `proxy.SetAgentTokenSource`, `SetRepoPausedFunc`, `SetAgentRepoScopeFunc`, `SetWriteSurfaceEnforceFunc`, `SetWriteRefusedAuditFunc`, `SetAgentGitHubBudgetConfig`, … | `crux/agent`, `crux/proxy` | `cmd/hive` closures over `pkg/config` | policy answers (bools, caps) and observer callbacks | none | runtime features configured by the host; they become API routes or config in stage 3, not callbacks | + +Two of the capability rows are not callback-shaped as they stand: + +- `AppTokenMinter.WriteAgentToken` has hive write a file into the agents' + token-cache directory as the agent's uid. Across a process boundary the + runtime owns that directory and those uids, so the capability collapses + onto *mint a scoped token* and the runtime writes the cache file itself. + Stage 3 replaces `SetAppAuth` in sidecar mode with a writer built over the + mint callback; the in-process path keeps `*github.AppAuth`. +- `SpanStarter` returns an in-process `trace.Span` whose `End` must be + called by the runtime; there is no sensible remote form. The runtime's + default (the OpenTelemetry global tracer) is the sidecar answer, with the + usual W3C context propagation on the API requests, and hive's + per-component reach bookkeeping stays a hive concern. This is why span + start is **not** in the callback contract. + +### Direction + +Runtime → hive. The runtime is the caller for every capability; hive never +pushes a credential into the runtime, and the runtime never receives the +App private key, the installation JWT, the canary registry, the Linear +OAuth refresh token or the rotation providers' credentials. Tokens are +minted per request, scoped to a tier and to the named repositories +(`ScopedTokenForRepos`' existing contract), attributed to an agent, and +carry an expiry the runtime must honour (GitHub installation tokens expire +in an hour; the response says when). The runtime keeps a token only as long +as the operation that asked for it (`pushbroker` holds it for one push, and +already keeps it out of `argv` and logs) or, for the per-agent cache file, +until the expiry the response carried. + +Trust boundary: the runtime process is trusted to attribute requests to +the right agent, because it is the enforcement point that identifies the +agent today (the proxy's per-agent routing, the broker's `AgentName`). The +callback endpoint authenticates *the runtime*, not individual agents, and +hive keeps the authority to refuse (an agent not in its fleet, a repo not +in the installation, a tier the agent is not entitled to). An agent cannot +reach the hive socket: it lives in hive's private runtime directory, mode +`0600`, and agents run as their own uids. + +### Transport + +The callback endpoint reuses `crux/api`'s conventions: HTTP+JSON, routes +under `/v1`, `api.Error` bodies with stable `code`s, a `0600` unix socket in +a uid-owned `0700` directory, optional TCP only behind a bearer token, and +a token-less server that answers only requests it saw arrive on its own +socket. The contract is the stdlib-only package `crux/api/callback`; a +hive that mounts it links nothing of the runtime, and a runtime that calls +it links nothing of hive. + +- **Hive serves.** Bidirectional means two listeners: `crux serve` keeps + its socket for hive's commands, and hive opens a second one for the + runtime's callbacks. Default location: `hive.sock` next to the runtime's + socket, so `$CRUX_HIVE_SOCKET` if set, else + `$XDG_RUNTIME_DIR/crux/hive.sock`, else `/crux-/hive.sock`. + The two processes run as one uid in one container by default (stage 4 + supervision), so the file mode is the credential, exactly as for + `crux.sock`. `$CRUX_HIVE_TOKEN` adds a bearer token; `$CRUX_HIVE_LISTEN` + (TCP, for the k8s sidecar-container option) requires one and must bind a + loopback address — the listener is plain http, so a routable bind would + carry the token and every minted installation token in cleartext. +- **Routes** (`callback.Path*`), one per capability, request and response + bodies as the `callback.Request/Response` types: + + | Capability | Route | In | Out | + |---|---|---|---| + | Health | `GET /v1/health` | — | `{version}` plus which capabilities are configured | + | Mint scoped token | `POST /v1/tokens` | agent, repo (`owner/name`) or repos (names), tier, requested TTL | token, `expires_at` | + | GitHub REST accounting | `POST /v1/github/rest` | caller, method, path, status, injected (bool), the `X-RateLimit-*` response headers | `204` | + | Core rate-limit snapshot | `GET /v1/github/rate-limit` | — | remaining, reset, ok | + | Inference usage | `POST /v1/usage` | agent, model, kind (`gateway` / `copilot` / `copilot_cache`), input, output, cache read, cache creation | `204` | + | Canary scan | `POST /v1/scan/canary` | agent, text, source | detected, agent, source (**never the canary token**) | + | Text scan | `POST /v1/scan/text` | direction (`input` / `output`), text | blocked, critical_injection, findings (kind, masked snippet, rule, high) | + | Condition | `POST /v1/conditions` | action (`observe` / `clear`), class, key, message, diagnostics | `204` | + | Linear credential | `GET /v1/linear/credential` | — | access_token or api_key | + | Create PR | `POST /v1/pulls` | repo, head, base, title, body | `crux.PRResult`'s fields | + + A capability hive does not have (no GitHub App, no canary registry, no + Linear) answers `404 not_configured`, which the runtime treats exactly as + the nil seam it stands in for today (no App → the push fails; no registry + → nothing is scanned; no Linear → no injection). A hive-side failure of a + configured capability (GitHub refused the mint) is `502 upstream_error` + with the message; a bug is `500 hive_error`. Both are refusals to the + caller. +- **Both ends in Go.** `callback.Capabilities` is the one interface set, + with wire types on both sides. `*callback.Client` implements it over + HTTP; the hive-side `pkg/cruxhost` implements it directly over hive's + packages (`*github.AppAuth`, `*ioscan.CanaryRegistry`, + `*tokens.InferenceSink`, `fleetreport`, `*github.Client`, the Linear + resolver) and `callback.NewServer(caps)` serves either. `crux/hivecap` + turns any `Capabilities` into the ring's seam interfaces + (`pushbroker.TokenScoper`, `proxy.GitHubRESTAccounting`, + `proxy.UsageSink`, `proxy.CanaryScanner`, `toolapprove.TextScanner`, + `crux.ConditionReporter`, `agent.PRCreator`) so the runtime's existing + setters take them unchanged. +- **In-process mode: no socket.** `cmd/hive` keeps installing its + implementations into the seams directly, as it does now; that path is + not changed by this stage and is deleted only in stage 6. Because + `crux/hivecap` is written against `callback.Capabilities` rather than + `*callback.Client`, a hive that wants one wiring path for both modes can + hand `pkg/cruxhost`'s `Capabilities` value to `hivecap` in-process and a + `*callback.Client` in sidecar mode; the parity test in `crux/hivecap` + asserts the two produce identical seam behaviour. Stage 3 selects between + them on `crux.mode`. + +### Failure semantics: fail closed + +When hive is unreachable, every capability refuses rather than degrades +into a less-safe answer. The runtime already has a refusal path for each +seam, and the callback adapters return the value that takes it: + +| Seam | Hive unreachable / unauthorized | Effect | +|---|---|---| +| `TokenScoper` | error wrapping `callback.ErrUnavailable` | `pushbroker` fails the push ("minting push token"); the agent's branch stays local. The per-agent cache file is not rewritten, so the proxy's token source has no token and `rewriteGitHubAuth` forwards the write **without credentials** → upstream 401. The agent continues read-only on public repos. | +| `CanaryScanner` | reports a detection with an empty token and the source suffixed ` (hive callback unavailable)` | with `canaryFailClosed` the proxy refuses the write; without it the write is audited as a leak. Both are what an unscannable body does today. | +| `TextScanner` | a `Blocked` verdict with one finding (`kind=callback`, `rule=hive_unavailable`) | `toolapprove` denies, as `ErrNoTextScanner` does with no scanner. | +| `GitHubRESTAccounting` | records are dropped; `CoreRateLimitSnapshot` returns `ok=false` | the agent GitHub budget applies the per-agent cap alone, the documented behaviour when no snapshot is installed. | +| `UsageSink` | records are dropped | billing undercounts for the outage; counted in `Client.Status()`. | +| `ConditionReporter` | reports are dropped | the fleet report misses the window; counted in `Client.Status()`. | +| `LinearCredential` | empty credential | the proxy injects nothing; the agent's Linear call fails upstream. | +| `PRCreator` | error wrapping `callback.ErrUnavailable` | the sandbox executor reports the kick as failed, same as a forge error. | + +`401` from hive (token mismatch) is `callback.ErrUnauthorized` and takes +the same rows; it is never retried with a different credential. The client +has no logger and its errors never contain the bearer token, a minted +token, or a request body; `MintTokenResponse` prints with the token +redacted so an accidental `%v` cannot leak it. + +The outage is a runtime condition. `callback.Client.Status()` reports +`Reachable`, the last error, when it began and how many calls failed +closed; the class name `hive_callback_unavailable` is reserved for it. +Stage 4 surfaced it on `GET /v1/health` (`hive_callback`) and as a +condition on the event stream, and defined the governor's reaction to +the other "socket is gone" case (`crux_runtime_unavailable`, kicks +refused, loop kept; see "Failure semantics decided in stage 4" above). Recovery is per call: the next successful callback +clears the condition; there is no circuit breaker, because the fail-closed +answers are already cheap and the per-call timeout +(`callback.DefaultTimeout`, 5 s) bounds the cost of a dead socket. + +### What stage 3 wires + +- `cmd/hive` (#14248, done): `cruxhost.New(cruxhost.Deps{...})` over the + values it already constructs, served on + `callback.Listen(callback.DefaultSocket())` next to the runtime socket, + in sidecar mode only. The Linear resolver and the usage ledger are read + lazily because `bootSupervision` and `bootProxy` build them after + `bootAgents`. +- `crux serve` (#14246, done): a `--hive-callback` address (or + `$CRUX_HIVE_SOCKET`), `callback.NewClient`, `hivecap.New(client)`, and + the manager setters fed from it instead of nil (`toolapprove` text + scanner, condition reporter, PR creator, Linear resolver, sandbox push + minter over the token scoper). The proxy setters follow the proxy move. +- Stage 4 (done): `AppTokenMinter` → `hivecap.AppTokenWriter` over + `POST /v1/tokens`; `AgentMintIssuer` → `hivecap.AgentMint` over + `POST /v1/tokens/mint`; the Bob resolvers → `hivecap.BobCredentials` + over `GET /v1/bob/credential`. `UsageProber` follows when the watchdog's + rotation probe moves (stage 5). ## CLI The Stage 2 plan called for a `crux` binary; it shipped in Stage 1 instead -(`src/cmd/crux`, logic in `src/pkg/crux/cli`, operator page +(`src/cmd/crux`, logic in `src/crux/cli`, operator page [crux-cli.md](../crux-cli.md)) because it is the cheapest proof that `crux.Runtime` is a sufficient surface: every subcommand is written against the interface, and the only `*agent.Manager` reference is the constructor in @@ -227,6 +985,11 @@ the ratchet boundary, so it adds no baseline row. | `logs` | `Observer.ResolveAgent`, `Observer.GetOutput` / `GetBufferOutput` / `FullLogHeader`+`CaptureFullLog` | | `kick` | `Observer.ResolveAgent`, `Kicker.SendKick` | | `stop` | `Observer.ResolveAgent`, `Lifecycle.Stop` | +| `serve` | all of the above, over the API (Stage 3; [crux-serve.md](../crux-serve.md)) | +| `agy-turn` | none: the pane-side `agent.RunAgyTurn` subprocess the manager spawns from its own executable (#14247) | + +With `--server ADDR` / `CRUX_SERVER` the first five go through `api.Client` +to a `crux serve` instead; `serve` itself is only ever local. Definitions are the two formats hive already reads (`kind: AgentDefinition` via `defsrc`, or an `agents_dir` overlay via `config.LoadAgentOverrides`), @@ -239,9 +1002,12 @@ adoption method, so a second process cannot stop or kick an agent that a first process started. The CLI therefore makes `crux run` the long-lived command and has it serve a unix control socket (`/crux.sock`) that `ps`/`logs`/`kick`/`stop` forward to when present. That socket is a -single-work-dir convenience and is not the Stage 3 API; what Stage 3 has to +single-work-dir convenience and is not the Stage 3 API; what Stage 3 had to add to `Runtime` (or beside it) is exactly the piece the socket papers over: a way for a process other than the launcher to own or adopt an agent. +`crux serve` is that piece, by making the server the one launcher and +everything else a client; adoption of an agent started by some other +process is still not a `Runtime` capability. ## Configuration leaf @@ -303,22 +1069,28 @@ the same change that swaps its imports): ## Outbound edges Imports from ring packages to Hive packages outside the ring, as the -outbound ratchet records them after the configuration-leaf migration: no -ring package imports `pkg/config`. Entries marked *(test)* appear only in -`_test.go` files. +outbound ratchet records them after Stage 2 prerequisites 1, 2 and 3 and +the step 4 pre-cut: none. The baseline is empty in both directions (test +and non-test). The last two rows were `pkg/agent → internal/testutil` +*(test)* and `pkg/inference → internal/testutil` *(test)*: the `Eventually*` +and `Skip[f]UnlessRequired` helpers those tests used copied into +`pkg/crux/testutil`, a ring package that becomes `crux/internal/testutil` +at the module cut; `internal/testutil` keeps the canonical copy and the +tests of the contract. -| Ring package | Outside Hive imports | -|---|---| -| `cmd/crux` | `config`, `defsrc` (the `DefinitionLoader` implementation; moved here from `pkg/crux/cli`, see [Configuration leaf](#configuration-leaf)) | -| `pkg/agent` | `effects`, `fleetreport`, `gitidentity`, `jev`, `outputschema`, `pushbroker`, `internal/testutil` *(test)* | -| `pkg/inference` | `watsonx`, `internal/testutil` *(test)* | -| `pkg/proxy` | `github`, `inferencehealth`, `ioscan`, `issueshape`, `tokens` | -| `pkg/toolapprove` | `ioscan` | -| `pkg/trajectory` | `jsonextract` | -| `pkg/watchdog` | `fleetreport`, `rotation` | +Before the pre-cut the table had those two test-only rows; before +prerequisite 3 the table also had `pkg/proxy → github, +inferencehealth, ioscan, issueshape, tokens` and `cmd/crux → config, +defsrc`; before prerequisite 2, `pkg/agent → effects, fleetreport, +gitidentity, jev, outputschema, pushbroker`, `pkg/inference → watsonx`, +`pkg/toolapprove → ioscan`, `pkg/trajectory → jsonextract` and +`pkg/watchdog → fleetreport, rotation`; the per-edge decisions are under +[Staged plan](#staged-plan). `cmd/crux` is no longer a ring package; it is +a host in the root module (prerequisite 3) and the inbound ratchet measures +it instead. Edges removed in #13785, and the shape each cut took, because the same -shapes apply to the rows above: +shapes applied to the rows that followed: | Edge | Cut | |---|---| @@ -330,26 +1102,26 @@ shapes apply to the rows above: The rule the inbound ratchet imposes on these cuts: an inversion must not add an `agent.` reference in a consumer, so seams are `Manager` -setters or `crux.` / `pkg/crux/types` re-exports, never new exported +setters or `crux.` definitions, never new exported `agent` symbols the host has to name. ## Running the Crux set alone -There is no separate module yet, so the ring is tested as a package list -from `src/`: +The ring is the module at `src/crux`, so it is tested as one: ```sh -cd src && go test ./pkg/agent/... ./pkg/crux/... ./cmd/crux \ - ./pkg/agentmode ./pkg/agentparse ./pkg/agentsmd ./pkg/claude \ - ./pkg/sandbox ./pkg/kubejob ./pkg/toolapprove ./pkg/skillreg \ - ./pkg/inference ./pkg/openrouter ./pkg/watchdog ./pkg/trajectory \ - ./pkg/logscrub ./pkg/proxy +cd src && go test ./crux/... # from the workspace +cd src/crux && go test ./... # from inside the module ``` -Both ratchets live in `./internal/testutil` and run with that package. In -CI, a PR whose diff is entirely inside the ring gets the `crux-only` verdict -from `crux-scope.sh` and runs only the agent slices (item 5). Two -`pkg/agent` tests fail on macOS because the tmux socket path under +`./cmd/crux` is a host that builds against it, like `./cmd/hive`, and is +tested with the rest of the root module; `./crux/cmd/crux` is the module's +own host and builds standalone (`cd src/crux && GOWORK=off go build +./cmd/crux`; `make smoke` there exercises `crux serve` end to end). Both ratchets live in +`./internal/testutil` and run with that package. In CI, a PR whose diff is +entirely under `src/crux/` gets the `crux-only` verdict from +`crux-scope.sh` and runs only the agent slices (Stage 1 item 5). Two +`crux/agent` tests fail on macOS because the tmux socket path under `$TMPDIR` exceeds `sun_path`; they pass on Linux and in CI. ## Not doing @@ -357,7 +1129,7 @@ from `crux-scope.sh` and runs only the agent slices (item 5). Two - Repo split before Stage 1: the co-change tax lands before any second consumer exists and doubles v5/v6 top-up work. - Rewriting engines or the proxy. This is a boundary, not a rewrite. -- Shortening the PR gate by splitting: `pkg/agent` is a 429 s serial test +- Shortening the PR gate by splitting: `crux/agent` is a 429 s serial test monolith already sliced five ways; the gate is bounded by the slowest shard, so only path filters (Stage 1, item 5, shipped) change anything, and they work in-repo. @@ -379,9 +1151,12 @@ Comb, Hum, Osmia, Apis. ## References - RFC: [hivecommons/hive#13536](https://github.com/hivecommons/hive/issues/13536) + (stage 1, closed); stage 2/3 tracker + [hivecommons/hive#14138](https://github.com/hivecommons/hive/issues/14138) - Ratchets: `src/internal/testutil/crux_boundary_ratchet_test.go` (inbound), `src/internal/testutil/crux_outbound_ratchet_test.go` (outbound) - CI scope: `.github/scripts/crux-scope.sh` (#13541) - Prior extractions from Hive: pluk, promptargs, rationguard, hotshot, spektacular, dibs - Related: #13532 (control arm) +- Generic MITM tool policy: [crux-tool-policy.md](crux-tool-policy.md) (#14228) diff --git a/src/docs/design/external-workflow-admission.md b/src/docs/design/external-workflow-admission.md index c654ddd6a1..f2bd15d469 100644 --- a/src/docs/design/external-workflow-admission.md +++ b/src/docs/design/external-workflow-admission.md @@ -432,6 +432,30 @@ token, and the fixture's stats assert both are absent. | Hub restarts with a run in flight | The replacement adapter holds no keys; `Recover` issues the keyed start and the workbench answers `deduplicated` for its existing run. | None. | | Cancel ignored by the workbench | `stopped: false` stays on record; the run stays visibly running; late output is rejected once authority is no longer current. | Wait or retire the lease; never assume stopped. | +## Claude Managed Agents self-hosted host (#14124) + +The first hosted-execution prototype uses Claude Managed Agents with a +contributor-private `self_hosted` environment. The adapter lives in +`src/pkg/extwork/managedagents` and is linked only by +`extwork_claude_managed_agents`; the relay token is +`ext-exec/claude-managed-agents`. It is not a CLI backend and is never added +to `KNOWN_BACKENDS` or `CLIBackends`. + +| Contract point | Prototype behavior | +| --- | --- | +| Admission and idempotency | `POST /v1/sessions` carries the Hive execution key, request digest, bounded prompt/repository context, pinned agent version, toolset version, and the remaining daily budget. A durable local session map plus a bounded vendor session scan reconciles a crash before Hive records the session ID; a different digest is `ErrConflict`. | +| Incarnation | `GET /v1/environments/{id}` is authenticated before dispatch and on observation. The response must be `self_hosted`; environment ID/version form the pinned incarnation. | +| Observation | `GET /v1/sessions/{id}` maps `created`/`queued`, `running`, `rescheduling`, `idle`, and `terminated` to the extwork states. `budget_reached` and `requires_action` remain visible as waiting. | +| Cancellation | `POST /v1/sessions/{id}/events?beta=true` sends only `user.interrupt`; HTTP acknowledgement and observed stop remain separate facts. | +| Artifacts | Session output files are listed through `GET /v1/files?scope_id={session}` and streamed through `/v1/files/{id}/content`; extwork receipt digest/size/schema checks remain authoritative. | +| Egress | The configured network policy is `forced-egress` or `proxy-only`, and `github_write_path` must be `none` or `proxy`; every other value, including `github_repository`, `vaulted_secret`, `github_mcp`, and `direct_github_write`, is refused when the adapter is built for every stage. The prototype has no GitHub path, and nothing routes the hosted worker through the proxy, so the `POST /pulls` and `PUT /pulls/{n}/merge` denial tests remain open. | +| Metering | `span.model_request_end` events from `GET /v1/sessions/{id}/events?beta=true` are deduplicated by vendor event ID into `pkg/tokens` JSONL. `session.usage.list_cost`, active seconds, and web-search counts are persisted in `managed-agents-ledger.json`; a hard USD 25 UTC-day cap sets the session budget to remaining cents and interrupts a running session at the cap. | + +The worker receives only its self-hosted environment key and per-session work +secret. The Hive control-plane API key is resolved from `api_key_env` or a +`/data/secrets` `api_key_file` reference and is never included in the bounded +bundle or initial event. + ## Deferred items - Any publication authority, target-repository mutation, broker integration, diff --git a/src/docs/design/github-mention-triggers.md b/src/docs/design/github-mention-triggers.md index e10982889a..4f861be68e 100644 --- a/src/docs/design/github-mention-triggers.md +++ b/src/docs/design/github-mention-triggers.md @@ -72,9 +72,9 @@ The last row is the only inbound trigger in the system, and it is not GitHub. **The reply half.** `AgentCapabilities.Converse` is documented in exactly the words this feature needs — "An ADVISORY agent with Converse can reply on a thread it was mentioned in; it still cannot file, edit or relabel anything" -(`src/pkg/agent/capabilities.go:32`). It is enforced at the MITM proxy as an +(`src/crux/agent/capabilities.go:32`). It is enforced at the MITM proxy as an additional grant beside the mode ladder (`proxy.AllowedByModeCaps`, -`src/pkg/proxy/rules.go:152,170`), and the issue-request watcher already carries a +`src/crux/proxy/rules.go:160,189`), and the issue-request watcher already carries a `Kind: "comment"` request that posts a comment as the App bot (`src/pkg/github/issue_request_watcher.go:70`). Outbound comment text is canary-scanned and secret-scrubbed on the way out @@ -176,7 +176,7 @@ The header carries repo, item number, item kind (issue / PR), comment id and URL as structured context — the same shape the review swarm already hands its reviewers (`pkg/review`). The body is the human's text with the leading mention and any `ask ` selector stripped, after the ioscan pass below. -Delivery is `agent.Manager.SendKick` (`src/pkg/agent/manager_kick.go:110`), +Delivery is `agent.Manager.SendKick` (`src/crux/agent/manager_kick.go:110`), recorded in the kick history with `source=mention` so the dashboard's kick outcome classifier (#7421) and the agent card can say where the turn came from. @@ -385,14 +385,12 @@ the decision follows each one, with the code on `v6` that now carries it. - `src/pkg/linearagent/webhook.go` — the inbound mention model to mirror. - `src/pkg/linearagent/responder.go` — ack-first, resolve, kick, track. - `src/pkg/linearagent/oauth.go:56` — `app:mentionable`. -- `src/pkg/agent/capabilities.go:32` — `Converse`, documented for mentions. -- `src/pkg/proxy/rules.go:152,170` — where `Converse` is enforced. - -- `src/pkg/config/work_sources.go:116` — `linear.session_agent` resolution rule. -- `src/pkg/config/validate.go:379` — the removed declarative channel types (#5591). -- `src/pkg/config/review_bots.go:32` — `classification.review_bots`, the loop - list. - +- `src/crux/agent/capabilities.go:32` — `Converse`, documented for mentions. +- `src/crux/proxy/rules.go:160,189` — where `Converse` is enforced. +- `src/pkg/config/work_sources.go:119-123` — `linear.session_agent` resolution rule. +- `src/pkg/config/validate.go:702` — the removed declarative channel types (#5591). +- `src/pkg/config/review_bots.go:33` — `classification.review_bots`, the loop + list and the per-thread cap. - `src/pkg/github/review_request_watcher.go`, `review_threads.go` — the App-authored in-thread reply path and its attempt counter. - `src/pkg/github/issue_request_watcher.go:70` — `Kind: "comment"`. diff --git a/src/docs/design/knowledge-system.md b/src/docs/design/knowledge-system.md index 0861012c7e..ce22a5d98b 100644 --- a/src/docs/design/knowledge-system.md +++ b/src/docs/design/knowledge-system.md @@ -2,7 +2,7 @@ ## Context -Hive today targets teams at ACMM Level 3-4 — repos with CI, agents running 24/7. The goal is to make Hive useful starting from Level 1 (someone with an idea and no code), guiding them through the full developer journey to full automation. Along the way, a layered knowledge base (powered by llm-wiki) accumulates learnings, test patterns, and project-specific facts that compound over time — making agents smarter and automation safer at every level. +Hive today targets teams at ACMM Level 3-4 — repos with CI, agents running 24/7. The goal is to make Hive useful starting from Level 1 (someone with an idea and no code), guiding them through the full developer journey to autonomous operation. Along the way, a layered knowledge base (powered by llm-wiki) accumulates learnings, test patterns, and project-specific facts that compound over time — making agents smarter and automation safer at every level. This design is informed by contrasting MetaSwarm (dsifry/metaswarm) with Hive. MetaSwarm's best idea — a self-improving knowledge base with automated extraction and selective priming — is adopted here. Its weakest idea — using more LLMs to verify LLM output — is replaced by Hive's existing deterministic pipeline approach, with **testing** as the primary trust mechanism instead of adversarial LLM review. @@ -292,7 +292,7 @@ src/pkg/knowledge/ | `pkg/governor/` | Add test maturity level to governor decisions; adapt agent directives by ACMM level | | `pkg/scheduler/` | Add curator schedule (daily or on-merge trigger) | | `pkg/dashboard/` | Add `/api/knowledge/*` endpoints proxying to llm-wiki MCP; Knowledge tab in frontend | -| `pkg/agent/` | Include primed facts in agent work orders | +| `crux/agent/` | Include primed facts in agent work orders | ### Pipeline flow diff --git a/src/docs/design/standby-contributors.md b/src/docs/design/standby-contributors.md index bf2205fa0c..d6a426575e 100644 --- a/src/docs/design/standby-contributors.md +++ b/src/docs/design/standby-contributors.md @@ -80,7 +80,7 @@ phase map stays small. selection already used to build kicks. Per-lane queue depth is a count over work the scheduler already computes. - **The tier vocabulary.** `RotationConfig.AgentTiers` - (`src/pkg/crux/config/agent_config.go:681`) already spells T1/T2/T3 in `hive.yaml`. It + (`src/crux/config/agent_config.go:681`) already spells T1/T2/T3 in `hive.yaml`. It maps *agent names*, not models — so standby borrows the vocabulary and needs its own mapping (below). - **The relay's configuration report.** `auth_response` already carries @@ -103,7 +103,7 @@ phase map stays small. rule needs, persisted in the contributors directory alongside the cooldown and verdict ledgers. - **Hold gating.** `agentmode.CanCreatePRs` - (`src/pkg/agentmode/agentmode.go:67`) is the mode ceiling, and the + (`src/crux/agentmode/agentmode.go:67`) is the mode ceiling, and the `holdByLevel` path (`src/pkg/github/pr_request_watcher.go:443-460`) applies the `hold` label server-side from authoritative config, failing the request rather than silently shipping an unlabelled PR. @@ -276,7 +276,7 @@ hub: `standby` is a new block on `AgentConfig` (`src/pkg/config/agent_config.go:142`); the three `hub.standby_*` keys are new fields on `HubConfig`. Both landed in S2: -`AgentConfig.Standby` (`src/pkg/crux/config/agent_config.go:426`) and +`AgentConfig.Standby` (`src/crux/config/agent_config.go:426`) and `HubConfig.StandbyContributors` (`src/pkg/config/hub_config.go:243`) with its two neighbours. The types, defaults and validation are `pkg/config/standby.go`. @@ -383,7 +383,7 @@ stating plainly: - **Full lane** — donated PR is held anyway. Standby subtracts merge; it never adds it. - **Advisory or measured lane** — the lane has no authority to open PRs at all - (`CanCreatePRs`, `src/pkg/agentmode/agentmode.go:67`). Standby dispatch on + (`CanCreatePRs`, `src/crux/agentmode/agentmode.go:67`). Standby dispatch on such a lane is **refused**, with reason `mode_ceiling`. Donating cannot raise a lane's ceiling, so there is no PR to hold and nothing to dispatch. @@ -869,7 +869,7 @@ construction. The live-hive runbook above stays a human's job. - Contributor protocol: `src/pkg/dashboard/contribute_protocol.go`, `src/pkg/dashboard/contribute_ws.go`, `bin/contributor-relay.js`. - Mode ladder and hold gating: [`src/docs/acmm-policy-matrix.md`](../acmm-policy-matrix.md), - `src/pkg/agentmode/agentmode.go`, `src/pkg/github/pr_request_watcher.go`. + `src/crux/agentmode/agentmode.go`, `src/pkg/github/pr_request_watcher.go`. - CI surface: [`src/docs/backend-smoke.md`](../backend-smoke.md). ### Outcome reconciliation diff --git a/src/docs/design/tool-approval-desk.md b/src/docs/design/tool-approval-desk.md index d2efe6705a..49314582c8 100644 --- a/src/docs/design/tool-approval-desk.md +++ b/src/docs/design/tool-approval-desk.md @@ -61,7 +61,7 @@ verdict (`CeilingApplied`), and it shows up in the audit record. Pinned by | --- | --- | --- | | L0–L2 | `operator-approve` | Nothing side-effectful resolves without a human. | | L3–L5 | `security-scan` | May reach `auto-approve`, but only *via* a green scan. | -| L6+ | `auto-approve` | Full autonomy; the desk is an audit record, not a gate. | +| L6+ | `auto-approve` | Autonomous; the desk is an audit record, not a gate. | An unknown or negative level clamps to the most restrictive lane, so a mis-parsed or absent `acmm_level` can never widen authority @@ -154,10 +154,10 @@ audited policy changes — never as conservative shipped defaults. > `TestNoConservativeDowngradeAtHighAutonomy` states requirement 3 directly. **4. Operator-lane items at L6 are probable policy bugs.** An inbox that quietly -accumulates on a hive whose level says "full autonomy" is a misconfiguration -signal. `Inbox.PendingAtFullAutonomy` surfaces them, the API returns +accumulates on a hive whose level says "Autonomous" is a misconfiguration +signal. `Inbox.PendingAtAutonomousLevel` surfaces them, the API returns `policy_bug_count`, and the dashboard renders a warning banner rather than -letting them wait politely (`TestPendingAtFullAutonomyIsFlagged`). +letting them wait politely (`TestPendingAtAutonomousLevelIsFlagged`). Unchanged and deliberate: operator kill-switches and the `hold` label sit **above** the desk at every level. diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 48ce74d6d5..2597760beb 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -21,6 +21,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Work-source terminology](work-source-terminology.md) — glossary and audit notes for keeping user-facing copy neutral across GitHub, GitLab, Gitea, Linear, Jira, and future work sources. - [Release channels](release-channels.md) — `stable`/`candidate`/`edge` moving image tags, per-line channel ownership (`v5` → `candidate`/`latest`, `v6` → `edge`, `v4` → none), switching a hive to a channel, the version pill, and the timeline for when a hive follows `stable` after the tag moves. - [Digest-verifiable rollback](release-rollback.md) - the operator runbook for pinning a hive back to a prior immutable short-SHA build for all three images (`hive`, `hive-contributor`, `hive-hub`), stopping the hub automation that would undo it, and verifying by digest rather than by tag that the pin landed on the running spoke. +- [Outcome benchmarks](benchmarks/README.md) — reproducible mediated-vs-unmediated comparison (revert rate, change-failure, time-to-merge, CI at merged head, changes requested pre-merge, cost per merged PR) from stored review evidence, with the monthly reports, generated charts and the README outcomes block it keeps in sync. - [v5 GA readiness bar](v5-ga.md) — measurable release-train, migration, safety, and governance criteria that must be evidenced before v5 can be promoted beyond the active-development `edge` channel. - [v5 GA candidate week sequencing](v5-candidate-week.md) — dependency-ordered plan for the remaining human-gated GA-bar steps: candidate-SHA designation, candidate-pinned measurements, the two bound exercises, and the v4 freeze evaluation. - [Stable soak and promotion policy (v5 line)](stable-soak-policy.md) — the CI-enforced gate between `candidate` and `stable`: soak conditions, immutable short-SHA rollback tags, the emergency exception path (what it waives, and the post-hoc evidence that closes its follow-up issue), and the ledger of exceptions taken with their per-image rollback digests. @@ -35,6 +36,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Dashboard-triggered standalone upgrades](dashboard-standalone-upgrades.md) — the dashboard upgrade button for standalone (Compose/Podman-Quadlet) hives: the `HIVE_DEPLOYMENT_RUNTIME`/`HIVE_DEPLOYMENT_PODMAN_MODE` runtime contract, the closed host-side helper (`hive-dashboard-upgrade-helper.sh`), and why an unproven runtime deliberately hides the button. - [Work sources](work-sources.md) — GitHub Issues, GitHub Projects, Linear, Jira Cloud, Jira Data Center / Server, run-stage, and Wavefront work enumeration. - [Work source providers](integrations/work-source-providers.md) — implement and contribute a provider that maps source-native work items onto `pkg/worksource.WorkSource`. +- [Forges: GitLab, Gitea and Forgejo](integrations/forges.md) — `project.forge`, the `gitlab:`/`gitea:` keys, which `pkg/forge` operations each adapter supports, which governor writes already use them, and how the dashboard shows the forge. - [Hive Labels and Control Signals](labels-and-control-signals.md) — authoritative operator reference for labels, holds, approvals, contributor skip labels, planning labels, and dashboard bands. The old [Hive issue labels](issue-labels.md) page redirects here. - [Central stage engine implementation plan](stage-engine-plan.md) — the ADR-0025 rollout plan for `pkg/stageengine`: the PR sequence, shadow→enforce gates, and the facts/deriver/writer seams the issue and PR sweeps share. - [GitHub webhooks: cadence, patch and reconcile](webhooks.md) — the three writers of the GitHub snapshot (reconcile / webhook patch / write-through), the accuracy guarantees of #13373, `eval_interval_s` vs `eval_interval_webhook_s` and the other cadence keys with defaults, and the automatic fallback to the polling cadence when webhooks degrade (`/api/status.webhooks.mode`). @@ -96,6 +98,8 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und [the design record](design/tui.md) for the reasoning behind it. - [`bd` beads CLI](beads-cli.md) — work-ledger and knowledge command reference for operators and contributors. - [`crux` CLI](crux-cli.md) — run, list, tail, kick and stop agents through `crux.Runtime` with no governor or dashboard; definition formats, the control socket served by `crux run`, and why the runtime being in-process shapes it. +- [`crux serve`: the runtime over a local API](crux-serve.md) — the Stage 3 process boundary: HTTP+JSON on a unix socket (TCP with a bearer token), the `/v1` routes and event stream, fail-closed rules, and how to drive it with curl, the `crux` CLI's `--server` mode and the Go client in `crux/api`. +- [Publishing Crux](crux-publish.md) — how `hivecommons/crux` is produced from `src/crux` (plain copy, module path rewritten, fast-forward commits), what is preserved there, the `CRUX_SYNC_TOKEN` credential, manual sync and how a `crux/vX.Y.Z` tag becomes a release. - [Backup and restore](backup-restore.md) — `hive-backup`, Kubernetes CronJob, spoke backup scope, and setting the backup encryption key from Governor Config (hosted flow). Host-level backup, restore, and `docker compose down -v` are given per runtime: Docker Compose, and Podman/Quadlet with the executed backup → wipe → restore cycle in both root modes, the rootless mapped-UID trap that makes a host-shell `tar` skip the GitHub App key, and the Docker→Podman migration (the two volume stores are never shared). Also covers restoring a `pkg/spokebackup` archive: the archive-path → container-path mapping, and the executed `hive-backup restore` path that applies it — including the identity guard that refuses to splice one hive's config and GitHub App keys onto another's ([#6529](https://github.com/hivecommons/hive/issues/6529)). - [Hub disaster recovery](../../docs/HUB_DISASTER_RECOVERY.md) — the hub-level runbook that goes beyond per-hive backup: hub backup and key escrow, spoke fleet recovery, Slack blast, and the full rebuild-from-zero procedure after a catastrophic loss. - [Deployment helper scripts](deployment-scripts.md) — the all-in-one LXC setup, Proxmox LXC, and blue-green Compose helpers. All are Docker-only; the page states each script's runtime scope and where a Podman operator should go instead. @@ -173,6 +177,9 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Podman preflight: subordinate IDs, graphroot, and networking](podman-preflight-ids.md) — read-only diagnostics for rootless subordinate UID/GID delegation, unsupported (NFS and other distributed) container storage, and the rootless network backend/helper, with remediation that never edits `/etc/subuid` or `/etc/subgid`. - [CLI backend setup](../../docs/backend-setup.md) — setup notes for Claude, Copilot, Goose, Bob, Pi, Codex, and Aider. - [Backend support tiers and acceptance bar](backend-support-tiers.md) — the policy for adding a CLI backend: the three tiers (core/headless-pod, supported/confined, experimental/unconfined), the criteria each requires with the exact file or test a PR must touch (list parity, declared posture, flag-honoring proof, credential detection, pinned install, refusal gate, confinement floor, headless entry point, unattended-credential verification, allowlist touchpoints), and how a backend moves between tiers. +- [Gap analysis: Claude Managed Agents](backend-gap-claude-managed-agents.md) — how Anthropic's hosted agent runtime maps onto the backend acceptance bar: task prompt and repo delivery, which session setups give the hosted sandbox a GitHub write path Hive's proxy cannot see (and so cap it at advisory mode), the usage and cost fields available for metering, and a recommendation for a hosted-execution tier. +- [Gap analysis: OpenAI Agents API](backend-gap-openai-agents-api.md) — how OpenAI's Responses API, hosted tools and Agents SDK map onto the backend acceptance bar: prompt and repo delivery, which hosted-container setups egress GitHub directly (and so cap the backend at advisory mode), the usage fields available for metering, and a recommendation for a hosted-execution tier. +- [Omnigent CLI contract and policy composition](backend-omnigent-cli-contract.md) — the contract of the Omnigent CLI against the acceptance bar: what Hive actually launches (`omnigent run --server local`, and why both words matter), the six contract facts (upstream's non-interactive `run -p`, not wired by Hive; how the prompt is passed, exit codes on success/failure/denial, where the transcript lands, model and cost reporting, the tool-denial list), which safety flags it honours, `detect_cli` credential detection, the confinement posture and refusal gate that place it at T3 (experimental/unconfined), and how its policy layer composes with Hive's. - [Inference backends](../../docs/inference-backends.md) — vLLM, llm-d, LiteLLM, and Model Gateway troubleshooting. - [apiproxy](apiproxy.md) — Anthropic-compatible proxy logging and deployment notes. - [Outreach anti-spam ruleset](../../docs/outreach-antispam.md) — the deduplication and anti-spam rules the outreach agent operates under across awesome lists, project issues, directories, and community threads. @@ -184,6 +191,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Automatic repo ACMM onboarding reconciler](design/repo-acmm-onboarding.md) — proposed design for per-repo ACMM targets, gap-to-work reconciliation, human hand-off, waivers, and anti-gaming controls (RFC #6235). - [Hive federation design](../../docs/federation-design.md) — the multi-hive registry: live `/api/hives` endpoints, project onboarding, contributor flow across hubs, and what remains future design work. - [Public roadmap](roadmap.md) — the v5 direction as Now / Next / Later, with the tracking issue behind each item. Directional rather than a promise, maintained by pull request; check the date in its header before relying on the ordering. +- [Integrations](integrations.md) — one page of logos and status (Supported / Opt-in / Planned) for agent backends, work sources, forges, sandboxes and upstream projects. - [Landscape and positioning](landscape.md) — how Hive's operations-plane design compares to nearby agentic orchestration tools, with public references per project. Explicitly time-sensitive; check the conducted date in its header before quoting product details. - [CNCF sandbox onboarding status](cncf-sandbox-onboarding.md) — repo-local map of the onboarding tracker and repository evidence (LICENSE, governance, security, OWNERS, DCO). - [CNCF reference architecture](cncf-reference-architecture.md) — CNCF submission/reference template. diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 3b9781096c..4e0db1da47 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -17,7 +17,7 @@ This reference is compiled by hand from the Go source under `src/`, the deployme | `HIVE_DASHBOARD_COOKIE` | No | none | **Client-side only** - read by `hivectl tui`, never by the server. Cookie header value (e.g. `hive_session=...`) carrying a per-user session, for hives that do not accept the shared token: hub-hosted ones, and spokes with an `authorized_users` allowlist. See [hivectl.md, Credentials](hivectl.md#credentials). | | `HIVE_DASHBOARD_BIND` | No | `127.0.0.1` for the legacy `dashboard/server.js` | Legacy Node dashboard listen address. Leave unset for loopback-only binding; set `HIVE_DASHBOARD_BIND=0.0.0.0` only when an authenticated reverse proxy or equivalent network control protects the unauthenticated legacy control endpoints. | | `HIVE_AUTHORIZED_USERS` | No | none | Comma-separated direct-route dashboard allowlist, with optional `user:role` entries. Used when `dashboard.authorized_users` is empty. | -| `HIVE_SELF_AUTHORIZATION_HOLD` | No | `github.self_authorization_hold` / `project.repo_policies[].self_authorization_hold`; when all are unset, default `true` through ACMM L5 and `false` at ACMM L6 Fully Autonomous | Process-level override for the #5117 self-authorization hold. Set `false` to let this hive skip and release self-authorization holds for every repo while preserving unrelated human holds; set `true` to keep the hold enabled even at L6. | +| `HIVE_SELF_AUTHORIZATION_HOLD` | No | `github.self_authorization_hold` / `project.repo_policies[].self_authorization_hold`; when all are unset, default `true` through ACMM L5 and `false` at ACMM L6 (Autonomous) | Process-level override for the #5117 self-authorization hold. Set `false` to let this hive skip and release self-authorization holds for every repo while preserving unrelated human holds; set `true` to keep the hold enabled even at L6. | | `HIVE_SELF_EXPERIMENTS_OPT_IN` | No | unset (off) | Opts this hive in to receiving hub-delivered Hive-self feature-flag assignments over heartbeat (`pkg/hub/spoke/self_experiment_flags.go`). Only the exact string `true` enables it; `1`, `yes`, `on` and any other value leave it off. Consent is local: the hub cannot opt a spoke in. Unsetting it withdraws the flags on the next heartbeat. Separate from the Experiment Lab/governor settings. | | `HIVE_REPORTER_TRUST_HOLD` | No | `github.reporter_trust_hold` / `project.repo_policies[].reporter_trust_hold`; when all are unset, follows `project.issue_filter.reporter_trust.enabled` | Process-level override for the #9665 reporter-trust hold (a PR whose rationale traces to an issue from an untrusted reporter is held at every ACMM level). When set, the dashboard toggle and per-repo overrides are locked and say so. | | `HIVE_REPO` | No | none | Bootstrap shortcut in `owner/repo` form; fills `project.org`, `project.repos`, and `project.primary_repo` if missing. | @@ -155,6 +155,7 @@ new value at the same time. | `HIVE_OUTPUT_LOG_DIR` | No | `/data/.hive/logs` | Directory (`0700`, files `0600`) where the hive keeps the `proxy`, `hive-output`, `litellm` and `requests` log sources read by the admin MCP `log_read` tool ([#11944](https://github.com/hivecommons/hive/issues/11944)); each keeps at most 5 files of 10 MiB and nothing older than 7 days. The `starts` and `crash` sources ([#11945](https://github.com/hivecommons/hive/issues/11945)) live here too: `starts.log` (newest 200 starts), `start-current.json` and at most 10 `crash-.log` files of 1 MiB. See [design/admin-mcp.md](design/admin-mcp.md#output-and-request-log-sources). | | `HIVE_PROXY_EGRESS_MARK` | No | `0x1112` | Packet mark exempted from the MITM egress redirect. | | `HIVE_PROXY_ADVISORY_OK` | No | `false` | Allows the spoke to start when the forced-proxy egress redirect cannot be installed (no `CAP_NET_ADMIN`/iptables). Enforcement becomes advisory-only — agents can bypass the proxy. Also gates whether the Go proxy trusts a self-asserted `Proxy-Authorization` header as agent identity when its UID map is unavailable (N7, #3841) — off by default, an unidentified caller is treated as `ADVISORY` (writes blocked) rather than whatever name it claims. See [security-model.md](security-model.md#5-network-and-credentials). | +| `HIVE_PROXY_ALLOW_PRIVATE_FORWARD` | No | `false` | Lifts the plain-HTTP forward destination guard ([#14082](https://github.com/hivecommons/hive/issues/14082)). Off by default: a non-CONNECT request through the proxy is dialled only when the resolved address is globally routable, so an agent cannot use the proxy to reach loopback services beside it, the link-local cloud metadata endpoint (`169.254.169.254`), RFC 1918/CGNAT cluster addresses or multicast, and the refusal is a `403` naming the class. Only the exact string `true` turns it on, for deployments that genuinely egress to a private network over plain HTTP (a LAN mirror on a native install). Loopback refusals stay at `DEBUG` and are not counted as violations, because agents are already handed `NO_PROXY` for loopback (#14076). | | `HIVE_PROXY_INJECT_GH_AUTH` | No | unset (off; opt-in only on every hive, hosted or not) | Proxy-side GitHub credential injection ([#1861](https://github.com/hivecommons/hive/issues/1861)). Opt-in only: on exclusively when set to `true`; unset, `false` and any other value are off, and the hub renders no value onto newly provisioned spokes (a brief default-on for hosted App spokes from #9597/#9625 was reverted, [#9586](https://github.com/hivecommons/hive/issues/9586)). `true`: the hive keeps each agent's tier-scoped App token in memory, the MITM proxy strips any agent-supplied `Authorization` on GitHub hosts and attaches the UID-identified agent's real token, and the agent's readable token cache (and the `GH_TOKEN`/`GITHUB_TOKEN` derived from it) holds only the inert placeholder `hive-proxy-injected-`. `false` is the explicit opt-out (same as unset, but recorded on the pod spec). The resolved state is logged once at boot as `proxy GitHub auth injection: on|off ()` - unset reads `off (opt-in: set HIVE_PROXY_INJECT_GH_AUTH=true)` - and shown in the dashboard Security tab. Under injection the Copilot CLI's `/copilot_internal/` auth exchange keeps the agent's own credential. **`true` together with `HIVE_PROXY_ADVISORY_OK=true` makes the spoke refuse to start** (exit code 19): advisory mode lets a caller self-assert another agent's identity and receive its injected token. Any other value (`1`, `TRUE`, `yes`, ...) is treated as off, as before, but is logged at ERROR on boot and shown in the dashboard Security tab's coherence warnings, because it leaves the real token in the agent's cache. Requires GitHub App auth; on a PAT-only hive there is nothing to inject. See [security-model.md](security-model.md#5-network-and-credentials). | | `HIVE_DEPLOYMENT_RUNTIME` | No | none (config `deployment.runtime` fallback; hub-registered Kubernetes hives are auto-detected) | Declares the deployment runtime (`kubernetes`, `podman-quadlet`, or `docker-compose`) so the dashboard can offer standalone upgrades. Unset/unrecognized means `unknown` and the upgrade button stays hidden. See [dashboard-standalone-upgrades.md](dashboard-standalone-upgrades.md). | | `HIVE_DEPLOYMENT_PODMAN_MODE` | Only when runtime is `podman-quadlet` | none (config `deployment.podman_mode` fallback) | `rootless` (`systemctl --user`) or `rootful` (system manager). Without an explicit mode the Podman/Quadlet upgrade path stays disabled. | @@ -202,16 +203,16 @@ new value at the same time. | `JEV_API_KEY` | No | none (falls back to the connected OpenRouter gateway key) | Key the hive uses for Jev typed decisions on behalf of agents with `jev_mode: assist` (see [agent-configuration.md](agent-configuration.md#jev-typed-decisions-jev_mode)). The variable name is configurable via `jev.api_key_env`. Read only by the hive; never exported to agents. **Secret.** | | `BD_DIR` | No | current directory | `bd` beads CLI data directory. | | `BD_DASHBOARD_URL` | No | none | Dashboard URL used by `bd kb` integration. | -| `OPENAI_API_KEY` | No | none | OpenAI-compatible API key consulted by agent credential probing (`pkg/agent/authprobe.go`) for Codex API-key mode, including `CODEX_HOME/auth.json` entries written under the same key. | +| `OPENAI_API_KEY` | No | none | OpenAI-compatible API key consulted by agent credential probing (`crux/agent/authprobe.go`) for Codex API-key mode, including `CODEX_HOME/auth.json` entries written under the same key. | | `OPENAI_HOST` | No | Goose CLI default | OpenAI-compatible API host consumed by Goose and forwarded into contributor containers. | | `OPENROUTER_API_KEY` | No | none | OpenRouter API key, forwarded into contributor containers so a Goose backend configured with `GOOSE_PROVIDER=openrouter` can authenticate. Also one of the per-model credential names the `pi` backend resolves (`bin/pi-backend.js`). | | `OPENAI_BASE_PATH` | No | Goose CLI default | OpenAI-compatible API request path consumed by Goose and forwarded into contributor containers. | -| `CODEX_API_KEY` | No | none | API key read by `pkg/agent/authprobe.go` for the Codex CLI backend; either this or `OPENAI_API_KEY` makes Codex API-key mode count as configured. | +| `CODEX_API_KEY` | No | none | API key read by `crux/agent/authprobe.go` for the Codex CLI backend; either this or `OPENAI_API_KEY` makes Codex API-key mode count as configured. | | `HIVE_AGENT_TOKEN_REFRESH_INTERVAL` | No | `40m` | Go duration overriding the per-agent token refresh interval. Invalid or non-positive values fall back to the default. | | `HIVE_CREDENTIAL_WATCHDOG_INTERVAL` | No | `5m` | Go duration overriding how often the credential watchdog verifies each in-use backend credential file. `0` does NOT disable the watchdog — disabling is intentionally not offered. | | `HIVE_PROVIDER_ERROR_BACKOFF_BASE` | No | `2m` | Go duration for the first inference-provider error backoff before another kick is allowed. Invalid or non-positive values fall back to the default. | | `HIVE_PROVIDER_ERROR_BACKOFF_MAX` | No | `30m` | Go duration cap for exponential inference-provider error backoff. Invalid or non-positive values fall back to the default. | -| `HIVE_START_FAILURE_BLOCK_THRESHOLD` | No | `3` | Consecutive identical start failures (positive integer) before an agent is blocked from automatic relaunch (`pkg/agent/start_failure.go`). Backoff is applied from the first failure regardless; invalid values fall back to the default. | +| `HIVE_START_FAILURE_BLOCK_THRESHOLD` | No | `3` | Consecutive identical start failures (positive integer) before an agent is blocked from automatic relaunch (`crux/agent/start_failure.go`). Backoff is applied from the first failure regardless; invalid values fall back to the default. | | `HIVE_START_FAILURE_BACKOFF_LADDER` | No | `1m,5m,15m,30m` | Comma-separated positive Go durations pacing the **automatic** relaunch loop after start failures, indexed by consecutive-failure count and capped at the last entry. Any invalid entry discards the whole override. Explicit relaunches (a saved key, the dashboard restart button) clear the backoff outright. | | `HIVE_COPILOT_SESSION_REFRESH_INTERVAL` | No | `10m` | Go duration overriding the Copilot session refresh interval. | | `HIVE_COPILOT_SESSION_REFRESH_START_DELAY` | No | `30s` | Go duration overriding the delay before the first Copilot session refresh. | @@ -250,7 +251,7 @@ Inside an **agent** session (set by the hive, never by the operator): ISSUES_ONL Everything in this section is **set by the hive, never by the operator** - the same convention as the Linear note above. The source of truth is -`agentEnvPairs` in `src/pkg/agent/manager.go` (the Linear pairs are injected +`agentEnvPairs` in `src/crux/agent/manager.go` (the Linear pairs are injected alongside it by the same manager); this table is the contract that agent policies, custom agent definitions, and helper scripts may rely on. Variables marked **secret** are delivered via `tmux set-environment` only and @@ -267,7 +268,8 @@ Always set: | `HIVE_ACMM_LEVEL` | The project's ACMM level as a decimal integer. | | `HIVE_AGENT_MODE` | The agent's effective operating mode (from `tools.mode` when set, otherwise the resolved mode). | | `HIVE_EXPLAIN_MODE` | The **resolved** explain mode (`off`, `brief`, `full`) after hive-wide default inheritance - always exported, `off` included, so scripts can branch on it without re-deriving precedence. | -| `HTTP_PROXY`, `HTTPS_PROXY` | The local hive proxy (`http://127.0.0.1:`) all agent HTTP(S) traffic must traverse. | +| `HTTP_PROXY`, `HTTPS_PROXY` | The local hive proxy (`http://127.0.0.1:`) all non-loopback agent HTTP(S) traffic must traverse. | +| `NO_PROXY` | `127.0.0.1,localhost,::1` — loopback never traverses the hive proxy, on every backend. | | `HIVE_PROXY_AGENT` | The agent's own name, so tooling (e.g. `hive-panes`) can identify and skip the calling agent. | | `GIT_TERMINAL_PROMPT` | `0` - git never prompts for credentials. | | `NODE_EXTRA_CA_CERTS`, `GIT_SSL_CAINFO` | Path of the proxy CA certificate. `SSL_CERT_FILE` is deliberately **not** set (it breaks Copilot API TLS). | @@ -280,7 +282,7 @@ Set conditionally: | `HIVE_SHA` | Always, unless the build commit is unknown | The hive's own `HIVE_SHA` when set; otherwise the binary's short build commit. | | `HIVE_REPO`, `HIVE_REPOS` | When the project has an org and at least one repo | `org/primary-repo`, and the full comma-separated `org/repo` list. Policy templates target `gh issue create --repo "$HIVE_REPO"`. | | `GH_HOST` | GHE spokes with a configured forge host | Forge hostname for the `gh` CLI; the gh wrapper pairs it with `GH_ENTERPRISE_TOKEN`. | -| `ANTHROPIC_BASE_URL`, `ANTHROPIC_API_KEY`, `NO_PROXY`, `CLAUDE_CODE_MAX_OUTPUT_TOKENS`, `DISABLE_TELEMETRY`, `DISABLE_ERROR_REPORTING`, `CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC`, `CLAUDE_CODE_AUTO_MODE_SERVER=0` | Inference-routed backends only | Local inference-translate endpoint, a **synthetic** per-agent key (`sk-hive-` - not a real credential), loopback proxy bypass, an output-token cap every fronted model accepts, telemetry switched off at the source, and auto mode's server-side classifier check turned off (the OpenAI-format gateway behind the translator cannot return the `safeguard_results` verdict, and without the variable Claude Code stops on an interactive "this session isn't eligible" notice no unattended pane can answer). | +| `ANTHROPIC_BASE_URL`, `ANTHROPIC_API_KEY`, `CLAUDE_CODE_MAX_OUTPUT_TOKENS`, `DISABLE_TELEMETRY`, `DISABLE_ERROR_REPORTING`, `CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC`, `CLAUDE_CODE_AUTO_MODE_SERVER=0` | Inference-routed backends only | Local inference-translate endpoint, a **synthetic** per-agent key (`sk-hive-` - not a real credential), an output-token cap every fronted model accepts, telemetry switched off at the source, and auto mode's server-side classifier check turned off (the OpenAI-format gateway behind the translator cannot return the `safeguard_results` verdict, and without the variable Claude Code stops on an interactive "this session isn't eligible" notice no unattended pane can answer). | | `COPILOT_GITHUB_TOKEN` | When the hive holds a Copilot auth token | Copilot OAuth token (authenticates the AI model, not GitHub writes). **Secret.** | | `GITHUB_TOKEN` | Only when App-authored PRs are enabled (`AppAuthoredPRs`), an App is configured, and the agent's mode can push | The per-agent tier-scoped App installation token, so the built-in GitHub MCP server writes as the App bot. Advisory agents have `GITHUB_TOKEN` deliberately stripped. **Secret.** | | `LINEAR_ACCESS_TOKEN` / `LINEAR_API_KEY` | ISSUES_ONLY+ agents on Linear-connected hives | See the [Linear agent integration](#linear-agent-integration) note above. | diff --git a/src/docs/fleet-health.md b/src/docs/fleet-health.md index 4db85e7044..ecc17d9532 100644 --- a/src/docs/fleet-health.md +++ b/src/docs/fleet-health.md @@ -322,7 +322,7 @@ dashboard said so until a human happened to open a terminal. Each agent tracks its own `backend_auth` state, derived from the same provider-error classifier the dashboard already uses -(`classifyProviderError` in `src/pkg/agent/pane_classify.go`): +(`classifyProviderError` in `src/crux/agent/pane_classify.go`): | Status | Meaning | |---|---| @@ -661,7 +661,7 @@ ACMM banding) and `src/pkg/hub/remediation.go` (detector layering and the signature → action map), from fields the spoke reports in its heartbeat (`src/pkg/hub/heartbeat.go`). The spoke-side detectors live in `src/pkg/tokens/error_streak.go` (error streaks, from chat recordings), -`src/pkg/agent/consent_wedge.go` (consent-screen restart tracking), and +`src/crux/agent/consent_wedge.go` (consent-screen restart tracking), and `src/pkg/governor/no_cadence.go` (cadence-less agents). The `/fleet` rendering — dot, WHY chip, and hint line — is in `src/pkg/hub/static/my-hives.html`. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index c0746156e9..692b606cf1 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -140,14 +140,29 @@ title is `[selfreport] : `. | `advisory_post_failed` | the advisory digest could not be posted (rate-limit skips excluded) | medium | | `agent_stall` | ≥3 consecutive "continue" nudges sent to one agent after a kick without the pane moving; a credited stall recovery (the pane moves before the next kick) clears the streak. Message carries the nudge count, last kick time and frozen pane hash; diagnostics carry agent, lane, backend, counters and a scrubbed 15-line pane tail (#13470, was the hive-code-defect "agent stall") | medium | | `agent_crash_loop` | the watchdog latches an agent as crash-looping and pauses it (acting mode only — observe mode would re-count every sweep); the healthy-window reset clears it. Diagnostics carry the classifier detail, failure count/cap, backoff ladder and healthy-reset window (#13470, was "agent crash loop") | high | -| `ci_poll_nudge` | the harness sent an agent the stop-polling-CI nudge (at most one per kick); a self-report fires only after the agent polled CI ≥2 times since the kick — a single nudged poll does not trigger a report. One per 24h window; message carries polls since kick / time since kick / nudges since boot, diagnostics the scrubbed pane tail (#13470, was "CI-poll nudge") | low | +| `ci_poll_nudge` | the harness sent a non-monitoring agent the stop-polling-CI nudge (at most one per kick); reviewer, CI-maintainer and quality lanes are exempt because reading CI status across PRs and repositories is their assigned work. Only a command line run more than once since the kick counts as polling (#14016): a sweep that reads each of several PRs' checks once does not cross the count threshold. A self-report fires only after an eligible agent ran ≥2 such repeated polls since the kick — a single nudged poll or a sweep does not trigger a report. One per 24h window; message carries polls since kick / time since kick / nudges since boot, diagnostics the scrubbed pane tail plus the repeated CI-status command lines with their counts (#14289; #13470, was "CI-poll nudge") | low | | `issue_request_stuck` | an issue/comment/label request file in the relay queue (`issue-requests/`) has failed ≥3 attempts **or** has been retrying for ≥30 min. Key is the request file name; message carries kind, attempts and age, diagnostics the last error (scrubbed). Clears when the request completes or is quarantined (`.failed`/`.bad`) | medium | | `effect_ledger_op_stuck` | the mutation boundary refused a retry because the journaled external effect (issue create, PR open, …) has been `Unknown`/`Planned` for ≥30 min — the caller keeps replaying the same logical op and it never resolves. Key is the op's logical ID; message carries kind, status and age, diagnostics the attempt count, repo and claim target. Clears when the op is applied, replayed as already-applied, or reconciled | medium | -| `agent_pane_timeout` | the agent-readiness wait timed out and the harness dumped the tmux pane (`prompt timeout — dumping pane` / `pane content at timeout`, `pkg/agent/manager.go`; also `startup kick dropped` / `cli never reached input prompt`; `busy_visibility.go`: `kick undeliverable`, `pkg/agent/manager_kick.go`). Log-fed: same ≥5-in-10-min repeat threshold and 20-min idle clear as the catch-all below; pane excerpts are scrubbed and bounded before storage | low | +| `agent_dir_setup_failed` | the harness failed to set up the per-agent sandbox directory structure — create XDG directories, restore or tighten directory permissions on interactive/inference home directories. Log-fed: keywords include `failed to create per-agent xdg dir`, `failed to restore per-agent dir mode`, `failed to restore interactive home mode`, `failed to tighten per-agent dir mode`, `failed to tighten interactive home mode`, `failed to restore inference home mode`, `failed to tighten inference home mode` (`crux/agent/manager.go`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `agent_pane_timeout` | the agent-readiness wait timed out and the harness dumped the tmux pane (`prompt timeout — dumping pane` / `pane content at timeout`, `crux/agent/manager.go`), and the startup kick dropped because the CLI never became ready or reached its input prompt (`startup kick dropped: …`, `crux/agent/manager_kick.go`, the same event as the pane dump). Log-fed: same ≥5-in-10-min repeat threshold and 20-min idle clear as the catch-all below; pane excerpts are scrubbed and bounded before storage | low | +| `agent_login_required` | the login scanner saw the agent CLI asking for `/login` (`login required detected`, `pkg/loginscan`) — the agent's auth lapsed and it is paused until an operator re-authenticates. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `pr_request_head_unpushed` | the pr-request watcher found the PR head branch missing on the remote (`head branch is not on the remote`) — the agent's push did not authenticate or never ran. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `bead_fact_ingest_failed` | the knowledge system failed to ingest a bead fact from an agent (`failed to ingest bead fact`, `pkg/knowledge`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `feedback_issue_refresh_failed` | the dashboard's feedback-issue watcher failed to refresh or update a feedback issue (`feedback: issue refresh failed`, `pkg/dashboard`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `github_app_repo_not_covered` | the GitHub App recheck (`state=repo-not-covered`) found the installation does not include every configured repo — an operator configuration condition (tick the repos in the App's repository access), not a hive defect. Log-fed: same ≥5-in-10-min repeat gate and 20-minute idle clear as `agent_pane_timeout` | low | +| `github_app_not_installed` | a per-agent scoped-token mint was attempted while the GitHub App has no installation ID (`installation_id` 0): the App is not installed on the hive's org, or discovery never found it. An operator configuration condition — install the App or set `github.installation_id` — not a hive defect. Key is the agent name; reported on the first observation and cleared by the next successful mint. The runtime logs the matching WARN once per agent instead of every supervisor tick (#14186) | medium | +| `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`) — typically no model endpoint resolved — so untrusted text is withheld. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | +| `github_fetch_transient` | a non-critical GitHub fetch failed and the feature degrades gracefully (`channel commit date: fetch failed — the row will render without a timestamp`, `pkg/hub/channel_commit_date.go`) — typically a rate limit or transient API error, not a hive defect. Reported only when persistent: log-fed, same ≥5-in-10-min repeat threshold and idle clear as `agent_pane_timeout`. The sibling `SHA poll: commit message fetch non-200` has its own class, `sha_poll_commit_message_unavailable` | low | +| `label_provisioning_failed` | provisioning the issue stage labels on a repo failed (`issue stage labels: provisioning failed`, `pkg/github/issue_stage_labels.go`) — typically the App lacking label permission on that repo or a transient API error; diagnostics carry the repo and scrubbed error. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | +| `agent_cli_restart_loop` | the agent CLI crashed (bare shell) or sat on a consent screen and the harness restarted it before the kick (`agent CLI crashed or stuck on consent screen, restarting before kick`, `crux/agent/manager_kick.go`, `kick_async.go`), repeated ≥5 times in 10 min — a restart loop, not transient noise. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `proxy_connect_dial_failed` | the hive proxy could not dial the upstream host of a CONNECT request (`proxy: CONNECT dial failed`, `crux/proxy/github_proxy.go`) — typically DNS or network unreachability of an agent-requested host, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `proxy_tls_handshake_failed` | the hive proxy's TLS handshake with an upstream host failed (`proxy client tls handshake failed`, `transparent proxy tls handshake failed`, `copilot sniff: client tls handshake failed`, `crux/proxy/github_proxy.go`) — typically a certificate validation failure or encryption mismatch on an agent-requested host, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `review_evidence_write_failed` | the github-client failed to write a review evidence bundle to persistent storage (`review evidence: could not write bundle`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `selfreport_cleared_comment_failed` | the fleet report publisher failed to post the "condition cleared" comment on a self-report issue (`fleet report: cleared comment failed`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `inference_upstream_error` | a failed or non-2xx response from the configured inference backend (LiteLLM, vLLM/llm-d) on the Anthropic-to-OpenAI translator or MITM reroute path (`inference upstream error` / `inference upstream failed`, `crux/proxy/github_proxy.go`), repeated ≥5 times in 10 min — covers transport failures and statuses other than the dedicated 401/403 auth signal. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | -| `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`) — typically no model endpoint resolved — so untrusted text is withheld. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | -| `agent_github_api_cap_reached` | an agent exhausted its per-agent GitHub API hourly budget (`agent GitHub API hourly cap reached`, `pkg/proxy/agent_github_budget.go`; fields `agent`, `count`, `cap`, `retry_after`). The proxy throttles further calls until the window resets, so this reports sustained polling by one agent rather than an unknown error. Log-fed | medium | +| `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (typically a 403 rate limit or token scope; `SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller backs off per SHA and keeps serving the SHA without a message. The WARN itself is deduped to once per hour per (SHA, status) (`warnGitHubFetchOnce`), so one stuck commit logs one line and never trips this class; it is reported only when ≥5 distinct SHA/status pairs fail within 10 min (a broad outage such as an exhausted rate limit or a revoked token). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | +| `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `agent_github_api_cap_reached` | an agent exhausted its per-agent GitHub API hourly budget (`agent GitHub API hourly cap reached`, `crux/proxy/agent_github_budget.go`; fields `agent`, `count`, `cap`, `retry_after`). The proxy throttles further calls until the window resets, so this reports sustained polling by one agent rather than an unknown error. Log-fed | medium | | `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low | The generic feed also skips the exact `review pending expired` message: it @@ -167,7 +182,7 @@ a burst across unrelated templates is not a repeated runtime defect log, already deduped per path per process. It also skips `copilot CLI substituted the configured model` -(`logCopilotModelFallback`, `src/pkg/agent/manager_poll.go`): each copilot +(`logCopilotModelFallback`, `src/crux/agent/manager_poll.go`): each copilot agent warns once per (requested, running) model tuple (hivecommons/hive#13659) and shows the substitution on its dashboard `LastError`, so a hive whose copilot agents share one model id the account's Copilot CLI rejects emits one @@ -267,9 +282,14 @@ with none. Observation issues additionally carry `hive-observation` and ## Deduplication, comments, and recovery -Every report gets a deterministic **fingerprint**: a SHA-256 over the error -class, component, version, and (for shortfalls) the criterion key. The same -fault on the same version always produces the same fingerprint. +Every report gets a deterministic **fingerprint**. A `runtime-error` +fingerprint is a SHA-256 over the error class and the normalised message +(numbers, hashes, paths and quoted strings collapsed) — it names the defect, +not the build, so the same fault on every version and every hive produces +the same fingerprint and a close by a maintainer keeps suppressing it across +the daily channel rollout (#14256 was the case where a per-version +fingerprint produced one new issue per build). An ACMM shortfall +fingerprint covers the class, component, version and criterion key. Before filing, the hive searches upstream for an existing issue carrying that fingerprint: @@ -286,8 +306,12 @@ For `runtime-error` reports the hive goes further and **joins first**. If no fingerprint match exists it looks for an open issue labelled `hive/selfreport` for the same class, then for a human-filed issue whose title or body mentions the class keywords (for example "restart storm", -"model discovery degraded", "LiteLLM 401"). When any of those matches, the -hive adds itself to that issue instead of opening a new one: +"model discovery degraded", "LiteLLM 401"). The catch-all +`unclassified_error_repeated` class says nothing about the defect, so its +class search is additionally scoped to the leading words of the message in +the title; with no usable message it does not join by class at all. When +any of those matches, the hive adds itself to that issue instead of opening +a new one: - one comment `Also seen on **** (v, ) at
LevelNameAgentsAllowed Actions