diff --git a/.github/scripts/competitor_report.py b/.github/scripts/competitor_report.py index 04343acb68..5407c48c65 100644 --- a/.github/scripts/competitor_report.py +++ b/.github/scripts/competitor_report.py @@ -186,6 +186,82 @@ def split_report(body): return report.strip(), json.loads(encoded[:-len(LEDGER_END)]) +WHO_HEADING = "Who does what" + + +def discord_normalize(text): + """Collapse the Discord-only formatting the prompt asks for so the two + visible reports can be compared: bold headings instead of `#`, `` + instead of bare/Markdown links, no GitHub-only `
` block.""" + text = re.sub(r"
.*?
", "", text, flags=re.S | re.I) + text = re.sub(r"^\s*#{1,6}\s*(.+?)\s*$", r"**\1**", text, flags=re.M) + text = re.sub(r"\[([^\]]+)\]\((https?://[^)\s]+)\)", r"\1 \2", text) + text = re.sub(r"<(https?://[^>\s]+)>", r"\1", text) + return text + + +def split_who_section(text): + """Return (text without the "Who does what" body, that body).""" + pattern = re.compile(r"^\*\*%s\*\*\s*$(.*?)(?=^\*\*[^*\n]+\*\*\s*$|\Z)" % re.escape(WHO_HEADING), + re.M | re.S) + match = pattern.search(text) + require(match is not None, f'Missing the "{WHO_HEADING}" section') + return text[:match.start(1)] + "\n" + text[match.end(1):], match.group(1) + + +def table_projects(section): + """Project names (first column) and the notes outside the table. The + table may be a Markdown pipe table or a fenced block of aligned columns.""" + lines = section.splitlines() + rows = [line for line in lines if line.strip().startswith("|")] + if rows: + cells = [[c.strip() for c in row.strip().strip("|").split("|")] for row in rows] + projects = [row[0] for row in cells[2:] if row and row[0]] + notes = [line for line in lines if not line.strip().startswith("|")] + return projects, notes + fenced = re.findall(r"```[^\n]*\n(.*?)```", section, re.S) + projects = [] + for block in fenced: + body = [line for line in block.splitlines()[1:] if line.strip() + and not re.fullmatch(r"[-+| ]+", line.strip())] + projects += [re.split(r"\s*\|\s*|\s{2,}", line.strip())[0] for line in body] + notes = re.sub(r"```.*?```", "", section, flags=re.S).splitlines() + return [p for p in projects if p], notes + + +def check_discord_matches(report, message): + """The Discord post must carry the same visible report as GitHub. The + prompt asks for Discord formatting (bold headings, the project table as a + fenced code block, `` links, no sources `
`), so compare the + prose after normalising those, and compare the table by project names.""" + github_text, github_who = split_who_section(discord_normalize(report)) + discord_text, discord_who = split_who_section(discord_normalize(message)) + squash = lambda s: " ".join(s.split()) + require(squash(github_text) == squash(discord_text), + "Discord and GitHub reports differ outside the project table " + "(ledger and sources details belong only on GitHub)") + projects, notes = table_projects(github_who) + require(projects, f'GitHub "{WHO_HEADING}" section needs a project table') + fenced = re.findall(r"```[^\n]*\n(.*?)```", discord_who, re.S) + require(fenced, f'Discord "{WHO_HEADING}" table must be a fenced code block') + block = squash(" ".join(fenced)).lower() + for project in projects: + # Discord rows are width-limited and abbreviated; the first word of the + # project name is the stable key ("Roo Code (archived)" -> "roo"). + key = re.search(r"[a-z0-9][\w.+-]*", project, re.I) + require(key is None or key.group(0).lower() in block, + f"Discord project table is missing {project!r}") + # Notes under the table get re-wrapped for Discord; hold them to the same + # provenance links rather than the same wording. + note_text = " ".join(notes) + discord_squashed = squash(discord_who) + require(not note_text.strip() or re.sub(r"```.*?```", "", discord_who, flags=re.S).strip(), + f'Discord "{WHO_HEADING}" section dropped the notes under the table') + for url in re.findall(r"https?://\S+", note_text): + require(url.rstrip(".,;)") in discord_squashed, + f'Discord "{WHO_HEADING}" notes dropped {url}') + + def check_source_url(url, bundle): """Local source URLs need no unauthenticated HEAD probes.""" if "github.com/hivecommons/hive/blob/" in url or "github.com/hivecommons/hive/tree/" in url: @@ -224,8 +300,7 @@ def validate(payload, bundle, fetch=api, open_issues=False, today=None): require(len(re.findall(r"https?://(?=\S)", body)) <= MAX_COMMENT_LINKS, "GitHub body exceeds add_comment's 50-link limit (ledger URLs count too)") report, ledger = split_report(body) - require(discord[0].get("message", "").strip() == report, - "Discord and GitHub reports differ (ledger belongs only on GitHub)") + check_discord_matches(report, discord[0].get("message", "").strip()) require(len(report.split()) >= 80, "Report is empty or implausibly short") require(ledger.get("sha") == bundle["sha"], "Ledger source SHA differs from fetched source") provenance = (f"Source: {bundle['repository']}; SHA: {bundle['sha']}; " @@ -234,8 +309,14 @@ def validate(payload, bundle, fetch=api, open_issues=False, today=None): require("Evidence limits:" in report, "Missing visible evidence/verification limitations") require(not re.search(r"\bHive (?:is|remains) GitHub[- ]only\b|Forge: GitHub today", report, re.I), "Known blanket GitHub-only regression; distinguish adapters from reachable execution") - require(not re.search(r"\b(?:no licen[cs]e|unlicensed)\b", report, re.I), - "Do not infer no product license from absent API detection") + for match in re.finditer(r"\b(?:no licen[cs]e|unlicensed)\b", report, re.I): + # "does not mean no licence" / "returned no licence, so unverified" are + # the disclaimers we want, not the inference. + lead = re.split(r"[.;\n]", report[max(0, match.start() - 40):match.start()])[-1] + tail = re.split(r"[.;\n]", report[match.end():match.end() + 60])[0] + require(re.search(r"\b(?:not|never|rather than)\b", lead, re.I) + or re.search(r"\b(?:unverified|not detected)\b", tail, re.I), + "Do not infer no product license from absent API detection") require(not re.search(r"\|\s*(?:none|null|NOASSERTION)\s*\|", report, re.I), "Use 'not detected (unverified)' for ambiguous API license metadata") diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index 6cbea58e7e..71d27716ce 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000; const MAX_AUTHOR_BATCH = 50; // This literal is inside the enrolled .github closure. A caller-supplied // preparedBy.sourceSHA must never choose a new executable verifier tree. -const TRUSTED_SCRIPTS_TREE = '5d965ad41a5898673930799efca57d03fe31a52d'; +const TRUSTED_SCRIPTS_TREE = '689fea04d4b10295f67b9de708797343938d7e0b'; const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4'; const sha = value => crypto.createHash('sha256').update(value).digest('hex'); const git = (args, cwd = process.cwd()) => diff --git a/.github/scripts/test_competitor_report.py b/.github/scripts/test_competitor_report.py index f95b4e35a1..e42d6edf0a 100644 --- a/.github/scripts/test_competitor_report.py +++ b/.github/scripts/test_competitor_report.py @@ -44,34 +44,96 @@ def url(self, ref): return (f"https://github.com/hivecommons/hive/blob/{self.bundle['sha']}/" f"{ref['path']}#L{ref['start']}-L{ref['end']}") - def payload(self): + GITHUB_TABLE = ( + "| Project | Who runs it | Category | Relationship to Hive | Licence |\n" + "|---|---|---|---|---|\n" + "| OpenHands | All Hands AI | terminal agent | runs as worker engine today | MIT |\n" + "| Roo Code (archived) | Roo | editor agent | could run as engine | Apache-2.0 |\n" + "| Devin, Jules, Codex cloud | vendors | cloud agent | complementary | not detected (unverified) |\n" + "Licence from the GitHub API https://api.github.com/repos/OpenHands/OpenHands.\n" + ) + DISCORD_TABLE = ( + "```\n" + "Project Category Relationship Licence\n" + "OpenHands terminal engine today MIT\n" + "Roo Code (arch.) editor could be Apache-2.0\n" + "Devin/Jules/etc cloud complementary not detected\n" + "```\n" + "Owners: OpenHands All Hands AI; Roo Code Roo.\n" + "Licences via GitHub API .\n" + ) + + def report_text(self, heading, table): text = ( f"Source: hivecommons/hive; SHA: {self.bundle['sha']}; " f"default: {self.bundle['default_branch']}\n" "**Hive weekly competitor report - 2026-11-02**\n" - "**What changed this week**\nno news found\n" - "**Who does what**\n" - "**SWOT for Hive**\n" + f"{heading('What changed this week')}\nno news found\n" + f"{heading('Who does what')}\n{table}" + f"{heading('SWOT for Hive')}\n" ) for claim in self.ledger["claims"]: text += claim["text"] + " " + " ".join( f"<{self.url(ref)}>" for ref in claim["sources"]) + "\n" text += ( - "**One thing to watch**\n" + f"{heading('One thing to watch')}\n" "Follow reachable operations rather than mistaking a source interface " "for deployed feature parity. No new unsupported work is proposed.\n" - "**Top 5 this week - stay ahead and stay integrated**\n" + f"{heading('Top 5 this week - stay ahead and stay integrated')}\n" "Evidence limits: quotations establish provenance, not semantic proof. " "All-state work searches are bounded, and inaccessible vendor news is " "not a basis for opening work.\n" + self.extra ) + return text.strip() + + def github_text(self): + return self.report_text(lambda h: f"### {h}", self.GITHUB_TABLE) + ( + "\n
Sources and queries\n- HN Algolia: coding agent\n
") + + def discord_text(self): + return self.report_text(lambda h: f"**{h}**", self.DISCORD_TABLE) + + def payload(self): return {"items": [ {"type": "add_comment", "item_number": 14613, - "body": text.strip() + "\n" + report.LEDGER_START + + "body": self.github_text() + "\n" + report.LEDGER_START + json.dumps(self.ledger) + report.LEDGER_END}, - {"type": "post_to_discord", "message": text.strip()}, + {"type": "post_to_discord", "message": self.discord_text()}, ]} + def test_discord_formatting_differences_are_allowed(self): + """Regression for run 42622204563: the prompt asks for bold headings, a + fenced table and no sources details on Discord, so byte-equality must + not be required.""" + self.validate() + + def test_github_table_may_be_a_fenced_block(self): + """Regression for run 43065486702: the agent used a fenced block on + GitHub too, which must count as the project table.""" + payload = self.payload() + payload["items"][0]["body"] = payload["items"][0]["body"].replace( + self.GITHUB_TABLE, self.DISCORD_TABLE.replace("Owners: OpenHands All Hands AI; Roo Code Roo.\n", "")) + self.validate(payload) + payload["items"][1]["message"] = self.discord_text().replace( + "Roo Code (arch.) editor could be Apache-2.0\n", "", 1) + with self.assertRaisesRegex(ValueError, "missing 'Roo Code"): + self.validate(payload) + + def test_discord_prose_must_match_github(self): + payload = self.payload() + payload["items"][1]["message"] = self.discord_text().replace( + "no news found", "OpenHands raised a round") + with self.assertRaisesRegex(ValueError, "differ outside the project table"): + self.validate(payload) + + def test_discord_table_must_keep_every_project_and_note(self): + for missing, message in (("Roo Code (arch.) editor could be Apache-2.0\n", "missing 'Roo Code"), + ("Licences via GitHub API .\n", "dropped")): + payload = self.payload() + payload["items"][1]["message"] = self.discord_text().replace(missing, "", 1) + with self.subTest(missing=missing), self.assertRaisesRegex(ValueError, message): + self.validate(payload) + def validate(self, payload=None, fetch=no_work, **kwargs): return report.validate(payload or self.payload(), self.bundle, fetch=fetch, today=TODAY, **kwargs) @@ -156,6 +218,12 @@ def test_license_detection_is_not_product_license(self): self.validate() self.extra = "| Project | not detected (unverified) |" self.validate() + # Regressions from runs 43065486702 / 43070009570: disclaimers are fine. + for text in ('"not detected (unverified)" means the API gave null; it does not mean no licence.', + "The API returned no licence, so it is unverified."): + self.extra = text + with self.subTest(text=text): + self.validate() def metric(self): claim = { diff --git a/.github/workflows/changelog-fragment-guard.yml b/.github/workflows/changelog-fragment-guard.yml index 3b5319af7e..850a0da40f 100644 --- a/.github/workflows/changelog-fragment-guard.yml +++ b/.github/workflows/changelog-fragment-guard.yml @@ -121,17 +121,10 @@ jobs: # Full history so the merge base with the target branch exists # locally and the three-dot diff below is computable. fetch-depth: 0 - # The autofix push uses TOPUP_PUSH_TOKEN when set: a push made with - # that PAT fires `pull_request` AND `pull_request_target` runs, so - # every check re-reports on the new head. With github.token the - # push lands and fires the `pull_request` runs, but NO - # `pull_request_target` run (observed on #16567's b9cd4bac: eleven - # pull_request runs with actor=github-actions[bot], no PR Verifier). - # Dependabot-triggered runs see no Actions secrets, so Dependabot PRs - # (#16567) always take the github.token path; the dispatch step - # after the push re-runs pr-verifier for them. Same optional-PAT - # convention as cli-pin-bump.yml / v5-topup.yml / notice-autofix.yml. - token: ${{ secrets.TOPUP_PUSH_TOKEN != '' && secrets.TOPUP_PUSH_TOKEN || github.token }} + # Nothing survives the checkout: the guard steps below run scripts + # from the PR's own tree. Only the autofix step pushes, and it gets + # the token through its step env (PUSH_TOKEN), never via .git/config. + persist-credentials: false - name: Code changes must carry a changelog.d fragment (or no-changelog) id: guard @@ -186,6 +179,17 @@ jobs: PR_AUTHOR_ID: ${{ github.event.pull_request.user.id }} HEAD_REF: ${{ github.event.pull_request.head.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} + # The autofix push uses TOPUP_PUSH_TOKEN when set: a push made with + # that PAT fires `pull_request` AND `pull_request_target` runs, so + # every check re-reports on the new head. With github.token the + # push lands and fires the `pull_request` runs, but NO + # `pull_request_target` run (observed on #16567's b9cd4bac: eleven + # pull_request runs with actor=github-actions[bot], no PR Verifier). + # Dependabot-triggered runs see no Actions secrets, so Dependabot PRs + # (#16567) always take the github.token path; the dispatch step + # after the push re-runs pr-verifier for them. Same optional-PAT + # convention as cli-pin-bump.yml / v5-topup.yml / notice-autofix.yml. + PUSH_TOKEN: ${{ secrets.TOPUP_PUSH_TOKEN != '' && secrets.TOPUP_PUSH_TOKEN || github.token }} MISPLACED_FILE: ${{ runner.temp }}/changelog-misplaced.txt run: | set -euo pipefail @@ -252,7 +256,9 @@ jobs: # When this push was made with github.token it fires no # `pull_request_target` run, so pr-verifier (a required context) # would stay "expected"; the step below re-triggers it (#17036). - if git push --force-with-lease="refs/heads/${HEAD_REF}:${HEAD_SHA}" origin "HEAD:refs/heads/${HEAD_REF}"; then + auth="$(printf 'x-access-token:%s' "$PUSH_TOKEN" | base64 -w0)" + echo "::add-mask::${auth}" + if git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth}" push --force-with-lease="refs/heads/${HEAD_REF}:${HEAD_SHA}" origin "HEAD:refs/heads/${HEAD_REF}"; then echo "pushed=true" >> "$GITHUB_OUTPUT" echo "fragment=$f" >> "$GITHUB_OUTPUT" else diff --git a/.github/workflows/ci-runner-image.yml b/.github/workflows/ci-runner-image.yml index 69705fe271..9982bcf61b 100644 --- a/.github/workflows/ci-runner-image.yml +++ b/.github/workflows/ci-runner-image.yml @@ -106,6 +106,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Resolve the base image and the tag once, up front, so every later step # (the exists-check, the build, the summary) agrees on them. diff --git a/.github/workflows/dco-post-merge.yml b/.github/workflows/dco-post-merge.yml index a176946472..d89a92d769 100644 --- a/.github/workflows/dco-post-merge.yml +++ b/.github/workflows/dco-post-merge.yml @@ -331,17 +331,25 @@ env: # 3fedd0f3cbf5703a306157411a7136ecff738f5c — same shape, person and # disposition as 003144299b above (#15542). Reversible: remove the entry. - # 76b20ebe56639360ff27e15ed3751afc96c04785 — the squash of #17464 (feat(dashboard): - # owner toggle for admin MCP writes), on v6, reported by the monitor in #17470. - # mismatched-signoff of the same shape as 7e01fee4 above: same person, two - # plain addresses — authored as Danathar (the - # identity on 230+ prior v6 commits), signed off as Danathar - # . Neither is a GitHub noreply address, so the - # checker's same-person evidence rule cannot and should not infer the link. - # Protected v6 history cannot be re-signed in place and no sign-off is added - # on the author's behalf, so it is waived per commit. Accepted by maintainer - # disposition in #17470: the maintainer merging the PR that adds this entry - # IS that disposition. Reversible: remove the entry. + # 76b20ebe56639360ff27e15ed3751afc96c04785 — the squash of #17464, on v6, + # with the mismatched-signoff shape created by the squash: the branch + # commits were authored AND signed off as Danathar + # , an Apple iCloud "Hide My Email" relay + # address, so the PR was internally consistent; GitHub's squash then wrote + # the PR author's profile-primary identity, Danathar + # , over it while keeping that sign-off. Same class + # as 7e01fee4a2c1a37f4a4072bf2d17e76e27113d01 and + # 4b9261183378a1a02a921cd0cc9df31e6f37993b: one maintainer, two addresses, + # nothing about the certification in doubt. The checker should not infer the + # link: a gmail or relay address carries no structural proof that it belongs + # to the authoring account (same reasoning as 7e01fee), so this is a + # per-commit waiver and not a DCO_ALLOWLIST_EMAILS entry. Protected v6 + # history cannot be re-signed in place and no sign-off is rewritten on + # another person's behalf. Reported by the monitor in #17558. Accepted by + # maintainer disposition in #17558: the maintainer merging the PR that adds + # this entry IS that disposition. Reversible: remove the entry. To stop the + # squash producing this shape, the contributor can set the relay address as + # their GitHub profile-primary email, or sign off with the primary address. # KNOWN FAILING COMMITS THAT ARE DELIBERATELY *NOT* WAIVED # diff --git a/.github/workflows/docs-index-reminder.yml b/.github/workflows/docs-index-reminder.yml index 8db796ca3b..88a9a8f0cc 100644 --- a/.github/workflows/docs-index-reminder.yml +++ b/.github/workflows/docs-index-reminder.yml @@ -48,6 +48,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Find docs added by this PR that the index does not link id: check diff --git a/.github/workflows/mutation-advisory.yml b/.github/workflows/mutation-advisory.yml index 0cda9765dd..30e30cb32c 100644 --- a/.github/workflows/mutation-advisory.yml +++ b/.github/workflows/mutation-advisory.yml @@ -28,6 +28,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + persist-credentials: false fetch-depth: 0 - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: @@ -43,6 +44,9 @@ jobs: - name: Run mutation report env: BASE_REF: ${{ github.base_ref }} + # Must stay under timeout-minutes (20m = 1200s): a job killed by that + # timeout is `cancelled` and loses the report; the script stops itself first. + MUTATION_BUDGET_SECONDS: '900' run: | export PATH="$(go env GOPATH)/bin:$PATH" bash src/scripts/mutation-report.sh run diff --git a/.github/workflows/pr-overlap.yml b/.github/workflows/pr-overlap.yml index e959ff8dc1..d439ffc1d4 100644 --- a/.github/workflows/pr-overlap.yml +++ b/.github/workflows/pr-overlap.yml @@ -23,6 +23,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + persist-credentials: false fetch-depth: 0 # Hermetic (local repos + gh double), under a second: a broken check # mislabels every open PR, so it must not ship untested (#9319). diff --git a/.github/workflows/stream-image.yml b/.github/workflows/stream-image.yml index 6bf45e2f18..f73a7ec646 100644 --- a/.github/workflows/stream-image.yml +++ b/.github/workflows/stream-image.yml @@ -73,6 +73,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # The supervisor scripts have no Go test lane; shellcheck them and run # the watchdog decision tests (fake ffmpeg, stubbed Twitch check) here, diff --git a/CHANGELOG.md b/CHANGELOG.md index e17165d169..76c9bfaad7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -396,6 +396,183 @@ Hive did not historically maintain a complete changelog. This file starts a prag - release: cap GitHub Release notes at `RELEASE_NOTES_MAX_CHARS` (120000) before the SBOM/NOTICE trailer, cutting the changelog section at a bullet boundary and linking the tag's full `CHANGELOG.md` section, so a first-on-the-line release such as v6.0.0 no longer fails `gh release create` with HTTP 422 "body is too long" after the tag and image retags have landed. +## 2026-11-11 (v5.220.0) + +### Added + +- Benchmarks: every hive now contributes **anonymized** per-PR benchmark records to the public leaderboard at (data repo `hivecommons/benchmarks`), built from the review evidence bundles it already seals. A record carries a salted hash of the hive id, the agent, harness, model, sandbox runtime, task kind, ACMM level, review verdicts and merge/CI/revert outcome — never the hive id, PR numbers, SHAs, prompts, diffs, logins or costs. Publication is **on by default**; `benchmarks.publish: false` opts a hive out, and `benchmarks.repo_consent` (default on) names repositories only when GitHub reports them public, so no new information is disclosed. Cadence is hub-driven and hourly: the spoke pushes new records to the hub's `POST /api/benchmarks/ingest` with its heartbeat bearer, and the hub commits month files to the data repo through its App identity; `GET /api/benchmarks/export?since=` and the `benchmarks` block in `/api/status` expose the same data and last-push state on the spoke, `GET /api/hub/benchmarks` on the hub, and Settings → Features → Benchmarks shows the toggles and the hive's hash. New bundles are stamped with the agent's sandbox runtime and task kind. See `docs/benchmarks.md` (#17694). +- Workflow models, RFC phase 4 UI: Settings → Workflows grows an **Editor** — a visual design surface for lifecycle models rather than a form. Columns are vertical lanes (add, rename by double-click, reorder and resize by dragging the header, toggle terminal); states are chips you drag within and between lanes or drop from a templates palette (common states and columns, hive-default stages); transitions are drawn by dragging a chip's port onto another chip, with backward edges dashed and self-loops drawn as loops. The canvas pans, zooms, fits, shows a minimap, supports marquee and shift-click multi-select, keyboard nudging and Delete, undo/redo, and an Auto-layout that orders lanes by topological depth. An inspector edits the selected state, transition, column or the model itself; a bottom drawer hosts Problems (live compile diagnostics, badged on the canvas and clickable), YAML (two-way via the new `POST /api/workflows/yaml`), Diff (client-side structural diff against the last save, or the stored versions diff) and Simulate (facts → `POST /api/workflows/simulate` against the unsaved model, highlighting the decided state). Save assigns the next version through `POST /api/workflows` with an empty `meta.version`; Save as… and Revert are one click; inferred states carry a ✓ Confirm affordance with a toolbar "Confirm all inferred". Export offers self-contained SVG, PNG, model JSON and YAML. Library rows and detail gain Edit (owner) / Edit a copy (hive-default), and ingest results gain "Open in editor" ([#17620](https://github.com/hivecommons/hive/issues/17620)) +- **workflow models:** two more ways into a draft lifecycle model on Settings → Workflows → Ingest. *From a cluster* lists a kube context's Tekton Pipelines, Tasks, newest PipelineRuns and Triggers through `kubectl`, feeds them to the same Tekton ingester as a repository (citations at `cluster///.yaml`) and records each pipeline's last run status as an *observed* annotation, never an inferred state; clear 4xx for missing CRDs, RBAC denials and unknown contexts, with namespace, run, object and byte caps. *From a written spec* turns pasted markdown, a repository file or a public URL into a draft: a deterministic pass reads step lists, if/when/after sentences and role nouns into states, guards and gates, and — only when a model gateway is configured and the operator opts in — the gateway refines that skeleton, its answer accepted solely if it parses strictly, passes `Mark` (every element forced to `confidence: inferred` with a `spec:#L-L` citation, provenance `ingested-unreviewed`) and compiles, with one diagnostic-quoting retry before falling back. The LLM is consulted nowhere else in the workflow-model stack. `POST /api/workflows/ingest/{cluster,spec}` and `GET /api/workflows/ingest/capabilities`, owner-only, audited as `workflow_ingest` ([#17620](https://github.com/hivecommons/hive/issues/17620)) +- Workflow models (RFC phase 4, backend): `GET /api/workflows/{name}/versions`, a deterministic structural diff between two versions (`GET …/{name}/diff?from&to`, optionally cross-model against hive-default), `POST …/{name}/{version}/confirm` to confirm an ingested model's inferred elements as a new local version (owner, audited `workflow_confirm`), `GET …/{name}/{version}/labels` to preview the labels and commands a model regenerates, and the stage-label provisioner now creates or recolours a bound model's own label vocabulary and re-provisions when the binding changes ([#17620](https://github.com/hivecommons/hive/issues/17620)) + +## 2026-11-11 (v5.219.1) + +### Fixed + +- Weekly competitor report: the publication gate keys Discord table rows by project first word, holds table notes to their provenance links, and accepts "does not mean no licence"-style disclaimers (catch-up run 43070009570 failed on these). + +## 2026-11-11 (v5.219.0) + +### Added + +- **workflow models:** the stage engine now remembers which model labels it provisioned per repository and, on unbind or rebind to a model with a different label set, deletes the stale ones — only labels it created, outside the built-in stage family and worn by no open issue or PR; each deletion is audited `workflow_label_removed` and `workflow.label_gc: false` keeps everything. A bound model's `replies` are now offered by the Activity Monitor's human-gate pill and accepted as `/hive ` on parked issues, applying the label the model maps to the reply's target state ([#17620](https://github.com/hivecommons/hive/issues/17620)) + +### Fixed + +- Weekly competitor report: the publication gate accepts the "Who does what" project table as a fenced block on GitHub as well as a Markdown table (catch-up run 43065486702 failed on this). + +## 2026-11-11 (v5.218.0) + +### Added + +- **workflow models:** ingesting a repository now proposes its *intake half* alongside its pipelines: issue templates become `requested-` intake states and events, a triage label (from templates, `settings.yml`, `labels.yml`, `labeler.yml` or `CONTRIBUTING.md`) becomes a `triage` state and gate, PR-template checklist items become manual checks, `CODEOWNERS`/Prow `OWNERS` become `approver`/`reviewer` roles owning the review gate, a `MAINTAINERS` file's `@handles` become the `maintainer` role's members, `kind/*`/`priority/*`/`area/*` labels are carried as classification rows, and the default branch's protection or rulesets (fetched when ingesting by `repo`, or a probot `settings.yml`) become required CI checks, `review.minApprovals`, a `codeowner-review` gate and bookkeeping notes. Everything is `inferred` and cites `path:line` or the API endpoint; ingesting `cncf/techdocs` now proposes its triage state. Trees without these inputs produce the byte-identical model as before; `workflow-survey` counts the proposals per repository ([#17620](https://github.com/hivecommons/hive/issues/17620)) +- Workflow models, RFC phase 3c ([#17620](https://github.com/hivecommons/hive/issues/17620)): simulate an issue or PR against a model before binding it. New read-only `POST /api/workflows/{name}/{version}/simulate` takes `{kind, facts}` (a JSON mirror of the engine's `Facts`) or `{repo, number}` to best-effort fill the facts from the forge (the response lists what was fetched and what stayed unknown), runs the stored model's deriver through the new `stageengine.Explain`, and returns the decision (state, column, stage label, reason, covered/abstain) with a per-rule trace — every rule in order, the first match highlighted, each predicate's holds/known verdict — or a single built-in entry for `derivation: builtin` models. Settings → Workflows → model detail grows a **Simulate** panel: pick `owner/repo#123` and Fetch facts, or describe the item in a form, Run, and the matched state lights up in the diagram. + +### Fixed + +- Weekly competitor report: the publication gate no longer rejects the Discord post for the Discord formatting the prompt itself requires (bold headings, fenced project table, no sources details), which silently blocked the 2026-11-09 report. +- The `hive-inference` vLLM Deployment's `model-download` init container now pulls `docker.io/curlimages/curl` by its fully-qualified name instead of the Docker Hub short name `curlimages/curl`. Oracle Linux 9 cri-o (OKE 1.34 node images) enforces short-name resolution (`short-name-mode = "enforcing"`), so on the upgraded hive-oke nodes the pod stuck in `Init:ImageInspectError` with "image name curlimages/curl:8.13.0 returns ambiguous list" and the model never downloaded; OL8 nodes silently resolved it to Docker Hub. Found during the hive-oke OL8→OL9 / Landlock node cycling (#17642). + +## 2026-11-11 (v5.217.1) + +### Changed + +- CI: the `pull_request` workflows `mutation-advisory`, `pr-overlap`, `stream-image`, `ci-runner-image` and `docs-index-reminder` now check out with `persist-credentials: false`. None of those jobs pushes or fetches with the job token; steps that call the API keep getting it through their step `env`, scoped by the job's `permissions:` block (#17636). + +### Security + +- **proxy:** the GitHub GraphQL mutation gate no longer fails open when `mutation` follows a same-line prefix (a leading fragment definition, comma or BOM). It now uses the same string- and comment-aware depth-zero token scan as the Linear path, so read-only lanes cannot slip a mutation past the capability tier, write-surface and auto-merge checks (#17676). + +## 2026-11-11 (v5.217.0) + +### Added + +- Workflow models (RFC phase 4, backend): `GET /api/workflows/{name}/versions`, a deterministic structural diff between two versions (`GET …/{name}/diff?from&to`, optionally cross-model against hive-default), `POST …/{name}/{version}/confirm` to confirm an ingested model's inferred elements as a new local version (owner, audited `workflow_confirm`), `GET …/{name}/{version}/labels` to preview the labels and commands a model regenerates, and the stage-label provisioner now creates or recolours a bound model's own label vocabulary and re-provisions when the binding changes ([#17620](https://github.com/hivecommons/hive/issues/17620)) +- Settings gains a **Workflows** tab (RFC phase 3b): a library of every workflow model the hive knows — built-in `hive-default`, locally saved models and drafts ingested from repository pipelines — with provenance and compile badges, bound-repository counts, View / Duplicate / Export / Delete, a read-only detail view that draws the compiled columns and states as an SVG diagram with transitions, diagnostics and citations, and an Ingest form that turns a repository's pipeline files into a draft model through `POST /api/workflows/ingest`. The Activity Monitor model badge deep-links into the library entry. Binding a repository stays on Settings → Repos; this tab never binds ([#17620](https://github.com/hivecommons/hive/issues/17620)). + +## 2026-11-11 (v5.216.0) + +### Added + +- Workflow models, RFC phase 1 ([#17620](https://github.com/hivecommons/hive/issues/17620)): the `workflow.hivecommons.dev/v1alpha1` schema gains the lifecycle half of the RFC — `columns`, `derivation`, `events`, `gates`, `roles`, `agents`, `checks`, `bookkeeping`, `labels`, `meta` and per-state `lifecycle` (column, stage number, item kinds, replies) — and `workflowmodel.Compile` turns a model into a `Profile` or refuses it with coded diagnostics (a non-terminal state with no exit, a gate with no releasing role, an agent-granted state with no human-released gate before a terminal outcome, two events bound to one fact, a state in no or two columns, duplicate stage numbers, an unknown fact or operator). `pkg/workflowmodel/hivedefault` generates the `hive-default` model from the live stage table and the engine's expected-edge table, so the monitor's seven columns and the stage numbers are now a compiled model rather than constants. `stageengine.DeriveFromModel` derives under a compiled profile: `hive-default` declares `derivation: builtin` and delegates to the existing derivers (the whole PR stage-golden corpus and the issue sweep derive identically), custom models declare `derivation: rules` (ordered first-match predicates over the fact vocabulary, uncovered items abstain `workflow: not covered`). New `workflow.enabled` config flag, default false; nothing changes for any hive until a repository is bound (phase 2). +- Workflow models, RFC phase 3a ([#17620](https://github.com/hivecommons/hive/issues/17620)): an ingested repository now yields a model that compiles. `ingest.Lifecycle` composes the hive-default lifecycle (states, columns, builtin derivation, roles, gates, events) with the repository's pipelines — PR-verify pipelines bound to the CI / Review states with file citations, the post-merge chain (main CI, release, per-environment deploys, done) as new columns after Merged — every ingested element marked inferred and the model's provenance `ingested-unreviewed`. New fact `pipeline.status` names the post-merge run the chain waits on (the engine reports `unknown` until it observes post-merge runs, so nothing moves on it yet). New owner-gated `POST /api/workflows/ingest` takes `{repo}` (default branch fetched read-only through the hive's GitHub client, CI files selected by `ingest.ReadTree`) or pasted `{files}`, returns the model, compile result and diagnostics, and with `save: true` stores it at the next integer version. `cmd/workflow-survey` compiles every repository's lifecycle model and reports pass/refusal counts by code; over the 317-repository CNCF corpus 317/317 compile with 1,117 inferred-element warnings and no refusals. +- Workflow models, phase 1 ([#17620](https://github.com/hivecommons/hive/issues/17620)): a new `pkg/workflowmodel` package ingests a repository's GitHub Actions workflows and Tekton / OpenShift Pipelines resources (`pkg/workflowmodel/githubactions`, `pkg/workflowmodel/tekton`, composed by `pkg/workflowmodel/ingest`) into one deterministic, content-hashed model of the repo's delivery process — pipelines with their trigger roles (PR verify, push verify, release, deploy, scheduled, manual), execution DAGs with topological levels, matrices, reusable workflows, environments and concurrency — bound onto the hive default lifecycle. `pkg/workflowmodel/render` turns a model into a Mermaid diagram and the Activity Monitor column set it implies (CI sub-stages under *CI / Review*, post-merge *Main CI*, *Release* and *Deploy: * columns), and a model with no pipelines renders today's seven columns unchanged. Nothing is wired into the v5/v6 stage engine yet; `cmd/workflow-survey` is the proving harness that produced the CNCF-landscape survey posted on the issue (317 repositories, 4,389 pipelines, 317 valid models). Golden fixtures under `pkg/workflowmodel/testdata/corpus` pin real repositories' results. +- Workflow models, RFC phase 2 ([#17620](https://github.com/hivecommons/hive/issues/17620)): models now have a home and a way to reach the engine. `pkg/workflowmodel/store` keeps versioned models under `/workflows//.json` (plus `index.json`); `hive-default` is virtual and read-only, drafts save but do not compile, every entry reserves `adoption`/`ratings`/`benchmarks` fields for the marketplace and carries them verbatim across saves. New owner-gated, audited `/api/workflows` routes list, save, read, delete and compile models; `project.workflow_bindings: [{repo, name, version}]` binds a repository to a compiling model (the binding is refused for a draft, and a bound model cannot be deleted). When `workflow.enabled` is on, the stage reconciler derives bound repositories through the bound model and everything else through the built-in derivers exactly as before. Settings → Repos gains a per-repo Workflow select, and the Activity Monitor response gains a `workflows` block (default columns plus per-repo columns and stage numbers) that the tracker uses to label the model column and to render a per-row column strip, with an amber "Not covered" chevron when a bound model abstains. Nothing changes for unbound repositories. + +## 2026-11-11 (v5.215.2) + +### Fixed + +- Hardened the automerge workflow-approval path against the read-only integration eligibility probe: the probe could reach the `/ok-to-test` post, where the mutation boundary refused the comment and the caller logged a WARN on every tick. The probe now skips the post and leaves it to the live sweep; live behaviour is unchanged. + +## 2026-11-11 (v5.215.1) + +### Security + +- Dashboard pane surfaces (`/api/agents/{name}/log`, `/api/status` summaries, Job log streams, archived kick/task logs) now mask JWTs, AWS access keys, Slack tokens and webhooks, `Authorization: Bearer`/`Basic` credentials and private-key blocks, not just GitHub and `sk-` tokens, because `logscrub.RedactPaneTokens` now also runs `ScrubString` ([#17665](https://github.com/hivecommons/hive/issues/17665)). + +## 2026-11-11 (v5.215.0) + +### Added + +- The dashboard's ▶ terminal pane now works for `sandbox.runtime: openshell` agents: it opens the read-only `/terminal/job/` viewer, follows the live OpenShell sandbox's gateway event log while a kick runs (scrubbed like the Job pod log; the pane says this is the sandbox's supervisor/policy log, not the kick's stdout, which an exec session does not expose) and shows the last persisted transcript — with the kick's own output and exit code — between kicks. + +## 2026-11-11 (v5.214.1) + +### Security + +- Agent tmux panes no longer inherit the hive container's dashboard shared token (`DASHBOARD_AUTH_TOKEN` / `HIVE_DASHBOARD_TOKEN`), the hub-injected per-hive signing keys (`HIVE_TERMINAL_KEY`, `HIVE_INVITE_KEY`, `HIVE_HEARTBEAT_KEY`, `HIVE_HUB_SECRET`), the ttyd credential or the corporate upstream-proxy settings; `applySessionEnv` removes them from the tmux global environment alongside the GitHub and Linear credentials it already stripped. + +## 2026-11-11 (v5.214.0) + +### Added + +- Added an OpenShell cluster preflight and an egress-policy prover: at boot and on reload, `AgentSandboxGateWarnings` now warns when an `openshell`-runtime gateway is unreachable, reports telemetry on, or has `allow_driver_config` off (read-only, cached, bounded to 30 s), and `allow_hosts` entries must be literal host names (userinfo, query, fragment and wildcards are rejected) with a test that fails if the rendered egress policy widens ([#17644](https://github.com/hivecommons/hive/issues/17644)). + +## 2026-11-11 (v5.213.0) + +### Added + +- **Upstream forward-proxy chaining for hosts behind a corporate proxy.** A hive whose enterprise GitHub is reachable only through a corporate HTTP forward proxy could not use it: the control-plane client honoured `HTTPS_PROXY`, but every agent `:443` flow is iptables-redirected into the in-process MITM proxy, which always dialled upstream directly — so agent `git`/`gh` traffic to the GHE host simply timed out. New `network.upstream_proxy` config (`url`, `hosts`, `no_proxy`; env `HIVE_UPSTREAM_PROXY_URL`/`HIVE_UPSTREAM_PROXY_HOSTS`/`HIVE_NO_PROXY`, falling back to standard `HTTPS_PROXY`/`NO_PROXY`) drives one selector used by the control-plane GitHub transport, in-process `git` runs (scoped `GIT_CONFIG_*` entries), and the MITM proxy, which now issues `CONNECT host:443` through the upstream proxy (Basic auth from URL userinfo) and runs TLS or splices bytes over the tunnel for matching hosts; non-matching hosts keep the exact direct path. Hosted hubs can set `upstream_proxy: {url, hosts, no_proxy}` per cluster in `clusters.json`, and the provisioner injects the env vars into new pool slots. See `docs/network-requirements.md` → "Egress through a corporate forward proxy". + +## 2026-11-11 (v5.212.0) + +### Added + +- Added `sandbox.runtime: openshell`, a third sandbox runtime behind the same `sandbox.Launcher` seam as `podman` and `job` (ADR-0026 phase 1, [#14092](https://github.com/hivecommons/hive/issues/14092)). A kick on it is created through an NVIDIA OpenShell gateway (`github.com/ai-openshell/sdk-go` v0.1.8) as a kernel-confined sandbox from the agent's image, runs the kick command with the same credential-filtered environment the other launchers pass, shares the workspace through the hive data PVC exactly as the Job runtime does, and is deleted on success, failure or kick timeout. Hive hands the gateway an egress policy derived from the hosts its own proxy brokers — `github.com` read-write for git, `api.github.com` read-only — plus `agent_sandbox.openshell.allow_hosts` for a model gateway or the hub; nothing else is reachable from the sandbox. Config lives under `agent_sandbox.openshell` (`gateway_url`, `namespace`, `token_env`, `workspace_claim`, `allow_hosts`, `timeout_s`, scheduling passthrough) with per-agent overrides; the runtime is inert unless selected, and a selected one with no `gateway_url` or `workspace_claim` is warned about at boot and refused at kick time before the agent is marked running. + +## 2026-11-11 (v5.211.30) + +### Changed + +- ADR-0026 (sandbox runtime selection) moves from Proposed to **Accepted (phase 1 implemented, runtime opt-in)** now that `sandbox.runtime: openshell` exists ([#17642](https://github.com/hivecommons/hive/issues/17642)), with an implementation-notes section recording where the real SDK differed from the record (module path moved to `github.com/ai-openshell/sdk-go`, workspace shared through the data PVC instead of copied, `policy_profile` advisory, egress derived from hive's proxy host set) and the kernel survey of hive's clusters. `sandbox-isolation.md` gains the operator section for the runtime (config, egress policy, credential boundary, lifecycle, verification, limitations), `integrations.md` lists OpenShell as in progress (opt-in) instead of planned, and `src/deploy/openshell.md` covers the cluster side — prerequisites, the upstream Helm install, `allow_driver_config`, the `openshell.ai/sandbox-attachable` PVC labels, `OPENSHELL_TELEMETRY_ENABLED=false`, digest pinning — without vendoring manifests. + +## 2026-11-11 (v5.211.29) + +### Security + +- Bumped the Go toolchain to 1.26.11 so the `govulncheck` gate stops flagging GO-2026-6859 (`net/http`), reachable from the dashboard, hub spoke heartbeat and GitHub client. The gate was red on the default branch toolchain itself, blocking every open PR. + +## 2026-11-11 (v5.211.28) + +### Fixed + +- **Unclaimed hives no longer file `github_app_not_installed` self-reports** — a hosted "available slot" hive that has not been onboarded yet legitimately has no GitHub App installation, so the runtime `github_app_not_installed` condition is now kept out of fleet self-reports for it. "Unclaimed" covers both an empty `project.org` and the synthetic `available-` / legacy `placeholder` org the pool seeds into a slot (the hive behind the report carried `project.org=available-…`, not an empty org); that prefix list now lives in one place (`config.PlaceholderOrgPrefixes`) shared with the dashboard's `project_org` health check. The local warning log and the dashboard onboarding banner are unchanged, and a hive that *does* have a real org configured but no App installation still self-reports it as a real misconfiguration. A tracker already opened by a pre-fix release is closed out through the normal self-recovery path on the first evaluation after upgrade. ([#17593](https://github.com/hivecommons/hive/issues/17593), [#17600](https://github.com/hivecommons/hive/issues/17600)) + +## 2026-11-11 (v5.211.27) + +### Fixed + +- The mutation advisory job now stops itself on a wall-clock budget and lists the packages it skipped, instead of being cancelled at the job timeout and losing its report. + +## 2026-11-11 (v5.211.26) + +### Fixed + +- Give the `inference no route` WARN its own `inference_no_route` fleet-report class and tag it with a `reason` (`cleared` or `never_set`), so it no longer lands in `unclassified_error_repeated`. + +## 2026-11-11 (v5.211.25) + +### Fixed + +- Knowledge Sources: **Clone & Index** now disables the button and shows "Cloning…" while the request runs, resolves a blank branch to the remote's default branch (`git ls-remote --symref`) and records it instead of guessing `main`, and keeps a failed connect visible as an error row with the reason and Retry/Dismiss (#17610). + +## 2026-11-11 (v5.211.24) + +### Security + +- The pane line a provider error is classified from is now credential-redacted before it is stored as an agent's `LastError` / `BackendAuth.LastError`. Those fields leave the spoke verbatim — crux `/v1` status, dashboard status JSON and the hub heartbeat — and were the one pane-derived path that skipped the token scrubbing every other export applies, so a GitHub token or `sk-` key printed beside a `401`/`403` could be shipped to the hub (#17611). + +## 2026-11-10 (v5.211.23) + +### Fixed + +- Knowledge panel: Add Git Source, Add Subscription, Import Document and Import now show required-field errors inline under the offending input (red border, focus, `role="alert"` message) instead of a transient info toast, and required inputs are marked; fixes Clone & Index appearing to do nothing with an empty name or URL (#17587, part of #17583). + +## 2026-11-10 (v5.211.22) + +### Fixed + +- Issue stage labels: a repository where the GitHub App is refused label writes (403 "Resource not accessible by integration", e.g. the App lacks Issues write access) is now skipped until restart after one clear warning, instead of retrying label provisioning for every issue on every sweep and flooding the log with "provisioning failed" (#17582). + +## 2026-11-10 (v5.211.21) + +### Fixed + +- Trusted-bot automerge lane: PRs from every login the operator enabled under `auto_merge.trusted_bot_authors` (Settings bot toggles or a custom login) are now honored by the trusted-bot lane end to end. The review swarm's verdict is advisory for them by default (`auto_merge.trusted_bot_skip_review`, default `true`), so a Dependabot bump a reviewer routed to `requires_human` no longer parks at "needs a person" while its checks are green; the contributor lane hands bot PRs off as `trusted-bot-lane` instead of `contributor-lane:app-authored`; every self-authored-sweep skip/hold line carries `tier`; and when `auto_merge.required_checks` names a check a repo never runs, the lane gates on the checks that do report (warning once per repo per hour) or holds with `required-check-never-reported:`, which the Activity Monitor shows as the human-gate reason (#17577). + +## 2026-11-10 (v5.211.20) + +### Changed + +- CI mutation report (`src/scripts/mutation-report.sh`): the `` PR comment now also lists NOT COVERED mutants in changed files and tells a zero-mutant run apart from a broken one — a SKIPPED mutant on an added line gremlins honours (it only matches the first `lines added` lines of each default-context hunk; the rest are noted, not flagged) is flagged as a diff-matching fault, a gremlins failure or missing report is an `error` verdict rather than a silent pass, while added lines with nothing gremlins can mutate (a unary `!`, a bare `return`) are reported as such instead of "no surviving mutants". A new `verdict` phase exits non-zero on `error`/`lived`/`misrouted` or a missing verdict file and warns on `uncovered` unless the PR carries `mutation/acknowledged`, so a workflow can block on it (#17349, Phase 2). + +## 2026-11-10 (v5.211.19) + +### Fixed + +- Activity monitor: **Release hold** and **Confirm safe** on an issue row drawn through its covering PR now act on that PR, which carries the hold or sentinel, instead of silently no-oping on the issue; a release that finds no hold on an issue whose covering PR is held now says so and names the PR. + ## 2026-11-10 (v5.211.18) ### Fixed diff --git a/NOTICE b/NOTICE index a2b84b2299..03e7b81219 100644 --- a/NOTICE +++ b/NOTICE @@ -28,9 +28,9 @@ release (see src/docs/releases.md, "Software bill of materials (SBOM)"). ----------------------------------------------------------------------------- Package: cel.dev/expr -Version: v0.25.2 +Version: v0.25.3 License: Apache-2.0 -Source: https://github.com/cel-expr/cel-spec/blob/v0.25.2/LICENSE +Source: https://github.com/cel-expr/cel-spec/blob/v0.25.3/LICENSE Apache License @@ -237,6 +237,217 @@ Source: https://github.com/cel-expr/cel-spec/blob/v0.25.2/LICENSE ----------------------------------------------------------------------------- +Package: github.com/ai-openshell/sdk-go +Version: v0.1.8 +License: Apache-2.0 +Source: https://github.com/ai-openshell/sdk-go/blob/v0.1.8/LICENSE + +Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +----------------------------------------------------------------------------- + Package: github.com/antlr4-go/antlr/v4 Version: v4.13.1 License: BSD-3-Clause @@ -6266,9 +6477,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ----------------------------------------------------------------------------- Package: golang.org/x/oauth2 -Version: v0.36.0 +Version: v0.37.0 License: BSD-3-Clause -Source: https://cs.opensource.google/go/x/oauth2/+/v0.36.0:LICENSE +Source: https://cs.opensource.google/go/x/oauth2/+/v0.37.0:LICENSE Copyright 2009 The Go Authors. @@ -6300,7 +6511,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ----------------------------------------------------------------------------- -Package: golang.org/x/sync/errgroup +Package: golang.org/x/sync Version: v0.24.0 License: BSD-3-Clause Source: https://cs.opensource.google/go/x/sync/+/v0.24.0:LICENSE @@ -6650,7 +6861,7 @@ Source: https://github.com/googleapis/go-genproto/blob/8a89bd6388cc/googleapis ----------------------------------------------------------------------------- -Package: google.golang.org/genproto/googleapis/rpc/status +Package: google.golang.org/genproto/googleapis/rpc Version: v0.0.0-20260928230214-8a89bd6388cc License: Apache-2.0 Source: https://github.com/googleapis/go-genproto/blob/8a89bd6388cc/googleapis/rpc/LICENSE @@ -6861,9 +7072,9 @@ Source: https://github.com/googleapis/go-genproto/blob/8a89bd6388cc/googleapis ----------------------------------------------------------------------------- Package: google.golang.org/grpc -Version: v1.83.2 +Version: v1.85.0 License: Apache-2.0 -Source: https://github.com/grpc/grpc-go/blob/v1.83.2/LICENSE +Source: https://github.com/grpc/grpc-go/blob/v1.85.0/LICENSE Apache License diff --git a/changelog.d/added-workflowmodel-marketplace.md b/changelog.d/added-workflowmodel-marketplace.md new file mode 100644 index 0000000000..bae3f3071c --- /dev/null +++ b/changelog.d/added-workflowmodel-marketplace.md @@ -0,0 +1 @@ +- **workflow models:** RFC phase 7 — signed bundles (`pkg/workflowmodel/bundle`: model + compiled profile + sha256; import always re-verifies and recompiles locally), a hub marketplace catalogue (`/api/marketplace/*`: immutable `name@version` entries, ETag index, tiers built-in · curated · ingested-unreviewed · local-published, nightly/on-demand refresh of the embedded CNCF corpus publishing `ingested-YYYYMMDD.N`, curated versions only flagged `newerIngestAvailable`), spoke import/publish/proxy routes with `workflow.marketplace.{enabled,url,refresh_interval}`, and a Settings → Workflows → Marketplace view with Import, Publish and Refresh catalogue ([#17620](https://github.com/hivecommons/hive/issues/17620)) diff --git a/changelog.d/changed-v6-topup-from-v5-20261111.md b/changelog.d/changed-v6-topup-from-v5-20261111.md new file mode 100644 index 0000000000..bd575666ed --- /dev/null +++ b/changelog.d/changed-v6-topup-from-v5-20261111.md @@ -0,0 +1 @@ +- v6 carries every v5 change through v5.220.0: the workflow-models RFC ([#17620](https://github.com/hivecommons/hive/issues/17620)) phases 1–7 — schema, compiler, store and repo bindings, the Settings → Workflows tab with visual editor, versions/diff/confirm, simulate, intake/cluster/spec ingest, safe label GC with model-defined human-gate replies, and signed bundles with a hub marketplace catalogue — plus OpenShell as a third `sandbox.runtime` (ADR-0026 phase 1, with cluster preflight and the dashboard terminal pane), an upstream corporate forward proxy for selected TLS egress, per-PR benchmark evidence export and opt-in publishing, the trusted-bot automerge lane honouring every configured bot, the GitHub GraphQL mutation gate no longer failing open on same-line prefixes, and the mutation report's verdict phase and wall-clock budget. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 22f41fe5fb..9ecc27e630 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -87,6 +87,10 @@ { "name": "Workflows", "description": "Workflow models (RFC hivecommons/hive#17620): the spoke-local library of ingested/authored workflow definitions, per-repository bindings, and compilation. Writes are owner-gated and audited (workflow_save, workflow_delete, workflow_bind, workflow_unbind). Every route answers 503 when the hive has no workflow store." + }, + { + "name": "Strategy Lab", + "description": "Nous experiment engine" } ], "paths": { @@ -27114,6 +27118,67 @@ } } }, + "/api/benchmarks/export": { + "get": { + "tags": [ + "System" + ], + "summary": "Export this hive's anonymized benchmark records", + "description": "Owner session. Folds the review evidence bundles under /data/evidence into hivecommons/benchmarks schema records (one per PR and agent role): model, harness, sandbox runtime, outcome, time-to-merge, ACMM level, keyed by this hive's salted hash. Repository and org names appear only for public repositories unless `benchmarks.repo_consent: false`. Returns `records: []` when `benchmarks.publish: false`. The same records are pushed hourly to the hub (hivecommons/hive#17694).", + "parameters": [ + { + "name": "since", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "RFC3339 or YYYY-MM-DD; PRs whose newest bundle is older are skipped. Default: 90 days ago." + } + ], + "responses": { + "200": { + "description": "Benchmark records", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hive_hash": { + "type": "string" + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "publish": { + "type": "boolean" + }, + "since": { + "type": "string", + "format": "date-time" + }, + "count": { + "type": "integer" + }, + "records": { + "type": "array", + "items": { + "type": "object", + "description": "hivecommons/benchmarks schema/result.schema.json record" + } + } + } + } + } + } + }, + "400": { + "description": "Bad since" + } + } + } + }, "/api/evidence/summary": { "get": { "tags": [ @@ -39100,6 +39165,116 @@ } } }, + "/api/workflows/yaml": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Convert a workflow model between JSON and YAML", + "description": "Editor helper. Send `model` to get its YAML (field names are the JSON tags), or `yaml` to get the parsed model back. A YAML parse or schema error answers 200 with `error` set so the editor can show it inline. Read-only; no store or owner role needed.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "model": { + "$ref": "#/components/schemas/WorkflowModel" + }, + "yaml": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "yaml, or model (or error) depending on the input", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "yaml": { + "type": "string" + }, + "model": { + "$ref": "#/components/schemas/WorkflowModel" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "Neither model nor yaml supplied, or malformed body" + } + } + } + }, + "/api/workflows/simulate": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Simulate an issue or PR against an unsaved workflow model", + "description": "Editor variant of /api/workflows/{name}/{version}/simulate: the model travels in the body and `facts` are required (no forge fetch). Same decision and trace shape. Read-only; no store or owner role needed.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "model", + "kind", + "facts" + ], + "properties": { + "model": { + "$ref": "#/components/schemas/WorkflowModel" + }, + "kind": { + "type": "string", + "enum": [ + "issue", + "pr" + ] + }, + "facts": { + "$ref": "#/components/schemas/WorkflowSimulateFacts" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Decision, trace and normalized facts", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowSimulateResult" + } + } + } + }, + "400": { + "description": "Missing model, kind or facts" + }, + "422": { + "description": "Model does not compile; body carries diagnostics" + } + } + } + }, "/api/workflows/bindings": { "get": { "tags": [ @@ -39357,6 +39532,313 @@ } } }, + "/api/workflows/{name}/versions": { + "get": { + "tags": [ + "Workflows" + ], + "summary": "List every saved version of a workflow model", + "description": "Newest save first. hive-default answers its single virtual, read-only entry.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Versions", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "versions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WorkflowEntry" + } + } + } + } + } + } + }, + "404": { + "description": "Unknown model" + }, + "503": { + "description": "No workflow store" + } + } + } + }, + "/api/workflows/{name}/diff": { + "get": { + "tags": [ + "Workflows" + ], + "summary": "Structural diff between two saved workflow model versions", + "description": "Compares spec columns, states (regions flattened), transitions (keyed from>to), labels, derivation rules and meta. from/to are versions of {name}; fromName/toName override the model on either side so a local model can be compared against hive-default.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "from", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "to", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "fromName", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "toName", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Diff", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "from": { + "$ref": "#/components/schemas/WorkflowEntry" + }, + "to": { + "$ref": "#/components/schemas/WorkflowEntry" + }, + "diff": { + "$ref": "#/components/schemas/WorkflowModelDiff" + } + } + } + } + } + }, + "400": { + "description": "from or to missing, or bad name/version" + }, + "404": { + "description": "Either side not found" + }, + "503": { + "description": "No workflow store" + } + } + } + }, + "/api/workflows/{name}/{version}/confirm": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Confirm an ingested model's inferred elements as a new local version", + "description": "Owner only. Copies {name}@{version} with every inferred state, event and gate marked confirmed, provenance set to local and metadata.annotations[workflow.hivecommons.dev/confirmed-by] set to the caller, then saves it as the next free integer version. The source version is untouched. Audited as workflow_confirm.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "Confirmed copy stored", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "entry": { + "$ref": "#/components/schemas/WorkflowEntry" + }, + "confirmed": { + "type": "integer", + "description": "Inferred elements that were confirmed" + }, + "from": { + "type": "string", + "description": "Source name@version" + } + } + } + } + } + }, + "400": { + "description": "Bad name or version" + }, + "403": { + "description": "Not an owner" + }, + "404": { + "description": "Not found" + }, + "409": { + "description": "hive-default is built-in and read-only" + }, + "503": { + "description": "No workflow store" + } + } + } + }, + "/api/workflows/{name}/{version}/labels": { + "get": { + "tags": [ + "Workflows" + ], + "summary": "Preview the labels and commands a workflow model regenerates when bound", + "description": "The label vocabulary the engine reconciles into a bound repository (the model's labels block, stage colour filled in where unset, descriptions clamped), the human-gate verbs each state offers, and the built-in stage label each numbered state maps onto.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Preview", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowLabelsPreview" + } + } + } + }, + "400": { + "description": "Model does not compile; body carries diagnostics" + }, + "404": { + "description": "Not found" + }, + "503": { + "description": "No workflow store" + } + } + } + }, + "/api/workflows/{name}/{version}/simulate": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Simulate an issue or PR against a workflow model", + "description": "Derives the state a stored model would put an item in, using the same stageengine Facts the engine uses, and returns a per-rule trace explaining why. Send `facts` directly, or `repo`+`number` to best-effort fill them from the forge (the response lists which facts were filled and which stay unknown). Read-only; RFC phase 3c of the workflow-models RFC.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowSimulateRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Decision, trace and normalized facts", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowSimulateResult" + } + } + } + }, + "400": { + "description": "Malformed body, unknown kind, or kind does not match the fetched item" + }, + "404": { + "description": "Not found" + }, + "422": { + "description": "Model does not compile, or no forge client is wired for repo+number fill" + }, + "502": { + "description": "Forge fetch failed" + }, + "503": { + "description": "No workflow store" + } + } + } + }, "/api/workflows/ingest": { "post": { "tags": [ @@ -39429,6 +39911,525 @@ } } } + }, + "/api/workflows/ingest/capabilities": { + "get": { + "tags": [ + "Workflows" + ], + "summary": "What the ingest sources can do on this spoke", + "description": "Whether a kube context is reachable for POST /api/workflows/ingest/cluster (kubectl on PATH with a kubeconfig or in-cluster token), the contexts kubectl knows, and whether a model gateway is configured for the optional LLM stage of POST /api/workflows/ingest/spec. RFC hivecommons/hive#17620 phase 6. Owner role required.", + "responses": { + "200": { + "description": "Capabilities", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowIngestCapabilities" + } + } + } + }, + "403": { + "description": "Owner role required" + } + } + } + }, + "/api/workflows/ingest/cluster": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Ingest Tekton pipelines from a live cluster context", + "description": "Lists tekton.dev Pipelines, Tasks, the newest PipelineRuns per pipeline and Tekton Triggers objects from a kube context through kubectl, serialises them as cluster///.yaml and runs the same Tekton ingester as POST /api/workflows/ingest, so citations point at those virtual paths. The last run status per pipeline is recorded as an observed annotation (workflow.hivecommons.dev/observed-run/) and a coverage finding, never as an inferred lifecycle element. Caps: 25 namespaces, 20 runs per pipeline, 500 objects, 8 MiB. Owner role required.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "context": { + "type": "string", + "description": "kube context name (default: the current context)" + }, + "namespace": { + "type": "string", + "description": "namespace to read; required unless allNamespaces" + }, + "allNamespaces": { + "type": "boolean", + "description": "read every namespace (capped at 25)" + }, + "selector": { + "type": "string", + "description": "label selector passed to kubectl -l" + }, + "runsPerPipeline": { + "type": "integer", + "description": "newest PipelineRuns kept per pipeline (default 3, max 20)" + }, + "repository": { + "type": "string", + "description": "owner/name to scope the model to (default: cluster/)" + }, + "save": { + "type": "boolean" + }, + "name": { + "type": "string", + "description": "store name (default: slug of the source)" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Ingested (not saved)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowClusterIngestResult" + } + } + } + }, + "201": { + "description": "Ingested and saved", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowClusterIngestResult" + } + } + } + }, + "400": { + "description": "Bad request, or namespace required" + }, + "403": { + "description": "Owner role required, or the kube credential may not list Tekton resources" + }, + "404": { + "description": "Unknown kube context" + }, + "413": { + "description": "Object or byte cap exceeded" + }, + "422": { + "description": "No kube access configured on this spoke, or Tekton CRDs not installed" + }, + "502": { + "description": "kubectl failed" + }, + "503": { + "description": "No workflow store" + } + } + } + }, + "/api/workflows/ingest/spec": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Ingest a written process spec into a draft workflow model", + "description": "Turns markdown or plain text (pasted, a default-branch file through the forge, or an https URL on a public host; 256 KiB cap) into a draft model. A deterministic stage reads step lists, conditional sentences and role nouns into a compiling skeleton; when useLLM is on (default: when a model gateway is configured) the configured gateway refines it and its answer is accepted only if it parses strictly and compiles, with one retry quoting the diagnostics before falling back to the skeleton. Every state, event and gate is confidence inferred and cites spec:#L-L; provenance is ingested-unreviewed; the LLM-assisted annotation names the model used. This is the only LLM use in the workflow-model stack. Owner role required.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "pasted spec; exactly one of text, repo+path or url" + }, + "repo": { + "type": "string", + "description": "owner/name of the repository holding the spec" + }, + "path": { + "type": "string", + "description": "repo-relative path of the spec file" + }, + "url": { + "type": "string", + "description": "https URL of the spec" + }, + "useLLM": { + "type": "boolean", + "description": "ask the configured model gateway to refine the draft (default: true when a gateway is configured); 422 when requested without one" + }, + "repository": { + "type": "string", + "description": "owner/name to scope the model to (default: repo, or spec/)" + }, + "save": { + "type": "boolean" + }, + "name": { + "type": "string", + "description": "store name (default: slug of the first heading)" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Ingested (not saved)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowSpecIngestResult" + } + } + } + }, + "201": { + "description": "Ingested and saved", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowSpecIngestResult" + } + } + } + }, + "400": { + "description": "Bad request, empty spec, or URL not https / private host" + }, + "403": { + "description": "Owner role required" + }, + "413": { + "description": "Spec exceeds 256 KiB" + }, + "422": { + "description": "useLLM requested but no model gateway is configured" + }, + "502": { + "description": "Spec fetch failed" + }, + "503": { + "description": "No forge client or no workflow store" + } + } + } + }, + "/api/workflows/{name}/{version}/bundle": { + "get": { + "tags": [ + "Workflows" + ], + "summary": "Export a stored workflow model as a signed bundle", + "description": "Compiles name@version and wraps model, compiled profile and the sha256 of the canonical model JSON in a WorkflowBundle (RFC #17620 phase 7). Any role. 422 when the model does not compile.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Bundle", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowBundle" + } + } + } + }, + "404": { + "description": "Not found" + }, + "422": { + "description": "Model does not compile" + }, + "503": { + "description": "Workflow store not enabled" + } + } + } + }, + "/api/workflows/import": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Import a workflow bundle", + "description": "Verifies the bundle (hash over the canonical model, local recompile compared structurally with the published profile) and stores the model. The version is pinned to the bundle's when that slot is free, else the next integer version. The store recompiles; the published profile is never persisted. Owner role; audited as workflow_import.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowBundle" + } + } + } + }, + "responses": { + "201": { + "description": "Imported and recompiled locally", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowImportResult" + } + } + } + }, + "422": { + "description": "Bundle verification failed (hash or published profile does not match local compile)" + }, + "400": { + "description": "Bad request" + }, + "403": { + "description": "Owner role required" + }, + "503": { + "description": "Workflow store or marketplace not enabled" + }, + "413": { + "description": "Bundle too large" + } + } + } + }, + "/api/marketplace/workflows": { + "get": { + "tags": [ + "Workflows" + ], + "summary": "Marketplace catalogue index (proxied from the hub)", + "description": "Returns the hub's catalogue index. The spoke caches it for workflow.marketplace.refresh_interval and revalidates with If-None-Match; ?refresh=1 forces revalidation. When the hub is unreachable the response is 502 with the last cached index and stale=true.", + "parameters": [ + { + "name": "refresh", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "any value forces revalidation with the hub" + } + ], + "responses": { + "200": { + "description": "Index", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MarketplaceIndexResponse" + } + } + } + }, + "404": { + "description": "Marketplace disabled (workflow.marketplace.enabled=false)" + }, + "502": { + "description": "Hub unreachable; body carries the cached index (possibly empty) with stale=true", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MarketplaceIndexResponse" + } + } + } + }, + "503": { + "description": "No hub URL configured" + } + } + } + }, + "/api/marketplace/workflows/{name}/{version}/import": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Import a catalogue entry from the hub", + "description": "Fetches name@version's bundle from the hub, verifies it locally and stores it exactly as POST /api/workflows/import does. Owner role; audited as workflow_import.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "Imported and recompiled locally", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowImportResult" + } + } + } + }, + "422": { + "description": "Bundle verification failed (hash or published profile does not match local compile)" + }, + "403": { + "description": "Owner role required" + }, + "404": { + "description": "Not in the marketplace or marketplace disabled" + }, + "502": { + "description": "Hub unreachable or returned an invalid bundle" + }, + "503": { + "description": "Workflow store not enabled or no hub URL" + } + } + } + }, + "/api/marketplace/workflows/publish": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Publish a library entry to the hub marketplace", + "description": "Exports name@version as a bundle (tier local-published unless the model is ingested-unreviewed) and posts it to the hub with the spoke's heartbeat credential. Owner role; audited as workflow_publish.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "name", + "version" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "201": { + "description": "Published", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "published": { + "type": "string", + "description": "name@version" + }, + "hash": { + "type": "string" + }, + "tier": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "Bad request" + }, + "403": { + "description": "Owner role required, or the hub refused the spoke credential" + }, + "404": { + "description": "Entry not found" + }, + "409": { + "description": "name@version already published with a different hash" + }, + "422": { + "description": "Model does not compile or hub rejected the bundle" + }, + "502": { + "description": "Hub unreachable" + }, + "503": { + "description": "Workflow store not enabled or no hub URL" + } + } + } + }, + "/api/marketplace/refresh": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Ask the hub to refresh its ingested catalogue", + "description": "Proxies POST /api/marketplace/refresh on the hub (optionally ?repo=owner/name). Owner role on the spoke; the hub applies its own admin policy. Audited as workflow_marketplace_refresh.", + "parameters": [ + { + "name": "repo", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "202": { + "description": "Refresh accepted by the hub" + }, + "403": { + "description": "Owner role required or hub refused" + }, + "404": { + "description": "Marketplace disabled" + }, + "502": { + "description": "Hub unreachable or failed" + }, + "503": { + "description": "No hub URL" + } + } + } } }, "components": { @@ -39864,6 +40865,731 @@ "$ref": "#/components/schemas/WorkflowEntry" } } + }, + "WorkflowSimulateRequest": { + "type": "object", + "description": "Either `facts` (with `kind`) or `repo`+`number` must be given. When both are present `facts` wins and `repo`/`number` only seed the item reference.", + "properties": { + "kind": { + "type": "string", + "enum": [ + "issue", + "pr" + ] + }, + "repo": { + "type": "string", + "description": "owner/repo" + }, + "number": { + "type": "integer" + }, + "facts": { + "$ref": "#/components/schemas/WorkflowSimulateFacts" + } + } + }, + "WorkflowSimulateFacts": { + "type": "object", + "description": "JSON mirror of stageengine.Facts. Enumerations are the engine's name helpers: author.kind hive|clanker|human|unknown; pr.checks green|red|pending|none|unknown; pr.review approved|changes_requested|pending|unavailable|requires_human; pr.dco yes|no|unknown.", + "properties": { + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "open": { + "type": "boolean" + }, + "merged": { + "type": "boolean" + }, + "draft": { + "type": "boolean" + }, + "origin": { + "type": "string" + }, + "author": { + "type": "object", + "properties": { + "login": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "selfAuthored": { + "type": "boolean" + }, + "role": { + "type": "string" + } + } + }, + "park": { + "type": "object", + "properties": { + "holds": { + "type": "array", + "items": { + "type": "string" + } + }, + "needsHuman": { + "type": "boolean" + }, + "decisionLabel": { + "type": "string" + }, + "possiblyExists": { + "type": "boolean" + }, + "sentinel": { + "type": "boolean" + }, + "humanOnlyReason": { + "type": "string" + }, + "approvedDirection": { + "type": "boolean" + }, + "decisionQuestion": { + "type": "string" + }, + "decisionSuperseded": { + "type": "boolean" + } + } + }, + "issue": { + "type": "object", + "additionalProperties": true, + "properties": { + "bucket": { + "type": "string" + } + } + }, + "pr": { + "type": "object", + "additionalProperties": true, + "properties": { + "checks": { + "type": "string" + }, + "review": { + "type": "string" + }, + "dco": { + "type": "string" + }, + "conflicting": { + "type": "boolean" + }, + "changesRequested": { + "type": "boolean" + }, + "humanApprover": { + "type": "boolean" + }, + "lane": { + "type": "object", + "properties": { + "lane": { + "type": "string" + }, + "reason": { + "type": "string" + } + } + } + } + } + } + }, + "WorkflowSimulateResult": { + "type": "object", + "properties": { + "decision": { + "type": "object", + "properties": { + "state": { + "type": "string" + }, + "column": { + "type": "string" + }, + "stage": { + "type": "integer" + }, + "label": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "abstain": { + "type": "boolean" + }, + "covered": { + "type": "boolean" + }, + "terminal": { + "type": "boolean" + } + } + }, + "trace": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "state": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "evaluated": { + "type": "boolean" + }, + "matched": { + "type": "boolean" + }, + "skipped": { + "type": "string" + }, + "builtin": { + "type": "boolean" + }, + "reason": { + "type": "string" + }, + "predicates": { + "type": "array", + "items": { + "type": "object", + "properties": { + "fact": { + "type": "string" + }, + "op": { + "type": "string" + }, + "value": { + "type": "string" + }, + "holds": { + "type": "boolean" + }, + "known": { + "type": "boolean" + }, + "actual": { + "type": "string" + } + } + } + } + } + } + }, + "facts": { + "$ref": "#/components/schemas/WorkflowSimulateFacts" + }, + "filled": { + "type": "array", + "items": { + "type": "string" + } + }, + "unknown": { + "type": "array", + "items": { + "type": "string" + } + }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "WorkflowFieldChange": { + "type": "object", + "properties": { + "field": { + "type": "string" + }, + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + } + }, + "WorkflowChangeSet": { + "type": "object", + "properties": { + "added": { + "type": "array", + "items": { + "type": "string" + } + }, + "removed": { + "type": "array", + "items": { + "type": "string" + } + }, + "changed": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "fields": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WorkflowFieldChange" + } + } + } + } + } + } + }, + "WorkflowModelDiff": { + "type": "object", + "description": "Structural difference between two models; deterministic, lists sorted by key.", + "properties": { + "columns": { + "$ref": "#/components/schemas/WorkflowChangeSet" + }, + "states": { + "$ref": "#/components/schemas/WorkflowChangeSet" + }, + "transitions": { + "$ref": "#/components/schemas/WorkflowChangeSet" + }, + "labels": { + "$ref": "#/components/schemas/WorkflowChangeSet" + }, + "rules": { + "$ref": "#/components/schemas/WorkflowChangeSet" + }, + "meta": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WorkflowFieldChange" + } + }, + "summary": { + "type": "string" + }, + "empty": { + "type": "boolean" + } + } + }, + "WorkflowLabelDef": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "color": { + "type": "string" + }, + "description": { + "type": "string" + }, + "state": { + "type": "string" + } + } + }, + "WorkflowLabelsPreview": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "hash": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "labels": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WorkflowLabelDef" + } + }, + "commands": { + "type": "array", + "items": { + "type": "object", + "properties": { + "state": { + "type": "string" + }, + "verbs": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "stages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "state": { + "type": "string" + }, + "number": { + "type": "integer" + }, + "label": { + "type": "string" + } + } + } + } + } + }, + "WorkflowIngestCapabilities": { + "type": "object", + "properties": { + "cluster": { + "type": "boolean", + "description": "kube access is configured on this spoke" + }, + "clusterReason": { + "type": "string", + "description": "why cluster is false, or why contexts could not be listed" + }, + "contexts": { + "type": "array", + "items": { + "type": "string" + } + }, + "spec": { + "type": "object", + "properties": { + "llm": { + "type": "boolean", + "description": "a model gateway is configured for the optional LLM stage" + }, + "model": { + "type": "string", + "description": "gateway model id the LLM stage would use" + } + } + } + } + }, + "WorkflowClusterIngestResult": { + "allOf": [ + { + "$ref": "#/components/schemas/WorkflowIngestResult" + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "cluster:/" + }, + "runs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "pipeline": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "run": { + "type": "string" + }, + "status": { + "type": "string" + }, + "completed": { + "type": "string" + } + } + } + }, + "namespaces": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + ] + }, + "WorkflowSpecIngestResult": { + "type": "object", + "properties": { + "model": { + "$ref": "#/components/schemas/WorkflowModel" + }, + "compiled": { + "type": "boolean" + }, + "diagnostics": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WorkflowDiagnostic" + } + }, + "profile": { + "type": "object" + }, + "source": { + "type": "string", + "description": "spec:" + }, + "llm": { + "type": "object", + "properties": { + "used": { + "type": "boolean" + }, + "model": { + "type": "string" + }, + "attempts": { + "type": "integer" + }, + "accepted": { + "type": "boolean", + "description": "the returned model is the gateway's answer (false: deterministic skeleton)" + }, + "error": { + "type": "string" + } + } + }, + "sections": { + "type": "array", + "items": { + "type": "object", + "properties": { + "heading": { + "type": "string" + }, + "level": { + "type": "integer" + }, + "startLine": { + "type": "integer" + }, + "endLine": { + "type": "integer" + } + } + } + }, + "entry": { + "$ref": "#/components/schemas/WorkflowEntry" + } + } + }, + "WorkflowBundle": { + "type": "object", + "description": "Portable workflow model envelope: model + the publisher's compiled profile + sha256 of the canonical model JSON. Importers verify the hash and recompile locally; the published profile is only ever compared, never trusted.", + "required": [ + "apiVersion", + "model", + "profile", + "hash" + ], + "properties": { + "apiVersion": { + "type": "string", + "enum": [ + "workflow.hivecommons.dev/v1alpha1" + ] + }, + "kind": { + "type": "string", + "enum": [ + "WorkflowBundle" + ] + }, + "model": { + "$ref": "#/components/schemas/WorkflowModel" + }, + "profile": { + "type": "object", + "description": "compiled profile (see WorkflowProfile)" + }, + "hash": { + "type": "string", + "description": "sha256 hex of the canonical (sorted-key) model JSON" + }, + "createdAt": { + "type": "string", + "format": "date-time" + }, + "createdBy": { + "type": "string" + }, + "tier": { + "type": "string", + "enum": [ + "built-in", + "curated", + "ingested-unreviewed", + "local", + "local-published" + ] + } + } + }, + "WorkflowImportResult": { + "type": "object", + "properties": { + "entry": { + "$ref": "#/components/schemas/WorkflowEntry" + }, + "verified": { + "type": "boolean" + }, + "source": { + "type": "string", + "description": "name@version in the marketplace, for marketplace imports" + } + } + }, + "MarketplaceIndexEntry": { + "type": "object", + "description": "One immutable catalogue row, keyed name@version plus hash.", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "hash": { + "type": "string" + }, + "tier": { + "type": "string", + "enum": [ + "built-in", + "curated", + "ingested-unreviewed", + "local", + "local-published" + ] + }, + "description": { + "type": "string" + }, + "source": { + "type": "string", + "description": "repository or author the model came from" + }, + "states": { + "type": "integer" + }, + "columns": { + "type": "integer" + }, + "publishedAt": { + "type": "string", + "format": "date-time" + }, + "publishedBy": { + "type": "string" + }, + "sourceSHA": { + "type": "string", + "description": "commit the model was ingested from" + }, + "newerIngestAvailable": { + "type": "boolean", + "description": "set on curated entries when a later ingested-unreviewed version exists" + }, + "adoption": { + "type": "object", + "description": "reserved; empty today" + }, + "ratings": { + "type": "object", + "description": "reserved; empty today" + }, + "benchmarks": { + "type": "object", + "description": "reserved; empty today" + } + } + }, + "MarketplaceIndex": { + "type": "object", + "properties": { + "generatedAt": { + "type": "string", + "format": "date-time" + }, + "entries": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MarketplaceIndexEntry" + } + } + } + }, + "MarketplaceIndexResponse": { + "type": "object", + "properties": { + "index": { + "$ref": "#/components/schemas/MarketplaceIndex" + }, + "stale": { + "type": "boolean", + "description": "true when served from cache because the hub was unreachable" + }, + "etag": { + "type": "string" + }, + "fetchedAt": { + "type": "string", + "format": "date-time" + }, + "error": { + "type": "string" + } + } } } } diff --git a/docs/weekly-competitor-report.md b/docs/weekly-competitor-report.md index 058bfd6ea8..ff37e186f5 100644 --- a/docs/weekly-competitor-report.md +++ b/docs/weekly-competitor-report.md @@ -43,8 +43,13 @@ unchanged. ## Publication gate -The report uses identical visible text for GitHub and Discord. The GitHub body -also ends with a fenced JSON provenance ledger. HTML comments cannot carry this +The report carries the same visible text on GitHub and Discord. Discord gets +the formatting the prompt asks for (bold headings instead of `#`, the "Who does +what" table as a fenced code block, `` links, no "Sources and queries" +details), so the validator compares the prose after normalising those +differences and checks the Discord table by project name and table notes rather +than byte for byte. The GitHub body also ends with a fenced JSON provenance +ledger. HTML comments cannot carry this ledger because gh-aw removes hidden comments. Exact source quotes travel as base64 UTF-8 data so URL, mention and template sanitization cannot alter quoted bytes; no decoded content is executed. diff --git a/src/Dockerfile b/src/Dockerfile index 92afd97605..1b81c1bd67 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -356,9 +356,9 @@ RUN npm install -g --ignore-scripts --no-fund --no-audit @anthropic-ai/claude-co # Installs from the OFFICIAL aaif-goose/goose repo, pinned to a fixed release and # SHA-256-verified before extraction. Download failures are tolerated because # Goose is an optional backend; checksum mismatches must hard-fail the build. -ARG GOOSE_VERSION=1.58.1 -ARG GOOSE_SHA256_AMD64=43adae6abf10d40d5b1d5cd23bcf6481ef72fddede8fe9a8577c385e325cba68 -ARG GOOSE_SHA256_ARM64=20d9586ac7d37d2700d41d85d0051547a959ae96576aae6cbd290f77b5605a61 +ARG GOOSE_VERSION=1.59.0 +ARG GOOSE_SHA256_AMD64=a774667de6c20206a60d78d7bece2d9dda8389667e4f5c2fc8aaa758d1e81ef3 +ARG GOOSE_SHA256_ARM64=ba96b78394e3050077482bb285ff35f34613646e90b84a18b5d674b71905166e RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ case "$ARCH" in \ amd64) GOOSE_ARCH=x86_64; GOOSE_SHA256="$GOOSE_SHA256_AMD64" ;; \ diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 21d2d2701c..69e47ba387 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -389,9 +389,9 @@ RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \ # build time (sec-check issue #2919). Pin to a specific release tag and verify # the tarball before extraction. Download failures are tolerated because Goose # is an optional contributor backend; checksum mismatches are not tolerated. -ARG GOOSE_VERSION=1.58.1 -ARG GOOSE_SHA256_AMD64=43adae6abf10d40d5b1d5cd23bcf6481ef72fddede8fe9a8577c385e325cba68 -ARG GOOSE_SHA256_ARM64=20d9586ac7d37d2700d41d85d0051547a959ae96576aae6cbd290f77b5605a61 +ARG GOOSE_VERSION=1.59.0 +ARG GOOSE_SHA256_AMD64=a774667de6c20206a60d78d7bece2d9dda8389667e4f5c2fc8aaa758d1e81ef3 +ARG GOOSE_SHA256_ARM64=ba96b78394e3050077482bb285ff35f34613646e90b84a18b5d674b71905166e RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64) GOOSE_ARCH=x86_64; GOOSE_SHA256="$GOOSE_SHA256_AMD64" ;; \ diff --git a/src/cmd/hive/delegatedecider.go b/src/cmd/hive/delegatedecider.go index d2a5e45541..fcd45594a9 100644 --- a/src/cmd/hive/delegatedecider.go +++ b/src/cmd/hive/delegatedecider.go @@ -26,6 +26,23 @@ type delegateDeciderRunner struct { // model returns the per-feature model override ("" keeps the reviewer // model); nil means the "Let hive decide" override. model func(*config.Config) string + // maxTokens caps the completion; zero means delegateDeciderMaxTokens. + maxTokens int +} + +// workflowSpecMaxTokens fits a whole WorkflowModel JSON document. +const workflowSpecMaxTokens = 8000 + +// newWorkflowSpecRunner is the same reviewer-endpoint turn for the optional +// LLM refinement of written-spec ingest (POST /api/workflows/ingest/spec, +// RFC #17620 phase 6): the reviewer model, a larger completion budget. +func newWorkflowSpecRunner(cfg func() *config.Config) delegatedecision.TurnRunner { + return &delegateDeciderRunner{ + cfg: cfg, + http: &http.Client{Timeout: delegateDeciderTimeout}, + model: func(*config.Config) string { return "" }, + maxTokens: workflowSpecMaxTokens, + } } const ( @@ -92,7 +109,7 @@ func (r *delegateDeciderRunner) RunDeciderTurn(ctx context.Context, profile, sys Model: model, Messages: []delegateDeciderMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, Temperature: 0, - MaxTokens: delegateDeciderMaxTokens, + MaxTokens: r.completionBudget(), }) if err != nil { return "", fmt.Errorf("%s: marshal request: %w", profile, err) @@ -128,3 +145,10 @@ func (r *delegateDeciderRunner) RunDeciderTurn(ctx context.Context, profile, sys } return cr.Choices[0].Message.Content, nil } + +func (r *delegateDeciderRunner) completionBudget() int { + if r.maxTokens > 0 { + return r.maxTokens + } + return delegateDeciderMaxTokens +} diff --git a/src/cmd/hive/delegatedecider_test.go b/src/cmd/hive/delegatedecider_test.go index bc30cd1ebd..1df14d950b 100644 --- a/src/cmd/hive/delegatedecider_test.go +++ b/src/cmd/hive/delegatedecider_test.go @@ -61,3 +61,27 @@ func TestDelegateDeciderRunner(t *testing.T) { t.Fatalf("nil config: %v", err) } } + +func TestWorkflowSpecRunnerUsesLargerBudget(t *testing.T) { + t.Setenv(config.LiteLLMEndpointEnv, "") + var got delegateDeciderRequest + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&got) + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{}"}}]}`)) + })) + t.Cleanup(srv.Close) + + cfg := &config.Config{} + cfg.Governor.Trajectory.Endpoint = srv.URL + cfg.Governor.Trajectory.Model = "reviewer-model" + runner := newWorkflowSpecRunner(func() *config.Config { return cfg }) + if _, err := runner.RunDeciderTurn(context.Background(), delegatedecision.DeciderProfile, "sys", "usr"); err != nil { + t.Fatal(err) + } + if got.Model != "reviewer-model" || got.MaxTokens != workflowSpecMaxTokens { + t.Fatalf("request = %+v", got) + } + if b := newDelegateDeciderRunner(func() *config.Config { return cfg }).(*delegateDeciderRunner).completionBudget(); b != delegateDeciderMaxTokens { + t.Fatalf("default budget = %d", b) + } +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 62c9721d2f..e9378be1ce 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -29,6 +29,7 @@ import ( "github.com/hivecommons/hive/crux/pushbroker" "github.com/hivecommons/hive/crux/taskmcp" "github.com/hivecommons/hive/crux/trajectory" + "github.com/hivecommons/hive/crux/upstreamproxy" "github.com/hivecommons/hive/crux/watchdog" "github.com/hivecommons/hive/crux/watsonx" "github.com/hivecommons/hive/pkg/advisor" @@ -683,6 +684,13 @@ func (b *boot) bootConfigWith(deps bootConfigDeps) bool { "persisted_path", config.PersistedConfigPath(b.configPath), ) + // Upstream forward-proxy chaining (enterprise GitHub reachable only via + // a corporate proxy): resolve the rule set once from network.upstream_proxy + // / HIVE_UPSTREAM_PROXY_* / HTTPS_PROXY and hand it to every outbound + // path — control-plane GitHub clients, in-process git, and the MITM proxy + // that carries agent :443 traffic (bootProxyWith). + configureUpstreamProxy(b.logger, b.cfg) + // Load or generate a unique Hive ID for this instance b.cfg.HiveID = deps.hiveID(b.logger) _ = os.Setenv("HIVE_ID", b.cfg.HiveID) // valid key/value; Setenv cannot fail on Unix @@ -1088,8 +1096,14 @@ func (b *boot) wireBootClosures() { Config: b.cfg, WorkflowStore: wfstore.Open(wfstore.Root(b.cfg.Data.WorkflowsDir)), WorkflowRepoFiles: workflowRepoFiles(func() *github.Client { return b.ghClient }), - AgentMgr: b.agentMgr, - CruxRuntimeHealth: cruxRuntimeHealthFunc(b.cruxSupervision), + WorkflowItemSnapshot: func(ctx context.Context, repo string, number int) (github.WorkflowItemSnapshot, error) { + return b.ghClient.WorkflowItemSnapshot(ctx, repo, number) + }, + WorkflowRepoFile: workflowRepoFile(func() *github.Client { return b.ghClient }), + WorkflowClusterRunner: workflowClusterRunner(), + WorkflowSpecLLM: newWorkflowSpecRunner(func() *config.Config { return b.cfg }), + AgentMgr: b.agentMgr, + CruxRuntimeHealth: cruxRuntimeHealthFunc(b.cruxSupervision), // Provider gateways (#5565 slice 3): concrete openrouter/watsonx/ // linearagent adapters behind the dashboard's consumer-defined // interfaces — this composition root is the only non-test place that @@ -4304,6 +4318,7 @@ func (b *boot) bootProxyWith(deps bootProxyDeps) { if err != nil { b.logger.Error("failed to create github proxy", "error", err) } else { + b.githubProxy.SetUpstreamProxy(upstreamproxy.Default()) b.githubProxy.SetCanaryScanner(b.cfg.Ioscan.IsEnabled() && b.cfg.Ioscan.CanariesEnabled(), b.cfg.Ioscan.FailClosedAtLevel(b.cfg.ACMMLevelOrZero()), ioscan.DefaultCanaries, canaryLeakHandler) // REST accounting and the shared core-bucket snapshot the agent // GitHub budget reads live in pkg/github; the proxy takes them through diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index 3495e04cd1..bfda555190 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -20,6 +20,7 @@ import ( "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/escalate" "github.com/hivecommons/hive/pkg/escalation" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" "github.com/hivecommons/hive/pkg/forge" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/github/automerge" @@ -32,6 +33,7 @@ import ( "github.com/hivecommons/hive/pkg/stageengine/prstage" "github.com/hivecommons/hive/pkg/stagelabels" "github.com/hivecommons/hive/pkg/tracing" + "github.com/hivecommons/hive/pkg/workflowmodel" "github.com/hivecommons/hive/pkg/worksource" "go.opentelemetry.io/otel/attribute" ) @@ -1557,10 +1559,17 @@ func runIssueUnparkSweepIfDue(ctx context.Context, ghClient *github.Client, cfg approveOpts.TrustedBotSkipReview = func() bool { return cfg.AutoMerge.TrustedBotSkipsReview() } } approveEngine := automerge.New(ghClient, approveOpts) + // A bound workflow model's replies are accepted as `/hive ` + // (hivecommons/hive#17620); unbound repositories see no change. + var workflowFor func(owner, repo string) *workflowmodel.Profile + if cfg != nil { + workflowFor = stageEngineOptions(cfg, logger).WorkflowFor + } result, err := ghClient.SweepIssueUnparkCommands(ctx, github.IssueUnparkSweepOptions{ PRApproveReply: approveEngine.PRApproveReply, PRReplyOptions: true, MaxActions: github.DefaultIssueUnparkSweepMaxActions, + WorkflowFor: workflowFor, LabelDriven: issueUnparkLabelDriven(cfg), LabelApproval: labelApprovalPolicyResolver(cfg), ReleaseOnApprove: issueReleaseOnApprove(cfg), @@ -2075,6 +2084,11 @@ func installReviewRelaySettings(client *github.Client, cfg *config.Config, logge client.SetReviewEvidence(func(repo string) github.ReviewEvidenceSettings { return reviewEvidenceSettings(cfg, repo) }) + // hivecommons/hive#17694: stamp the sandbox runtime and benchmark task + // kind on agent-authored bundles so the exporter need not guess. + github.EvidenceAgentContext = func(agent string) (string, string) { + return cfg.BenchmarkSandboxRuntime(agent), benchmarks.TaskKindFor(agent) + } // hivecommons/hive#14581: issue-author credit on every merge path, read // live so a reload of auto_merge.credit_issue_authors applies. client.SetCreditIssueAuthors(func(string) bool { return cfg.AutoMerge.CreditIssueAuthorsEnabled() }) diff --git a/src/cmd/hive/stage_engine.go b/src/cmd/hive/stage_engine.go index 72d5f41066..6038b3da79 100644 --- a/src/cmd/hive/stage_engine.go +++ b/src/cmd/hive/stage_engine.go @@ -48,6 +48,10 @@ func stageEngineOptions(cfg *config.Config, logger *slog.Logger) *stageengine.Op Metrics: stageengine.DefaultCounters(), WorkflowFor: stageEngineWorkflowFor(cfg, wfstore.Open(wfstore.Root(cfg.Data.WorkflowsDir)), logger), } + if cfg != nil { + stageEngine.opts.WorkflowProvisioned = wfstore.Open(wfstore.Root(cfg.Data.WorkflowsDir)) + stageEngine.opts.WorkflowLabelGC = func() bool { return cfg.Workflow.LabelGCEnabled() } + } }) return stageEngine.opts } diff --git a/src/cmd/hive/upstream_proxy_boot.go b/src/cmd/hive/upstream_proxy_boot.go new file mode 100644 index 0000000000..cb80f3e387 --- /dev/null +++ b/src/cmd/hive/upstream_proxy_boot.go @@ -0,0 +1,34 @@ +package main + +import ( + "log/slog" + "os" + + "github.com/hivecommons/hive/crux/upstreamproxy" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +// configureUpstreamProxy resolves the upstream forward-proxy rule set +// (config block, HIVE_UPSTREAM_PROXY_* env, or HTTPS_PROXY fallback) and +// installs it process-wide: crux/upstreamproxy.Default for in-process git and +// the MITM proxy, pkg/github's shared transport for control-plane API calls. +// A malformed setting is logged and ignored so the hive still boots with +// direct egress rather than refusing to start. +func configureUpstreamProxy(logger *slog.Logger, cfg *config.Config) { + sel, err := upstreamproxy.Resolve(cfg.Network.UpstreamProxy.UpstreamProxy(), os.Getenv) + if err != nil { + logger.Error("boot: upstream proxy config ignored", "error", err) + sel = nil + } + upstreamproxy.SetDefault(sel) + github.SetUpstreamProxy(sel) + if sel == nil { + return + } + logger.Info("boot: upstream proxy chaining enabled", + "proxy", sel.URL().Scheme+"://"+sel.URL().Host, + "source", sel.Source(), + "restricted_hosts", sel.Restricted(), + ) +} diff --git a/src/cmd/hive/upstream_proxy_boot_test.go b/src/cmd/hive/upstream_proxy_boot_test.go new file mode 100644 index 0000000000..ff0fbfd41d --- /dev/null +++ b/src/cmd/hive/upstream_proxy_boot_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "bytes" + "log/slog" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/upstreamproxy" + "github.com/hivecommons/hive/pkg/config" +) + +func TestConfigureUpstreamProxyInstallsSelectorFromConfig(t *testing.T) { + t.Setenv("HIVE_UPSTREAM_PROXY_URL", "") + t.Setenv("HIVE_UPSTREAM_PROXY_HOSTS", "") + t.Setenv("HIVE_NO_PROXY", "") + t.Setenv("HTTPS_PROXY", "") + t.Setenv("https_proxy", "") + t.Setenv("HTTP_PROXY", "") + t.Setenv("http_proxy", "") + t.Cleanup(func() { upstreamproxy.SetDefault(nil) }) + + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + + cfg := &config.Config{} + cfg.Network.UpstreamProxy = config.UpstreamProxyConfig{ + URL: "http://user:secret@proxy.corp.example:3128", + Hosts: []string{"ghe.corp.example"}, + } + configureUpstreamProxy(logger, cfg) + + sel := upstreamproxy.Default() + if sel == nil { + t.Fatal("selector not installed") + } + if sel.ProxyFor("ghe.corp.example:443") == nil || sel.ProxyFor("api.github.com:443") != nil { + t.Fatal("selector rules do not match the config block") + } + if !strings.Contains(buf.String(), "upstream proxy chaining enabled") || strings.Contains(buf.String(), "secret") { + t.Fatalf("boot log must announce chaining without leaking credentials:\n%s", buf.String()) + } + + // Malformed input is logged and leaves egress direct instead of aborting boot. + buf.Reset() + cfg.Network.UpstreamProxy = config.UpstreamProxyConfig{URL: "socks5://nope:1080"} + configureUpstreamProxy(logger, cfg) + if upstreamproxy.Default() != nil || !strings.Contains(buf.String(), "upstream proxy config ignored") { + t.Fatalf("bad config must clear the selector and log:\n%s", buf.String()) + } + + // Nothing configured: no selector, no log line. + buf.Reset() + cfg.Network.UpstreamProxy = config.UpstreamProxyConfig{} + configureUpstreamProxy(logger, cfg) + if upstreamproxy.Default() != nil || buf.Len() != 0 { + t.Fatalf("unset config must be silent and direct:\n%s", buf.String()) + } +} diff --git a/src/cmd/hive/workflow_ingest.go b/src/cmd/hive/workflow_ingest.go index 00f80f6d45..eee09f74de 100644 --- a/src/cmd/hive/workflow_ingest.go +++ b/src/cmd/hive/workflow_ingest.go @@ -2,10 +2,14 @@ package main import ( "context" + "encoding/json" "errors" + "log" + "strings" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/workflowmodel/ingest" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/cluster" ) // errNoForgeClient is returned by the workflow ingest fetcher before a @@ -14,9 +18,11 @@ var errNoForgeClient = errors.New("no GitHub client") // workflowRepoFiles is the dashboard's WorkflowRepoFiles dependency // (POST /api/workflows/ingest, RFC #17620 P3): it downloads the repo's -// default branch read-only, keeps only the CI definition files and -// removes the checkout. client is read per call so a credential rebuild is -// followed. +// default branch read-only, keeps only the CI definition and process +// convention files and removes the checkout. The default branch's rules +// (protection or rulesets, P5) ride along at ingest.BranchRulesPath when +// the forge reports any; a failure to read them is logged, not fatal. +// client is read per call so a credential rebuild is followed. func workflowRepoFiles(client func() *github.Client) func(ctx context.Context, repo string) (map[string][]byte, string, error) { return func(ctx context.Context, repo string) (map[string][]byte, string, error) { c := client() @@ -32,6 +38,41 @@ func workflowRepoFiles(client func() *github.Client) func(ctx context.Context, r if err != nil { return nil, "", err } + switch rules, err := c.FetchBranchRules(ctx, repo, co.Ref); { + case err != nil: + log.Printf("workflow ingest: %s branch rules: %v", repo, err) + case rules != nil: + if b, err := json.Marshal(rules); err == nil { + files[ingest.BranchRulesPath] = b + } + } return files, co.Ref, nil } } + +// workflowRepoFile is the dashboard's WorkflowRepoFile dependency +// (POST /api/workflows/ingest/spec {repo, path}, RFC #17620 phase 6): one +// default-branch file through the forge API. +func workflowRepoFile(client func() *github.Client) func(ctx context.Context, repo, path string) (string, error) { + return func(ctx context.Context, repo, path string) (string, error) { + c := client() + if c == nil { + return "", errNoForgeClient + } + owner, name, ok := strings.Cut(repo, "/") + if !ok || owner == "" || name == "" { + return "", errors.New("repo must be owner/name") + } + return c.GetFileContent(ctx, owner, name, strings.TrimPrefix(path, "/")) + } +} + +// workflowClusterRunner is the dashboard's WorkflowClusterRunner dependency: +// kubectl on PATH with a kubeconfig or in-cluster token, else nil so the +// cluster ingest route answers 422 instead of failing per call. +func workflowClusterRunner() cluster.Runner { + if ok, _ := cluster.Available(); !ok { + return nil + } + return cluster.Kubectl{} +} diff --git a/src/cmd/workflow-survey/main.go b/src/cmd/workflow-survey/main.go index 31ad3e2752..692c3f2901 100644 --- a/src/cmd/workflow-survey/main.go +++ b/src/cmd/workflow-survey/main.go @@ -19,6 +19,7 @@ import ( "github.com/hivecommons/hive/pkg/workflowmodel" "github.com/hivecommons/hive/pkg/workflowmodel/ingest" + corpuslist "github.com/hivecommons/hive/pkg/workflowmodel/ingest/corpus" "github.com/hivecommons/hive/pkg/workflowmodel/render" ) @@ -74,6 +75,21 @@ type repoResult struct { CompileError string `json:"compileError,omitempty"` CompileCodes map[string]int `json:"compileCodes,omitempty"` LifecycleCols int `json:"lifecycleColumns"` + // Intake is what the P5 intake half proposed from the repository's + // process conventions (issue/PR templates, OWNERS, branch rules). + Intake intakeStats `json:"intake"` +} + +// intakeStats counts the intake-half proposals in one lifecycle model. +type intakeStats struct { + IntakeStates int `json:"intakeStates"` + Triage bool `json:"triage"` + MemberRoles int `json:"memberRoles"` + InferredGates int `json:"inferredGates"` + ManualChecks int `json:"manualChecks"` + CIChecks int `json:"ciChecks"` + BranchRules bool `json:"branchRules"` + IntakeFindings int `json:"intakeFindings"` } type summary struct { @@ -92,10 +108,22 @@ type summary struct { Severities map[workflowmodel.Severity]int `json:"severities"` ColumnSets map[string]int `json:"columnSets"` ReposChangedCols int `json:"reposWithChangedColumns"` - ByProject map[string]*projectStats `json:"byProject"` - Worst []repoResult `json:"worst"` - Top []repoResult `json:"top"` - Repos_ []repoResult `json:"results"` + // Intake counts repos where the intake half proposed each element. + Intake intakeSummary `json:"intake"` + ByProject map[string]*projectStats `json:"byProject"` + Worst []repoResult `json:"worst"` + Top []repoResult `json:"top"` + Repos_ []repoResult `json:"results"` +} + +// intakeSummary counts repositories, not elements. +type intakeSummary struct { + ReposWithIntakeStates int `json:"reposWithIntakeStates"` + ReposWithTriage int `json:"reposWithTriage"` + ReposWithRoles int `json:"reposWithRoles"` + ReposWithGates int `json:"reposWithGates"` + ReposWithChecks int `json:"reposWithChecks"` + ReposWithBranchRules int `json:"reposWithBranchRules"` } type projectStats struct { @@ -141,11 +169,13 @@ func realMain(args []string, stdout, stderr io.Writer) int { // loadProjects maps owner/repo to project maturity from the collector's // projects.json, when present. +// loadProjects reads /projects.json; when the corpus directory +// carries none it falls back to the embedded CNCF list (#17620 phase 7). func loadProjects(corpus string) map[string]string { out := map[string]string{} b, err := os.ReadFile(filepath.Join(corpus, "projects.json")) if err != nil { - return out + return corpuslist.Maturity() } var projects []struct { Project string `json:"project"` @@ -272,6 +302,7 @@ func surveyRepo(dir, repo, outDir string) (repoResult, error) { } else { res.Compiled = true } + res.Intake = countIntake(lm) if err := os.MkdirAll(outDir, 0o755); err != nil { return res, err @@ -296,6 +327,50 @@ func surveyRepo(dir, repo, outDir string) (repoResult, error) { return res, writeJSON(filepath.Join(outDir, "result.json"), res) } +// countIntake tallies the intake-half proposals in a lifecycle model. +func countIntake(m *workflowmodel.Model) intakeStats { + var st intakeStats + for _, s := range m.Spec.States { + switch { + case s.ID == ingest.StateTriage: + st.Triage = true + case strings.HasPrefix(s.ID, ingest.StateRequestedPrefix): + st.IntakeStates++ + } + } + for _, r := range m.Spec.Roles { + if r.Kind == workflowmodel.RoleMembers { + st.MemberRoles++ + } + } + for _, g := range m.Spec.Gates { + if g.Confidence == workflowmodel.ConfidenceInferred { + st.InferredGates++ + } + } + for _, c := range m.Spec.Checks { + switch c.Kind { + case workflowmodel.CheckManual: + st.ManualChecks++ + case workflowmodel.CheckCI: + st.CIChecks++ + st.BranchRules = true + } + } + for _, g := range m.Spec.Gates { + if g.ID == ingest.GateCodeownerReview || (g.ID == ingest.GateReview && g.MinApprovals > 0) { + st.BranchRules = true + } + } + st.BranchRules = st.BranchRules || len(m.Spec.Bookkeeping) > 0 + for _, f := range m.Spec.Coverage.Findings { + if strings.HasPrefix(f.Code, "issue-template-") { + st.IntakeFindings++ + } + } + return st +} + func countConstructs(c map[string]int, p workflowmodel.Pipeline) { c["pipelines"]++ if p.Concurrency != nil { @@ -357,6 +432,24 @@ func accumulate(s *summary, r repoResult) { if len(r.ColumnDiff) > 0 { s.ReposChangedCols++ } + if r.Intake.IntakeStates > 0 { + s.Intake.ReposWithIntakeStates++ + } + if r.Intake.Triage { + s.Intake.ReposWithTriage++ + } + if r.Intake.MemberRoles > 0 { + s.Intake.ReposWithRoles++ + } + if r.Intake.InferredGates > 0 { + s.Intake.ReposWithGates++ + } + if r.Intake.ManualChecks+r.Intake.CIChecks > 0 { + s.Intake.ReposWithChecks++ + } + if r.Intake.BranchRules { + s.Intake.ReposWithBranchRules++ + } for k, v := range r.Roles { s.Roles[k] += v } @@ -411,6 +504,8 @@ func markdown(s *summary) string { fmt.Fprintf(&b, "| %s | %d |\n", k, s.CompileCodes[k]) } b.WriteString("\n") + fmt.Fprintf(&b, "## Intake proposals (RFC phase 5)\n\n| Proposed | Repos |\n|---|---|\n| Intake states (issue templates) | %d |\n| Triage state | %d |\n| Member roles (CODEOWNERS/OWNERS) | %d |\n| Inferred gates | %d |\n| Checks (PR template / required status) | %d |\n| Branch rules | %d |\n\n", + s.Intake.ReposWithIntakeStates, s.Intake.ReposWithTriage, s.Intake.ReposWithRoles, s.Intake.ReposWithGates, s.Intake.ReposWithChecks, s.Intake.ReposWithBranchRules) b.WriteString("## Pipeline roles\n\n| Role | Pipelines |\n|---|---|\n") for _, k := range sortedKeys(s.Roles) { fmt.Fprintf(&b, "| %s | %d |\n", k, s.Roles[workflowmodel.Role(k)]) diff --git a/src/cmd/workflow-survey/main_test.go b/src/cmd/workflow-survey/main_test.go index 59d40c9f32..6a939a1a45 100644 --- a/src/cmd/workflow-survey/main_test.go +++ b/src/cmd/workflow-survey/main_test.go @@ -44,6 +44,15 @@ func buildCorpus(t *testing.T) string { if n == 0 { t.Fatalf("no fixture repositories under %s", fixtureCorpus) } + // One repo with process conventions but no pipelines exercises the + // intake counts (RFC phase 5). + intake, err := filepath.Abs(filepath.Join("..", "..", "pkg", "workflowmodel", "ingest", "testdata", "k8s-owners")) + if err != nil { + t.Fatal(err) + } + if err := os.CopyFS(filepath.Join(repos, "example__k8s-owners"), os.DirFS(intake)); err != nil { + t.Fatal(err) + } // A stray file in repos/ must be ignored, as must a repo not in projects.json. if err := os.WriteFile(filepath.Join(repos, "README.md"), []byte("x"), 0o644); err != nil { t.Fatal(err) @@ -82,8 +91,19 @@ func TestRealMainSurveysCorpus(t *testing.T) { if err := json.Unmarshal(raw, &sum); err != nil { t.Fatal(err) } - if sum.Repos != 6 || sum.Valid != sum.Repos { - t.Errorf("repos=%d valid=%d, want 6 valid", sum.Repos, sum.Valid) + if sum.Repos != 7 || sum.Valid != sum.Repos { + t.Errorf("repos=%d valid=%d, want 7 valid", sum.Repos, sum.Valid) + } + if sum.Compiled != sum.Repos { + t.Errorf("compiled=%d of %d", sum.Compiled, sum.Repos) + } + if in := sum.Intake; in.ReposWithIntakeStates != 1 || in.ReposWithTriage != 1 || in.ReposWithRoles != 1 || in.ReposWithGates != 1 || in.ReposWithChecks != 1 || in.ReposWithBranchRules != 1 { + t.Errorf("intake summary %+v, want exactly the k8s-owners repo", in) + } + for _, r := range sum.Repos_ { + if r.Repo == "example/k8s-owners" && (r.Intake.IntakeStates != 3 || !r.Intake.Triage || r.Intake.MemberRoles != 2 || r.Intake.ManualChecks != 4 || r.Intake.CIChecks != 3 || r.Intake.IntakeFindings != 1) { + t.Errorf("k8s-owners intake %+v", r.Intake) + } } if sum.ReposWithGHA == 0 || sum.ReposWithTekton == 0 { t.Errorf("expected both systems in the corpus: gha=%d tekton=%d", sum.ReposWithGHA, sum.ReposWithTekton) @@ -94,7 +114,7 @@ func TestRealMainSurveysCorpus(t *testing.T) { if got := sum.ByProject["graduated"]; got == nil || got.Repos != 1 { t.Errorf("projects.json maturity join (case-insensitive) not applied: %+v", sum.ByProject) } - if got := sum.ByProject["unknown"]; got == nil || got.Repos != 4 { + if got := sum.ByProject["unknown"]; got == nil || got.Repos != 5 { t.Errorf("repos without a maturity should land in unknown: %+v", sum.ByProject) } if len(sum.Worst) != sum.Repos || len(sum.Top) != sum.Repos { @@ -110,7 +130,7 @@ func TestRealMainSurveysCorpus(t *testing.T) { if err != nil { t.Fatal(err) } - for _, want := range []string{"# Workflow model survey", "## Pipeline roles", "## Column sets (Activity Monitor)", "| graduated | 1 |", "trigger:pull_request"} { + for _, want := range []string{"# Workflow model survey", "## Pipeline roles", "## Column sets (Activity Monitor)", "## Intake proposals (RFC phase 5)", "| Triage state | 1 |", "| graduated | 1 |", "trigger:pull_request"} { if !strings.Contains(string(md), want) { t.Errorf("summary.md missing %q", want) } @@ -233,8 +253,8 @@ func TestHelpers(t *testing.T) { if got := sortedKeys(map[string]int{"b": 1, "a": 2}); strings.Join(got, ",") != "a,b" { t.Errorf("sortedKeys = %v", got) } - if loadProjects(t.TempDir()) == nil { - t.Error("loadProjects without projects.json should return an empty map") + if m := loadProjects(t.TempDir()); m["project-akri/akri"] != "sandbox" { + t.Errorf("loadProjects without projects.json should fall back to the embedded corpus, got %d entries", len(m)) } dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, "projects.json"), []byte("{not json"), 0o644); err != nil { diff --git a/src/crux/agent/backend_auth_test.go b/src/crux/agent/backend_auth_test.go index 5124a4ce2d..38dbd555d8 100644 --- a/src/crux/agent/backend_auth_test.go +++ b/src/crux/agent/backend_auth_test.go @@ -1,6 +1,7 @@ package agent import ( + "strings" "testing" "time" @@ -134,6 +135,50 @@ func TestMarkProviderErrorLockedBare403IsUndetermined(t *testing.T) { } } +// TestMarkProviderErrorLockedRedactsCredentials drives a classified pane line +// that carries a GitHub token and an sk- API key through the real +// classifyProviderError → markProviderErrorLocked path and asserts no copy +// of the line the spoke publishes (LastError, ProviderErrorLine, +// BackendAuth.LastError) still contains either credential. These fields +// reach the /v1 status, the dashboard and the hub heartbeat verbatim, so the +// redaction has to happen at the source. +func TestMarkProviderErrorLockedRedactsCredentials(t *testing.T) { + t.Setenv(ProviderErrorBackoffBaseEnv, "10s") + t.Setenv(ProviderErrorBackoffMaxEnv, "40s") + m := NewManager(map[string]config.AgentConfig{ + "cop": {Backend: "copilot"}, + }, discardLogger(), ProjectContext{}) + m.mu.RLock() + agent := m.agents["cop"] + m.mu.RUnlock() + + const ghToken = "ghp_abcdefghijklmnopqrstuvwxyz0123456789" + const apiKey = "sk-abcdefghijklmnop" + pane := "API Error: 401 Unauthorized for Authorization: Bearer " + ghToken + " (fallback key " + apiKey + ")\n" + + match, ok := classifyProviderError(pane) + if !ok || match.Class != "auth" { + t.Fatalf("classifyProviderError = %+v, ok=%v; want auth class", match, ok) + } + m.markProviderErrorLocked(agent, match, time.Now()) + + if agent.BackendAuth.Status != BackendAuthTokenExpired { + t.Fatalf("BackendAuth.Status = %q, want %q (redaction must not change the verdict)", agent.BackendAuth.Status, BackendAuthTokenExpired) + } + for name, got := range map[string]string{ + "LastError": agent.LastError, + "ProviderErrorLine": agent.ProviderErrorLine, + "BackendAuth.LastError": agent.BackendAuth.LastError, + } { + if got == "" { + t.Errorf("%s is empty; want the redacted line kept for the operator", name) + } + if strings.Contains(got, ghToken) || strings.Contains(got, apiKey) { + t.Errorf("%s = %q still carries a credential", name, got) + } + } +} + func TestMarkBackendAuthLocked_FirstObservationSetsSince(t *testing.T) { agent := newBackendAuthTestAgent(t) now := time.Now() diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index e386f0319d..ded2ce4658 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -799,7 +799,10 @@ type Manager struct { // agent on sandbox.runtime: job (#6311). Nil means the real in-cluster // launcher; tests inject a fake through setSandboxJobLauncherFactoryForTest. sandboxJobLauncherFactory func(config.SandboxJobConfig) sandbox.Launcher - sandboxRunner sandboxCommandRunner + // sandboxOpenShellLauncherFactory is the same seam for + // sandbox.runtime: openshell (ADR-0026). + sandboxOpenShellLauncherFactory func(config.SandboxOpenShellConfig) sandbox.Launcher + sandboxRunner sandboxCommandRunner // sandboxRunPersister overrides persistSandboxRun in tests. sandboxRunPersister func(workDir string, entry SandboxRunHistoryEntry) error sandboxPushMinter pushbroker.TokenMinter diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index 98cd183475..de2636493e 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os" "strings" "time" @@ -15,6 +16,27 @@ import ( "github.com/hivecommons/hive/crux/sandbox" ) +func init() { + config.SetOpenShellPreflight(openShellPreflightWarnings) +} + +// openShellPreflightWarnings checks each distinct gateway the openshell +// agents resolve to, once. +func openShellPreflightWarnings(blocks []config.SandboxOpenShellConfig) []string { + var out []string + seen := map[string]bool{} + for _, b := range blocks { + opts := OpenShellOptionsFromConfig(b) + key := opts.GatewayURL + "\x00" + opts.Token + if seen[key] { + continue + } + seen[key] = true + out = append(out, sandbox.DefaultOpenShellPreflighter.Check(context.Background(), opts)...) + } + return out +} + // SetSandboxConfig wires the disabled-by-default sandbox kick executor gate. func (m *Manager) SetSandboxConfig(cfg config.AgentSandboxConfig) { m.mu.Lock() @@ -90,13 +112,23 @@ func (m *Manager) validateSandboxKickLocked(agent *AgentProcess) (string, error) // does, instead of failing it after the state machine has moved. runtime := agent.Config.SandboxRuntime(m.sandboxConfig) if !config.ValidSandboxRuntimes[runtime] { - return "", fmt.Errorf("agent %s sandbox runtime %q is not one hive has (%s or %s)", agent.Name, runtime, config.SandboxRuntimePodman, config.SandboxRuntimeJob) + return "", fmt.Errorf("agent %s sandbox runtime %q is not one hive has (%s, %s or %s)", agent.Name, runtime, config.SandboxRuntimePodman, config.SandboxRuntimeJob, config.SandboxRuntimeOpenShell) } - if runtime == config.SandboxRuntimeJob { + switch runtime { + case config.SandboxRuntimeJob: job := agent.Config.SandboxJob(m.sandboxConfig) if strings.TrimSpace(job.WorkspaceClaim) == "" { return "", fmt.Errorf("agent %s sandbox job runtime requires job.workspace_claim — the PVC hive's sandbox workspace root lives on, mounted RWX so the Job can see it from another node", agent.Name) } + case config.SandboxRuntimeOpenShell: + // ADR-0026: the OpenShell runtime is inert unless selected, and a + // selected one refuses the kick before the state machine moves. + if err := OpenShellOptionsFromConfig(agent.Config.SandboxOpenShell(m.sandboxConfig)).Validate(); err != nil { + return "", fmt.Errorf("agent %s sandbox openshell runtime: %w", agent.Name, err) + } + if effectiveBackend(agent) == "cline" { + return "", fmt.Errorf("agent %s: the Cline backend is not supported on the openshell sandbox runtime (no credential path into the sandbox); use podman or job", agent.Name) + } } if effectiveBackend(agent) == "cline" { if err := validateClineSandbox(agent, runtime, agent.Config.SandboxJob(m.sandboxConfig), m.project.TaskMCPURL); err != nil { @@ -113,9 +145,11 @@ func (m *Manager) startSandboxKickLocked(agent *AgentProcess, message, source st } repo := m.project.PrimaryRepo() launcher := m.sandboxLauncher - if runtime == config.SandboxRuntimeJob { - job := agent.Config.SandboxJob(m.sandboxConfig) - launcher = m.jobLauncherLocked(job) + switch runtime { + case config.SandboxRuntimeJob: + launcher = m.jobLauncherLocked(agent.Config.SandboxJob(m.sandboxConfig)) + case config.SandboxRuntimeOpenShell: + launcher = m.openshellLauncherLocked(agent.Config.SandboxOpenShell(m.sandboxConfig)) } now := time.Now() agent.State = StateRunning @@ -354,6 +388,48 @@ func (m *Manager) jobLauncherLocked(job config.SandboxJobConfig) sandbox.Launche return &kubejob.Launcher{Options: JobOptionsFromConfig(job)} } +func (m *Manager) setSandboxOpenShellLauncherFactoryForTest(f func(config.SandboxOpenShellConfig) sandbox.Launcher) { + m.mu.Lock() + defer m.mu.Unlock() + m.sandboxOpenShellLauncherFactory = f +} + +// openshellLauncherLocked returns the launcher for a sandbox.runtime: +// openshell kick: the injected factory's, or a real gateway-backed +// sandbox.OpenShellLauncher (ADR-0026). +func (m *Manager) openshellLauncherLocked(cfg config.SandboxOpenShellConfig) sandbox.Launcher { + if m.sandboxOpenShellLauncherFactory != nil { + return m.sandboxOpenShellLauncherFactory(cfg) + } + return &sandbox.OpenShellLauncher{Options: OpenShellOptionsFromConfig(cfg)} +} + +// OpenShellOptionsFromConfig converts the operator's openshell block into +// the launcher's options. The gateway token is read from the environment +// variable the block names, so the token itself never sits in hive.yaml. +func OpenShellOptionsFromConfig(cfg config.SandboxOpenShellConfig) sandbox.OpenShellOptions { + opts := sandbox.OpenShellOptions{ + GatewayURL: cfg.GatewayURL, + Workspace: cfg.Namespace, + PolicyProfile: cfg.PolicyProfile, + InsecureTLS: cfg.InsecureTLS, + Timeout: time.Duration(cfg.TimeoutS) * time.Second, + WorkspaceClaim: cfg.WorkspaceClaim, + WorkspaceClaimMount: cfg.WorkspaceClaimMount, + AllowHosts: append([]string(nil), cfg.AllowHosts...), + Binaries: append([]string(nil), cfg.Binaries...), + RuntimeClassName: cfg.RuntimeClassName, + NodeSelector: cfg.NodeSelector, + } + if env := strings.TrimSpace(cfg.TokenEnv); env != "" { + opts.Token = os.Getenv(env) + } + for _, t := range cfg.Tolerations { + opts.Tolerations = append(opts.Tolerations, sandbox.OpenShellToleration{Key: t.Key, Operator: t.Operator, Value: t.Value, Effect: t.Effect}) + } + return opts +} + // JobOptionsFromConfig converts the operator's job block into the launcher's // options. The config package deliberately owns its own types so it does not // import the launcher; this is the one place the two shapes meet. diff --git a/src/crux/agent/manager_status.go b/src/crux/agent/manager_status.go index d83e8708b4..295146ff98 100644 --- a/src/crux/agent/manager_status.go +++ b/src/crux/agent/manager_status.go @@ -6,9 +6,15 @@ import ( "time" "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/logscrub" ) func (m *Manager) markProviderErrorLocked(agent *AgentProcess, match providerErrorMatch, now time.Time) time.Duration { + // The matched line is pane text and leaves the spoke verbatim: LastError + // and BackendAuth.LastError go to the /v1 status, the dashboard and the + // hub heartbeat. Redact credentials once here, like compactPaneTail does + // for the stuck-login diagnostic, so no downstream copy can carry a token. + match.Line = strings.TrimSpace(logscrub.RedactPaneTokens(logscrub.ScrubString(match.Line))) if status, ok := classifyBackendAuthStatus(match.Class, match.Line); ok { agent.markBackendAuthLocked(status, match.Line, now) // #6767: an unlicensed verdict against a copilot agent is direct diff --git a/src/crux/agent/manager_tmux.go b/src/crux/agent/manager_tmux.go index cfc63d1124..b3ccfe0f2c 100644 --- a/src/crux/agent/manager_tmux.go +++ b/src/crux/agent/manager_tmux.go @@ -13,6 +13,7 @@ import ( "github.com/hivecommons/hive/crux/agent/tmux" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/internal/textutil" + "github.com/hivecommons/hive/crux/upstreamproxy" ) // paneCaptureSleep is a pacing var, not a const, for the same reason as the @@ -56,6 +57,34 @@ var inheritedCredentialEnvVars = []string{ "LINEAR_CLIENT_ID", "LINEAR_CLIENT_SECRET", "LINEAR_WEBHOOK_SECRET", + // The dashboard's shared token: X-Hive-Internal / bearer with this value + // is a server-verified OWNER on the Go API, including GET + // /api/config/download (the raw hive.yaml). Hosted hives receive it as + // DASHBOARD_AUTH_TOKEN from the provisioning template; Compose and Podman + // stacks as HIVE_DASHBOARD_TOKEN. + "DASHBOARD_AUTH_TOKEN", + "HIVE_DASHBOARD_TOKEN", + // Per-hive signing keys the hub injects into the hive container, and the + // legacy master they were derived from. The terminal key alone forges a + // /terminal grant to a ttyd shell as the hive user. + "HIVE_TERMINAL_KEY", + "HIVE_INVITE_KEY", + "HIVE_HEARTBEAT_KEY", + "HIVE_HUB_SECRET", + "HIVE_TTYD_CREDENTIAL", + // The corporate forward-proxy rules. The URL's userinfo is the proxy's + // Basic credential; agents reach the upstream only through the MITM + // proxy and are never handed the corporate proxy URL. The list vars + // carry no secret but are hive-process settings, not agent ones — agents + // get their own NO_PROXY from agentEnvPairs. The names come from + // crux/upstreamproxy so the strip list cannot drift from what the + // loader reads; HIVE_NO_PROXY is the name the hub's provisioning + // template injects (saas_provision.go), stripped alongside until the + // hub and loader agree on one. + upstreamproxy.URLEnv, + upstreamproxy.HostsEnv, + upstreamproxy.NoProxyEnv, + "HIVE_NO_PROXY", } // githubTokenEnvVars are the environment variables gh, git tooling and the diff --git a/src/crux/agent/sandbox_openshell_runtime_test.go b/src/crux/agent/sandbox_openshell_runtime_test.go new file mode 100644 index 0000000000..abf69fd6f7 --- /dev/null +++ b/src/crux/agent/sandbox_openshell_runtime_test.go @@ -0,0 +1,107 @@ +package agent + +import ( + "context" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +// The OpenShell runtime (ADR-0026) routes a sandboxed kick through the +// launcher built from the agent's openshell block, refuses a kick whose +// gateway is unconfigured before the state machine moves, and stays inert +// for every agent that did not select it. + +func openshellRuntimeManager(t *testing.T, override *config.AgentSandboxOverride, global config.AgentSandboxConfig) (*Manager, *int32, *int32) { + t.Helper() + cfg := map[string]config.AgentConfig{"deps": {Backend: "claude", Sandbox: override}} + m := NewManager(cfg, quietTestLogger(), ProjectContext{Org: "torch-spyre", Repos: []string{"spyre-inference"}}) + if global.WorkspaceDir == "" { + global.WorkspaceDir = t.TempDir() + } + m.SetSandboxConfig(global) + var podmanRuns, openshellRuns int32 + m.SetSandboxLauncher(countingLauncher{runs: &podmanRuns}) + m.setSandboxOpenShellLauncherFactoryForTest(func(os config.SandboxOpenShellConfig) sandbox.Launcher { + if os.GatewayURL != "gw.openshell.svc:8080" { + t.Errorf("factory got gateway_url=%q", os.GatewayURL) + } + return countingLauncher{runs: &openshellRuns} + }) + m.setSandboxRunnerForTest(&sandboxFakeRunner{}) + if err := m.Start(context.Background(), "deps"); err != nil { + t.Fatalf("Start: %v", err) + } + return m, &podmanRuns, &openshellRuns +} + +func TestOpenShellRuntimeKickUsesTheOpenShellLauncher(t *testing.T) { + yes := true + m, podman, osRuns := openshellRuntimeManager(t, + &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeOpenShell}, + config.AgentSandboxConfig{Enabled: true, Image: "dev-image", + OpenShell: &config.SandboxOpenShellConfig{GatewayURL: "gw.openshell.svc:8080", WorkspaceClaim: "hive-data"}}) + if err := m.SendKick("deps", "bump the pinned dependency"); err != nil { + t.Fatalf("SendKick: %v", err) + } + waitIdle(t, m, "deps") + if got := atomic.LoadInt32(osRuns); got != 1 { + t.Errorf("openshell launcher runs = %d, want 1", got) + } + if got := atomic.LoadInt32(podman); got != 0 { + t.Errorf("podman launcher ran %d time(s) for an openshell-runtime agent", got) + } +} + +func TestOpenShellRuntimeRefusesKickWithoutGateway(t *testing.T) { + yes := true + m, _, osRuns := openshellRuntimeManager(t, + &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeOpenShell}, + config.AgentSandboxConfig{Enabled: true, Image: "dev-image", + OpenShell: &config.SandboxOpenShellConfig{WorkspaceClaim: "hive-data"}}) + err := m.SendKick("deps", "fix") + if err == nil || !strings.Contains(err.Error(), "gateway_url") { + t.Fatalf("SendKick error = %v, want a refusal naming gateway_url", err) + } + if got := atomic.LoadInt32(osRuns); got != 0 { + t.Errorf("launcher ran %d time(s) despite the refusal", got) + } + if st := m.AllStatuses()["deps"].State; st == StateRunning { + t.Errorf("agent marked running after a refused kick") + } +} + +func TestOpenShellOptionsFromConfig(t *testing.T) { + t.Setenv("OPENSHELL_TEST_TOKEN", "sekrit") + opts := OpenShellOptionsFromConfig(config.SandboxOpenShellConfig{ + GatewayURL: "https://gw", + Namespace: "hive", + PolicyProfile: "strict", + TimeoutS: 90, + TokenEnv: "OPENSHELL_TEST_TOKEN", + WorkspaceClaim: "hive-data", + AllowHosts: []string{"litellm.example.com"}, + Tolerations: []config.SandboxJobToleration{{Key: "k", Operator: "Exists"}}, + }) + if opts.Token != "sekrit" || opts.Workspace != "hive" || opts.PolicyProfile != "strict" || opts.Timeout != 90*time.Second { + t.Errorf("opts = %+v", opts) + } + if len(opts.Tolerations) != 1 || opts.Tolerations[0].Key != "k" { + t.Errorf("tolerations = %+v", opts.Tolerations) + } + if err := opts.Validate(); err != nil { + t.Errorf("Validate: %v", err) + } +} + +func TestOpenShellLauncherDefaultsToSDK(t *testing.T) { + m := NewManager(map[string]config.AgentConfig{}, quietTestLogger(), ProjectContext{}) + l := m.openshellLauncherLocked(config.SandboxOpenShellConfig{GatewayURL: "gw:1"}) + if _, ok := l.(*sandbox.OpenShellLauncher); !ok { + t.Fatalf("launcher = %T, want *sandbox.OpenShellLauncher", l) + } +} diff --git a/src/crux/agent/tmux_inherited_env_test.go b/src/crux/agent/tmux_inherited_env_test.go index 74a728cb7b..2b8a40b799 100644 --- a/src/crux/agent/tmux_inherited_env_test.go +++ b/src/crux/agent/tmux_inherited_env_test.go @@ -47,6 +47,21 @@ func TestApplySessionEnv_RemovesInheritedCredentialsFromPanes(t *testing.T) { "LINEAR_WEBHOOK_SECRET=webhook_secret_leaked", "HIVE_GITHUB_TOKEN=ghs_full_token_leaked", "GITHUB_TOKEN=ghs_inherited_leaked", + // Hub-provisioned and Compose/Podman per-hive secrets: the dashboard + // shared token (owner-equivalent on the Go API), the terminal, invite + // and heartbeat signing keys, the legacy master, the ttyd credential + // and the corporate proxy URL whose userinfo is a Basic credential. + "DASHBOARD_AUTH_TOKEN=dash_auth_leaked", + "HIVE_DASHBOARD_TOKEN=hive_dash_leaked", + "HIVE_TERMINAL_KEY=terminal_key_leaked", + "HIVE_INVITE_KEY=invite_key_leaked", + "HIVE_HEARTBEAT_KEY=heartbeat_key_leaked", + "HIVE_HUB_SECRET=hub_secret_leaked", + "HIVE_TTYD_CREDENTIAL=hive:ttyd_leaked", + "HIVE_UPSTREAM_PROXY_URL=http://proxyuser:proxypass@proxy.corp.example:3128", + "HIVE_UPSTREAM_PROXY_HOSTS=ghe.corp.example", + "HIVE_UPSTREAM_PROXY_NO_PROXY=10.0.0.0/8", + "HIVE_NO_PROXY=10.0.0.0/8", ) if out, err := start.CombinedOutput(); err != nil { t.Fatalf("starting tmux server: %v: %s", err, out) @@ -86,6 +101,9 @@ func TestApplySessionEnv_RemovesInheritedCredentialsFromPanes(t *testing.T) { if before["LINEAR_API_KEY"] != "lin_api_leaked" || before["LINEAR_CLIENT_SECRET"] != "client_secret_leaked" { t.Fatalf("control: expected the planted credentials in a fresh pane, got LINEAR_API_KEY=%q LINEAR_CLIENT_SECRET=%q", before["LINEAR_API_KEY"], before["LINEAR_CLIENT_SECRET"]) } + if before["DASHBOARD_AUTH_TOKEN"] != "dash_auth_leaked" || before["HIVE_TERMINAL_KEY"] != "terminal_key_leaked" { + t.Fatalf("control: expected the planted hub secrets in a fresh pane, got DASHBOARD_AUTH_TOKEN=%q HIVE_TERMINAL_KEY=%q", before["DASHBOARD_AUTH_TOKEN"], before["HIVE_TERMINAL_KEY"]) + } // An ISSUES_ONLY agent on this socket whose sanctioned Linear credential // is the OAuth token. It may hold LINEAR_ACCESS_TOKEN and nothing else @@ -104,7 +122,12 @@ func TestApplySessionEnv_RemovesInheritedCredentialsFromPanes(t *testing.T) { m.applySessionEnv(agent) after := paneEnv("after") - for _, k := range []string{"LINEAR_API_KEY", "LINEAR_CLIENT_SECRET", "LINEAR_WEBHOOK_SECRET", "HIVE_GITHUB_TOKEN", "GITHUB_TOKEN"} { + for _, k := range []string{ + "LINEAR_API_KEY", "LINEAR_CLIENT_SECRET", "LINEAR_WEBHOOK_SECRET", "HIVE_GITHUB_TOKEN", "GITHUB_TOKEN", + "DASHBOARD_AUTH_TOKEN", "HIVE_DASHBOARD_TOKEN", "HIVE_TERMINAL_KEY", "HIVE_INVITE_KEY", "HIVE_HEARTBEAT_KEY", + "HIVE_HUB_SECRET", "HIVE_TTYD_CREDENTIAL", "HIVE_UPSTREAM_PROXY_URL", "HIVE_UPSTREAM_PROXY_HOSTS", + "HIVE_UPSTREAM_PROXY_NO_PROXY", "HIVE_NO_PROXY", + } { if v, ok := after[k]; ok { t.Errorf("%s=%q still reaches a pane after applySessionEnv — the inherited credential was not removed", k, v) } diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index 21520d78a8..2033319af8 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -1,114 +1,6 @@ package config -import ( - "strings" - "time" -) - -// Sandbox runtimes. SandboxRuntimePodman is the default when nothing is set. -const ( - SandboxRuntimePodman = "podman" - SandboxRuntimeJob = "job" -) - -// ValidSandboxRuntimes is the closed set a config may name. -var ValidSandboxRuntimes = map[string]bool{SandboxRuntimePodman: true, SandboxRuntimeJob: true} - -// SandboxJobConfig is everything the Job runtime needs beyond the image -// (#6311). The pod-template fields pass through to Kubernetes verbatim. -// -// The workspace is shared between hive and the Job by mounting the SAME -// PersistentVolumeClaim in both: WorkspaceClaim names it, and -// WorkspaceClaimMount is where hive sees it (default /data, where the -// standalone and hub-provisioned deployments mount hive-data). The sandbox -// workspace root must live under that mount. When the Job lands on a -// different node than hive, the claim has to be ReadWriteMany. -// -// EnvFromSecrets is the ONLY credential path into the Job. Hive's brokered -// GitHub token never enters it; hive pushes and opens the PR from the -// workspace after the Job exits, exactly as it does for the Podman runtime. -type SandboxJobConfig struct { - WorkspaceClaim string `yaml:"workspace_claim,omitempty" json:"workspace_claim,omitempty"` - WorkspaceClaimMount string `yaml:"workspace_claim_mount,omitempty" json:"workspace_claim_mount,omitempty"` - NodeSelector map[string]string `yaml:"node_selector,omitempty" json:"node_selector,omitempty"` - Tolerations []SandboxJobToleration `yaml:"tolerations,omitempty" json:"tolerations,omitempty"` - Resources SandboxJobResources `yaml:"resources,omitempty" json:"resources,omitempty"` - ServiceAccount string `yaml:"service_account,omitempty" json:"service_account,omitempty"` - // SchedulerName sets the pod's schedulerName. Accelerator clusters often - // gate device-requesting pods behind a custom scheduler via an admission - // webhook (e.g. `A pod must use "schedulerName: spyre-scheduler"`); - // without this the Job create is denied before any pod exists. - SchedulerName string `yaml:"scheduler_name,omitempty" json:"scheduler_name,omitempty"` - EnvFromSecrets []string `yaml:"env_from_secrets,omitempty" json:"env_from_secrets,omitempty"` - Volumes []SandboxJobVolume `yaml:"volumes,omitempty" json:"volumes,omitempty"` - TTLSeconds int `yaml:"ttl_seconds,omitempty" json:"ttl_seconds,omitempty"` -} - -// SandboxJobToleration mirrors the subset of a Kubernetes toleration an -// operator writes by hand. -type SandboxJobToleration struct { - Key string `yaml:"key,omitempty" json:"key,omitempty"` - Operator string `yaml:"operator,omitempty" json:"operator,omitempty"` - Value string `yaml:"value,omitempty" json:"value,omitempty"` - Effect string `yaml:"effect,omitempty" json:"effect,omitempty"` -} - -// SandboxJobResources mirrors Kubernetes resource requirements with string -// quantities so a device limit (`.com/: "2"`) passes through. -type SandboxJobResources struct { - Limits map[string]string `yaml:"limits,omitempty" json:"limits,omitempty"` - Requests map[string]string `yaml:"requests,omitempty" json:"requests,omitempty"` -} - -// SandboxJobVolume is an additional PVC mounted into the Job (a compile -// cache, a model store). -type SandboxJobVolume struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - Claim string `yaml:"claim" json:"claim"` - MountPath string `yaml:"mount_path" json:"mount_path"` - ReadOnly bool `yaml:"read_only,omitempty" json:"read_only,omitempty"` -} - -// SandboxRuntime returns the per-agent runtime override, then the global -// runtime, then SandboxRuntimePodman. The value is returned as written; the -// manager rejects one outside ValidSandboxRuntimes at kick time. -func (a *AgentConfig) SandboxRuntime(global AgentSandboxConfig) string { - if a != nil && a.Sandbox != nil && strings.TrimSpace(a.Sandbox.Runtime) != "" { - return strings.TrimSpace(a.Sandbox.Runtime) - } - if strings.TrimSpace(global.Runtime) != "" { - return strings.TrimSpace(global.Runtime) - } - return SandboxRuntimePodman -} - -// SandboxJob returns the effective Job runtime settings. A per-agent job -// block replaces the global one wholesale for the pod-template fields (node -// selector, tolerations, resources, secrets, volumes, service account) — an -// agent that needs an accelerator names its own scheduling. The claim -// fields and the TTL are cluster facts rather than per-agent choices, so -// they fall back to the global block when the per-agent one leaves them -// empty. -func (a *AgentConfig) SandboxJob(global AgentSandboxConfig) SandboxJobConfig { - var g SandboxJobConfig - if global.Job != nil { - g = *global.Job - } - if a == nil || a.Sandbox == nil || a.Sandbox.Job == nil { - return g - } - out := *a.Sandbox.Job - if strings.TrimSpace(out.WorkspaceClaim) == "" { - out.WorkspaceClaim = g.WorkspaceClaim - } - if strings.TrimSpace(out.WorkspaceClaimMount) == "" { - out.WorkspaceClaimMount = g.WorkspaceClaimMount - } - if out.TTLSeconds == 0 { - out.TTLSeconds = g.TTLSeconds - } - return out -} +import "time" // AgentRuntimeConfig is the top-level hive.yaml surface owned by the agent // runtime (Crux, hivecommons/hive#13536, stage 1): the agent definitions, @@ -541,6 +433,8 @@ type AgentSandboxConfig struct { WorkspaceDir string `yaml:"workspace_dir,omitempty" json:"workspace_dir,omitempty"` Runtime string `yaml:"runtime,omitempty" json:"runtime,omitempty"` Job *SandboxJobConfig `yaml:"job,omitempty" json:"job,omitempty"` + // OpenShell holds the settings for sandbox.runtime: openshell (ADR-0026). + OpenShell *SandboxOpenShellConfig `yaml:"openshell,omitempty" json:"openshell,omitempty"` // Publish gates what a sandboxed run may publish; see SandboxPublishConfig. Publish *SandboxPublishConfig `yaml:"publish,omitempty" json:"publish,omitempty"` } @@ -579,6 +473,9 @@ type AgentSandboxOverride struct { TimeoutS int `yaml:"timeout_s,omitempty" json:"timeout_s,omitempty"` Runtime string `yaml:"runtime,omitempty" json:"runtime,omitempty"` Job *SandboxJobConfig `yaml:"job,omitempty" json:"job,omitempty"` + // OpenShell, when set, overrides the global openshell block field by + // field for this agent; see AgentConfig.SandboxOpenShell. + OpenShell *SandboxOpenShellConfig `yaml:"openshell,omitempty" json:"openshell,omitempty"` // Publish, when set, replaces the global publish preflight for this // agent; an empty verify list here turns the preflight off for it. Publish *SandboxPublishConfig `yaml:"publish,omitempty" json:"publish,omitempty"` diff --git a/src/crux/config/agent_config_openshell.go b/src/crux/config/agent_config_openshell.go new file mode 100644 index 0000000000..61c307ddf1 --- /dev/null +++ b/src/crux/config/agent_config_openshell.go @@ -0,0 +1,141 @@ +package config + +import ( + "strings" + "sync" +) + +var ( + openshellPreflightMu sync.RWMutex + openshellPreflight func([]SandboxOpenShellConfig) []string +) + +// SetOpenShellPreflight registers the cluster-prerequisite check +// AgentSandboxGateWarnings runs for openshell-runtime agents. This package +// (and pkg/config above it) cannot import the OpenShell SDK or crux/sandbox, +// so the code that owns the gateway client plugs itself in here. Passing nil +// unregisters it. +func SetOpenShellPreflight(f func([]SandboxOpenShellConfig) []string) { + openshellPreflightMu.Lock() + defer openshellPreflightMu.Unlock() + openshellPreflight = f +} + +// OpenShellPreflightWarnings runs the registered preflight against the given +// openshell blocks; with none registered it reports nothing. +func OpenShellPreflightWarnings(cfgs []SandboxOpenShellConfig) []string { + openshellPreflightMu.RLock() + f := openshellPreflight + openshellPreflightMu.RUnlock() + if f == nil || len(cfgs) == 0 { + return nil + } + return f(cfgs) +} + +// SandboxOpenShellConfig is everything the OpenShell runtime needs beyond the +// image (ADR-0026 phase 1). Hive talks to an OpenShell gateway over its gRPC +// API; the gateway owns the sandbox pod, the Landlock/seccomp confinement and +// the egress proxy, so most of what the Job runtime spells out as a pod +// template is a gateway concern here. +// +// The workspace is shared the same way the Job runtime shares it: the +// sandbox mounts the SAME PersistentVolumeClaim hive's /data lives on, at a +// subPath pointing at the per-kick workspace. The OpenShell SDK's file +// transport is single-file and not available on every gateway, so a copy +// in/out was not an option. This needs the gateway's `allow_driver_config` +// setting on and the claim labelled `openshell.ai/sandbox-attachable`. +// +// AllowHosts is the egress allowlist handed to the gateway as a network +// policy: the hosts hive's own MITM proxy (ADR-0002) would let a Podman +// sandbox reach — github.com, api.github.com — plus whatever the operator +// adds (a model gateway, the hub). Nothing else is reachable from the +// sandbox. Credentials follow the Job rule: none of hive's enter it. +type SandboxOpenShellConfig struct { + // GatewayURL is the OpenShell gateway's gRPC address (host:port, or an + // http(s):// URL whose scheme picks TLS). Required when the runtime is + // selected. + GatewayURL string `yaml:"gateway_url,omitempty" json:"gateway_url,omitempty"` + // Namespace is the OpenShell workspace sandboxes are created in. It maps + // to the OpenShell notion of a workspace, not a Kubernetes namespace. + // Default "default". + Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"` + // PolicyProfile names a gateway-side policy profile to apply alongside + // hive's egress policy. Empty means hive's derived policy only. + PolicyProfile string `yaml:"policy_profile,omitempty" json:"policy_profile,omitempty"` + // TimeoutS bounds one sandbox from create to delete. Non-positive uses + // the sandbox timeout (agent_sandbox.timeout_s) hive already applies to + // the kick. + TimeoutS int `yaml:"timeout_s,omitempty" json:"timeout_s,omitempty"` + // TokenEnv names the environment variable holding the gateway bearer + // token. Empty means the gateway is reached unauthenticated (dev only). + TokenEnv string `yaml:"token_env,omitempty" json:"token_env,omitempty"` + // InsecureTLS skips gateway certificate verification. Dev only. + InsecureTLS bool `yaml:"insecure_tls,omitempty" json:"insecure_tls,omitempty"` + // WorkspaceClaim / WorkspaceClaimMount mirror SandboxJobConfig: the PVC + // hive's sandbox workspace root lives on, and where hive sees it. + WorkspaceClaim string `yaml:"workspace_claim,omitempty" json:"workspace_claim,omitempty"` + WorkspaceClaimMount string `yaml:"workspace_claim_mount,omitempty" json:"workspace_claim_mount,omitempty"` + // AllowHosts are additional egress hosts (host or host:port) the + // sandbox may reach, on top of the GitHub hosts hive always allows. + AllowHosts []string `yaml:"allow_hosts,omitempty" json:"allow_hosts,omitempty"` + // Binaries are the in-image executables the egress policy applies to. + // OpenShell matches network rules per binary; a rule with no binaries + // allows nothing. Empty uses the launcher's default set. + Binaries []string `yaml:"binaries,omitempty" json:"binaries,omitempty"` + // RuntimeClassName selects a sandbox RuntimeClass (gVisor, Kata) on the + // cluster; empty uses the gateway default. + RuntimeClassName string `yaml:"runtime_class_name,omitempty" json:"runtime_class_name,omitempty"` + // NodeSelector and Tolerations pass through to the sandbox pod via the + // gateway's Kubernetes driver config, like the Job runtime's. + NodeSelector map[string]string `yaml:"node_selector,omitempty" json:"node_selector,omitempty"` + Tolerations []SandboxJobToleration `yaml:"tolerations,omitempty" json:"tolerations,omitempty"` +} + +// SandboxOpenShell returns the effective OpenShell runtime settings. The +// gateway, workspace, token, TLS and claim fields are cluster facts, so a +// per-agent block falls back to the global one for each it leaves empty; +// scheduling fields (node selector, tolerations, runtime class) and the +// extra egress hosts are per-agent choices and replace the global ones +// wholesale when the per-agent block sets them. +func (a *AgentConfig) SandboxOpenShell(global AgentSandboxConfig) SandboxOpenShellConfig { + var g SandboxOpenShellConfig + if global.OpenShell != nil { + g = *global.OpenShell + } + if a == nil || a.Sandbox == nil || a.Sandbox.OpenShell == nil { + return g + } + out := *a.Sandbox.OpenShell + fallback := func(dst *string, src string) { + if strings.TrimSpace(*dst) == "" { + *dst = src + } + } + fallback(&out.GatewayURL, g.GatewayURL) + fallback(&out.Namespace, g.Namespace) + fallback(&out.PolicyProfile, g.PolicyProfile) + fallback(&out.TokenEnv, g.TokenEnv) + fallback(&out.WorkspaceClaim, g.WorkspaceClaim) + fallback(&out.WorkspaceClaimMount, g.WorkspaceClaimMount) + fallback(&out.RuntimeClassName, g.RuntimeClassName) + if out.TimeoutS == 0 { + out.TimeoutS = g.TimeoutS + } + if !out.InsecureTLS { + out.InsecureTLS = g.InsecureTLS + } + if len(out.AllowHosts) == 0 { + out.AllowHosts = g.AllowHosts + } + if len(out.Binaries) == 0 { + out.Binaries = g.Binaries + } + if len(out.NodeSelector) == 0 { + out.NodeSelector = g.NodeSelector + } + if len(out.Tolerations) == 0 { + out.Tolerations = g.Tolerations + } + return out +} diff --git a/src/crux/config/agent_config_openshell_test.go b/src/crux/config/agent_config_openshell_test.go new file mode 100644 index 0000000000..d371febd08 --- /dev/null +++ b/src/crux/config/agent_config_openshell_test.go @@ -0,0 +1,121 @@ +package config + +import ( + "reflect" + "testing" +) + +func TestSandboxOpenShellFallsBackToGlobal(t *testing.T) { + var nilA *AgentConfig + if got := nilA.SandboxOpenShell(AgentSandboxConfig{}); !reflect.DeepEqual(got, SandboxOpenShellConfig{}) { + t.Fatalf("nil agent, nil global = %+v, want zero value", got) + } + + global := AgentSandboxConfig{OpenShell: &SandboxOpenShellConfig{ + GatewayURL: "https://gw.example:443", Namespace: "hive", PolicyProfile: "strict", + TimeoutS: 900, TokenEnv: "OPENSHELL_TOKEN", InsecureTLS: true, + WorkspaceClaim: "hive-data", WorkspaceClaimMount: "/data", + AllowHosts: []string{"hub.example"}, Binaries: []string{"git", "gh"}, + RuntimeClassName: "gvisor", NodeSelector: map[string]string{"pool": "sandbox"}, + Tolerations: []SandboxJobToleration{{Key: "sandbox", Operator: "Exists"}}, + }} + if got := nilA.SandboxOpenShell(global); !reflect.DeepEqual(got, *global.OpenShell) { + t.Fatalf("nil agent = %+v, want global block", got) + } + noBlock := &AgentConfig{Sandbox: &AgentSandboxOverride{Image: "a-img"}} + if got := noBlock.SandboxOpenShell(global); !reflect.DeepEqual(got, *global.OpenShell) { + t.Fatalf("agent without openshell block = %+v, want global block", got) + } + + // A per-agent block that names nothing but whitespace inherits every + // cluster fact and every scheduling field from the global one. + empty := &AgentConfig{Sandbox: &AgentSandboxOverride{OpenShell: &SandboxOpenShellConfig{GatewayURL: " "}}} + if got := empty.SandboxOpenShell(global); !reflect.DeepEqual(got, *global.OpenShell) { + t.Fatalf("blank per-agent block = %+v, want global block", got) + } +} + +func TestSandboxOpenShellPerAgentFieldsWin(t *testing.T) { + global := AgentSandboxConfig{OpenShell: &SandboxOpenShellConfig{ + GatewayURL: "gw-global:50051", Namespace: "global-ns", PolicyProfile: "global-policy", + TimeoutS: 900, TokenEnv: "GLOBAL_TOKEN", InsecureTLS: false, + WorkspaceClaim: "global-claim", WorkspaceClaimMount: "/global", + AllowHosts: []string{"global.example", "hub.example"}, Binaries: []string{"git", "gh", "curl"}, + RuntimeClassName: "gvisor", NodeSelector: map[string]string{"pool": "global"}, + Tolerations: []SandboxJobToleration{{Key: "global", Operator: "Exists"}, {Key: "spot", Operator: "Exists"}}, + }} + a := &AgentConfig{Sandbox: &AgentSandboxOverride{OpenShell: &SandboxOpenShellConfig{ + GatewayURL: "gw-agent:50051", TimeoutS: 60, InsecureTLS: true, + WorkspaceClaimMount: "/agent", + AllowHosts: []string{"models.example"}, Binaries: []string{"node"}, + RuntimeClassName: "kata", NodeSelector: map[string]string{"gpu": "a100"}, + Tolerations: []SandboxJobToleration{{Key: "gpu", Operator: "Exists"}}, + }}} + got := a.SandboxOpenShell(global) + + // Cluster facts: set per-agent wins, empty per-agent inherits. + if got.GatewayURL != "gw-agent:50051" || got.TimeoutS != 60 || !got.InsecureTLS || got.WorkspaceClaimMount != "/agent" { + t.Fatalf("per-agent cluster facts not kept: %+v", got) + } + if got.Namespace != "global-ns" || got.PolicyProfile != "global-policy" || got.TokenEnv != "GLOBAL_TOKEN" || got.WorkspaceClaim != "global-claim" { + t.Fatalf("empty per-agent cluster facts not inherited: %+v", got) + } + + // Scheduling and egress fields replace the global ones wholesale: the + // global hosts, binaries, selector keys and tolerations must not leak in. + if !reflect.DeepEqual(got.AllowHosts, []string{"models.example"}) { + t.Fatalf("allow_hosts = %v, want per-agent list only", got.AllowHosts) + } + if !reflect.DeepEqual(got.Binaries, []string{"node"}) { + t.Fatalf("binaries = %v, want per-agent list only", got.Binaries) + } + if got.RuntimeClassName != "kata" { + t.Fatalf("runtime_class_name = %q, want kata", got.RuntimeClassName) + } + if !reflect.DeepEqual(got.NodeSelector, map[string]string{"gpu": "a100"}) { + t.Fatalf("node_selector = %v, want per-agent map only", got.NodeSelector) + } + if !reflect.DeepEqual(got.Tolerations, []SandboxJobToleration{{Key: "gpu", Operator: "Exists"}}) { + t.Fatalf("tolerations = %v, want per-agent list only", got.Tolerations) + } + + // A global InsecureTLS cannot be switched off per-agent (false is the + // zero value, so it reads as "unset" and inherits the global true). + global.OpenShell.InsecureTLS = true + a.Sandbox.OpenShell.InsecureTLS = false + if !a.SandboxOpenShell(global).InsecureTLS { + t.Fatal("per-agent InsecureTLS=false inherits global true") + } + + // The merge must not write the inherited values back into its inputs. + if global.OpenShell.AllowHosts[0] != "global.example" || a.Sandbox.OpenShell.Namespace != "" { + t.Fatal("merge mutated its inputs") + } +} + +func TestOpenShellPreflightRegistry(t *testing.T) { + t.Cleanup(func() { SetOpenShellPreflight(nil) }) + cfgs := []SandboxOpenShellConfig{{GatewayURL: "gw:50051"}} + + SetOpenShellPreflight(nil) + if got := OpenShellPreflightWarnings(cfgs); got != nil { + t.Fatalf("unregistered preflight = %v, want nil", got) + } + + var seen []SandboxOpenShellConfig + SetOpenShellPreflight(func(c []SandboxOpenShellConfig) []string { + seen = c + return []string{"gateway unreachable"} + }) + if got := OpenShellPreflightWarnings(nil); got != nil || seen != nil { + t.Fatalf("empty cfgs = %v (called=%v), want nil and no call", got, seen != nil) + } + if got := OpenShellPreflightWarnings(cfgs); !reflect.DeepEqual(got, []string{"gateway unreachable"}) || !reflect.DeepEqual(seen, cfgs) { + t.Fatalf("registered preflight = %v (seen %v), want its warnings for the given cfgs", got, seen) + } + + SetOpenShellPreflight(nil) + if got := OpenShellPreflightWarnings(cfgs); got != nil { + t.Fatalf("after unregister = %v, want nil", got) + } +} diff --git a/src/crux/config/sandbox_runtime.go b/src/crux/config/sandbox_runtime.go new file mode 100644 index 0000000000..994c45d788 --- /dev/null +++ b/src/crux/config/sandbox_runtime.go @@ -0,0 +1,109 @@ +package config + +import "strings" + +// Sandbox runtimes. SandboxRuntimePodman is the default when nothing is set. +const ( + SandboxRuntimePodman = "podman" + SandboxRuntimeJob = "job" + SandboxRuntimeOpenShell = "openshell" +) + +// ValidSandboxRuntimes is the closed set a config may name. +var ValidSandboxRuntimes = map[string]bool{SandboxRuntimePodman: true, SandboxRuntimeJob: true, SandboxRuntimeOpenShell: true} + +// SandboxJobConfig is everything the Job runtime needs beyond the image +// (#6311). The pod-template fields pass through to Kubernetes verbatim. +// +// The workspace is shared between hive and the Job by mounting the SAME +// PersistentVolumeClaim in both: WorkspaceClaim names it, and +// WorkspaceClaimMount is where hive sees it (default /data, where the +// standalone and hub-provisioned deployments mount hive-data). The sandbox +// workspace root must live under that mount. When the Job lands on a +// different node than hive, the claim has to be ReadWriteMany. +// +// EnvFromSecrets is the ONLY credential path into the Job. Hive's brokered +// GitHub token never enters it; hive pushes and opens the PR from the +// workspace after the Job exits, exactly as it does for the Podman runtime. +type SandboxJobConfig struct { + WorkspaceClaim string `yaml:"workspace_claim,omitempty" json:"workspace_claim,omitempty"` + WorkspaceClaimMount string `yaml:"workspace_claim_mount,omitempty" json:"workspace_claim_mount,omitempty"` + NodeSelector map[string]string `yaml:"node_selector,omitempty" json:"node_selector,omitempty"` + Tolerations []SandboxJobToleration `yaml:"tolerations,omitempty" json:"tolerations,omitempty"` + Resources SandboxJobResources `yaml:"resources,omitempty" json:"resources,omitempty"` + ServiceAccount string `yaml:"service_account,omitempty" json:"service_account,omitempty"` + // SchedulerName sets the pod's schedulerName. Accelerator clusters often + // gate device-requesting pods behind a custom scheduler via an admission + // webhook (e.g. `A pod must use "schedulerName: spyre-scheduler"`); + // without this the Job create is denied before any pod exists. + SchedulerName string `yaml:"scheduler_name,omitempty" json:"scheduler_name,omitempty"` + EnvFromSecrets []string `yaml:"env_from_secrets,omitempty" json:"env_from_secrets,omitempty"` + Volumes []SandboxJobVolume `yaml:"volumes,omitempty" json:"volumes,omitempty"` + TTLSeconds int `yaml:"ttl_seconds,omitempty" json:"ttl_seconds,omitempty"` +} + +// SandboxJobToleration mirrors the subset of a Kubernetes toleration an +// operator writes by hand. +type SandboxJobToleration struct { + Key string `yaml:"key,omitempty" json:"key,omitempty"` + Operator string `yaml:"operator,omitempty" json:"operator,omitempty"` + Value string `yaml:"value,omitempty" json:"value,omitempty"` + Effect string `yaml:"effect,omitempty" json:"effect,omitempty"` +} + +// SandboxJobResources mirrors Kubernetes resource requirements with string +// quantities so a device limit (`.com/: "2"`) passes through. +type SandboxJobResources struct { + Limits map[string]string `yaml:"limits,omitempty" json:"limits,omitempty"` + Requests map[string]string `yaml:"requests,omitempty" json:"requests,omitempty"` +} + +// SandboxJobVolume is an additional PVC mounted into the Job (a compile +// cache, a model store). +type SandboxJobVolume struct { + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Claim string `yaml:"claim" json:"claim"` + MountPath string `yaml:"mount_path" json:"mount_path"` + ReadOnly bool `yaml:"read_only,omitempty" json:"read_only,omitempty"` +} + +// SandboxRuntime returns the per-agent runtime override, then the global +// runtime, then SandboxRuntimePodman. The value is returned as written; the +// manager rejects one outside ValidSandboxRuntimes at kick time. +func (a *AgentConfig) SandboxRuntime(global AgentSandboxConfig) string { + if a != nil && a.Sandbox != nil && strings.TrimSpace(a.Sandbox.Runtime) != "" { + return strings.TrimSpace(a.Sandbox.Runtime) + } + if strings.TrimSpace(global.Runtime) != "" { + return strings.TrimSpace(global.Runtime) + } + return SandboxRuntimePodman +} + +// SandboxJob returns the effective Job runtime settings. A per-agent job +// block replaces the global one wholesale for the pod-template fields (node +// selector, tolerations, resources, secrets, volumes, service account) — an +// agent that needs an accelerator names its own scheduling. The claim +// fields and the TTL are cluster facts rather than per-agent choices, so +// they fall back to the global block when the per-agent one leaves them +// empty. +func (a *AgentConfig) SandboxJob(global AgentSandboxConfig) SandboxJobConfig { + var g SandboxJobConfig + if global.Job != nil { + g = *global.Job + } + if a == nil || a.Sandbox == nil || a.Sandbox.Job == nil { + return g + } + out := *a.Sandbox.Job + if strings.TrimSpace(out.WorkspaceClaim) == "" { + out.WorkspaceClaim = g.WorkspaceClaim + } + if strings.TrimSpace(out.WorkspaceClaimMount) == "" { + out.WorkspaceClaimMount = g.WorkspaceClaimMount + } + if out.TTLSeconds == 0 { + out.TTLSeconds = g.TTLSeconds + } + return out +} diff --git a/src/crux/go.mod b/src/crux/go.mod index b40cfe6e66..ec82f74fff 100644 --- a/src/crux/go.mod +++ b/src/crux/go.mod @@ -3,6 +3,7 @@ module github.com/hivecommons/hive/crux go 1.26.11 require ( + github.com/ai-openshell/sdk-go v0.1.8 github.com/google/cel-go v0.31.0 github.com/nats-io/nats-server/v2 v2.15.2 github.com/nats-io/nats.go v1.55.0 @@ -21,7 +22,7 @@ require ( require go.opentelemetry.io/otel/log v1.47.0 // indirect require ( - cel.dev/expr v0.25.2 // indirect + cel.dev/expr v0.25.3 // indirect github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect @@ -43,10 +44,12 @@ require ( golang.org/x/crypto v0.58.0 // indirect golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect golang.org/x/net v0.62.0 // indirect + golang.org/x/oauth2 v0.37.0 // indirect + golang.org/x/sync v0.24.0 // indirect golang.org/x/text v0.43.0 // indirect golang.org/x/time v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect - google.golang.org/grpc v1.83.2 // indirect + google.golang.org/grpc v1.85.0 // indirect google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/src/crux/go.sum b/src/crux/go.sum index 8bac4b5df6..43da421f41 100644 --- a/src/crux/go.sum +++ b/src/crux/go.sum @@ -1,5 +1,7 @@ -cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= -cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/ai-openshell/sdk-go v0.1.8 h1:p5i7VAZxlB/tzWy6KQWkdvX3cLIu8x0AinUx1X6wp+s= +github.com/ai-openshell/sdk-go v0.1.8/go.mod h1:nYYe/HYeEct0KG5YyF2dbLHjUWb7EEA+I1lLi6A8j14= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= @@ -83,6 +85,10 @@ golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++ golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= +golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= @@ -96,8 +102,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1: google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= -google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= -google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw= +google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/src/crux/logscrub/pane.go b/src/crux/logscrub/pane.go index 887cb69086..9de05d20c6 100644 --- a/src/crux/logscrub/pane.go +++ b/src/crux/logscrub/pane.go @@ -36,7 +36,9 @@ var deviceCodeLineRedactor = regexp.MustCompile(`(?m)^.*(?:login/device|Waiting var apiKeyRedactor = regexp.MustCompile(`(^|[^A-Za-z0-9._-])(sk-[A-Za-z0-9._\-]{6,})`) // RedactPaneTokens masks GitHub tokens, sk- API keys and device-login one-time -// codes/lines in text derived from an agent pane. +// codes/lines in text derived from an agent pane, then runs ScrubString so every +// other secret shape it covers (JWTs, AWS keys, Slack tokens/webhooks, Bearer +// and Basic credentials, private-key blocks) is masked too (#17665). func RedactPaneTokens(s string) string { s = statusJWTRedactor.ReplaceAllString(s, "[REDACTED]") s = statusBearerRedactor.ReplaceAllString(s, "[REDACTED]") @@ -48,5 +50,7 @@ func RedactPaneTokens(s string) string { s = apiKeyRedactor.ReplaceAllString(s, "${1}sk-***REDACTED***") s = deviceCodeRedactor.ReplaceAllString(s, "${1}****-****") s = deviceCodeLineRedactor.ReplaceAllString(s, "[auth flow redacted]") - return s + // Runs last so the pane-style replacements above stay byte-identical: neither + // "ghp_abc***" nor "sk-***REDACTED***" matches a ScrubString pattern. + return ScrubString(s) } diff --git a/src/crux/logscrub/pane_test.go b/src/crux/logscrub/pane_test.go index 30c9852fde..70fe992a2a 100644 --- a/src/crux/logscrub/pane_test.go +++ b/src/crux/logscrub/pane_test.go @@ -23,6 +23,12 @@ func TestRedactPaneTokens(t *testing.T) { {"waiting for authorization", "Waiting for authorization...\n", "[auth flow redacted]\n"}, {"press any key", "Press any key to copy code\ndone", "[auth flow redacted]\ndone"}, {"multiple secrets across lines", "a ghs_abcdefghij1234\nb sk-hive-qa1234\n", "a ghs_abc***\nb sk-***REDACTED***\n"}, + {"jwt", "t eyJhbGciOiJIUzI1NiIsInR5cCI6.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFt.SflKxwRJSMeKKF2QT4fwpMeJf36POk end", "t [REDACTED] end"}, + {"aws access key", "key AKIAIOSFODNN7EXAMPLE end", "key [REDACTED] end"}, + {"slack token", "tok xoxb-1234567890-abcdefghij end", "tok [REDACTED] end"}, + {"bearer header", "Authorization: Bearer abcdefghij0123456789xyz", "Authorization: [REDACTED]"}, + {"private key block", "a\n-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAA\n-----END OPENSSH PRIVATE KEY-----\nb", "a\n[REDACTED]\nb"}, + {"github and sk with aws", "ghp_abcdefghij1234567890 sk-hive-qa1234 AKIAIOSFODNN7EXAMPLE", "ghp_abc*** sk-***REDACTED*** [REDACTED]"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index aa131f4065..35a1ad0844 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -21,6 +21,7 @@ import ( "github.com/hivecommons/hive/crux/agent" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/inferencehealth" + "github.com/hivecommons/hive/crux/upstreamproxy" ) const ( @@ -289,6 +290,11 @@ type GitHubProxy struct { // without network access. tunnelDial func(host string) (net.Conn, error) + // upstream, when set, names the hosts whose upstream leg is chained + // through a corporate HTTP forward proxy (SetUpstreamProxy). nil keeps + // every upstream dial direct. + upstream *upstreamproxy.Selector + canariesEnabled bool canaryFailClosed bool canaryRegistry CanaryScanner diff --git a/src/crux/proxy/github_proxy_connect.go b/src/crux/proxy/github_proxy_connect.go index ab26876bae..6cb72dd604 100644 --- a/src/crux/proxy/github_proxy_connect.go +++ b/src/crux/proxy/github_proxy_connect.go @@ -213,7 +213,7 @@ func (p *GitHubProxy) handleTransparentTLS(conn net.Conn, peeked []byte) { } defer func() { _ = tlsClientConn.Close() }() - upstreamConn, err := tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", host+":443", &tls.Config{ServerName: host}) + upstreamConn, err := p.dialUpstreamTLS(host+":443", host, upstreamDialTimeout) if err != nil { p.logger.Error("transparent proxy upstream dial failed", "host", host, "error", err) return @@ -282,7 +282,7 @@ func (p *GitHubProxy) dialTunnelUpstream(host string) (net.Conn, error) { if p.tunnelDial != nil { return p.tunnelDial(host) } - return p.tunnelDialer(transparentProxyTimeout).Dial("tcp", host+":443") + return p.dialUpstreamRaw(host+":443", transparentProxyTimeout) } const tlsClientHelloMaxSize = 4096 @@ -534,9 +534,7 @@ func (p *GitHubProxy) handleConnectDirect(conn net.Conn, r *http.Request) { // curl sits in poll() waiting for "200 Connection established" that never // comes). The deadline is lifted once established so the relay is not // time-boxed. - upstreamConn, err := tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", r.Host, &tls.Config{ - ServerName: host, - }) + upstreamConn, err := p.dialUpstreamTLS(r.Host, host, upstreamDialTimeout) if err != nil { p.logger.Error("proxy upstream dial failed", "host", r.Host, "error", err) return @@ -611,7 +609,7 @@ func (p *GitHubProxy) tunnelDirect(conn net.Conn, r *http.Request) { // is destination-guarded like the plain forward path: a CONNECT names an // arbitrary host:port, and without the guard it was a raw TCP tunnel from // the proxy's uid to the cloud metadata service or the cluster estate. - upstream, err := p.tunnelDialer(tunnelDialTimeout).Dial("tcp", r.Host) + upstream, err := p.dialUpstreamRaw(r.Host, tunnelDialTimeout) if err != nil { var denied *plainForwardDenied if errors.As(err, &denied) { diff --git a/src/crux/proxy/github_proxy_copilot.go b/src/crux/proxy/github_proxy_copilot.go index f4ba208f26..b05b7677ee 100644 --- a/src/crux/proxy/github_proxy_copilot.go +++ b/src/crux/proxy/github_proxy_copilot.go @@ -21,7 +21,7 @@ func (p *GitHubProxy) dialCopilotUpstream(host string) (net.Conn, error) { // SO_MARK the outbound socket so the forced-egress redirect exempts the // proxy's own upstream traffic (see proxyEgressMark). The override hook above // still wins when set, so tests are unaffected. - return tls.DialWithDialer(markDialer(upstreamDialTimeout), "tcp", net.JoinHostPort(host, "443"), &tls.Config{ServerName: host}) + return p.dialUpstreamTLS(net.JoinHostPort(host, "443"), host, upstreamDialTimeout) } // copilotInternalPathPrefix marks the Copilot CLI's auth-exchange endpoints on diff --git a/src/crux/proxy/github_proxy_inference.go b/src/crux/proxy/github_proxy_inference.go index 489d1ba0a5..1d22313c5d 100644 --- a/src/crux/proxy/github_proxy_inference.go +++ b/src/crux/proxy/github_proxy_inference.go @@ -411,7 +411,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { route := p.inference.Get(agentName) if route == nil { - p.logger.Warn("inference no route", "agent", agentName) + p.logger.Warn("inference no route", "agent", agentName, "reason", p.inference.MissReason(agentName)) http.Error(w, `{"type":"error","error":{"type":"api_error","message":"no inference route for agent"}}`, http.StatusBadGateway) return } diff --git a/src/crux/proxy/graphql_mutation.go b/src/crux/proxy/graphql_mutation.go new file mode 100644 index 0000000000..84f0de2b29 --- /dev/null +++ b/src/crux/proxy/graphql_mutation.go @@ -0,0 +1,11 @@ +package proxy + +// isGitHubGraphQLMutation reports whether the document holds a depth-zero +// mutation operation. An anchored `^\s*mutation` regex misses same-line +// prefixes (a leading fragment, comma or BOM), so every GitHub GraphQL gate +// uses the string- and comment-aware token scan the Linear path already +// relies on. A document the scan cannot read (an unterminated string) is +// treated as a mutation: unclassifiable input fails closed. +func isGitHubGraphQLMutation(query string) bool { + return countGraphQLKeyword(query, "mutation") != 0 +} diff --git a/src/crux/proxy/graphql_mutation_prefix_test.go b/src/crux/proxy/graphql_mutation_prefix_test.go new file mode 100644 index 0000000000..cfc94ba007 --- /dev/null +++ b/src/crux/proxy/graphql_mutation_prefix_test.go @@ -0,0 +1,53 @@ +package proxy + +import ( + "testing" + + "github.com/hivecommons/hive/crux/agent" +) + +func TestGraphQLMutationGate_SameLinePrefixes(t *testing.T) { + const mut = `mutation { addComment(input:{subjectId:"I_kw", body:"hi"}) { clientMutationId } }` + cases := []struct { + name, query string + write bool + }{ + {"plain mutation", mut, true}, + {"fragment-first", `fragment F on Repository { id } ` + mut, true}, + {"leading comma", `,` + mut, true}, + {"leading BOM", "\uFEFF" + mut, true}, + {"plain query", `query { viewer { login } }`, false}, + {"fragment-first query", `fragment F on Repository { id } query { viewer { login } }`, false}, + {"mutation word in a string", `query { search(query: "mutation { x }") { issueCount } }`, false}, + {"escaped triple quote in block string", `fragment F on Repository { ref(qualifiedName: """\"""""") { id } } +mutation { addComment(input:{subjectId:"x", body:"y"}) { subject { ... on Issue { repository { ...F } } } } }`, true}, + {"unterminated string fails closed", `query { search(query: "abc) { issueCount } }`, true}, + {"unterminated block string fails closed", `query { search(query: """abc) { issueCount } }`, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + body := graphQLBody(t, tc.query) + allowed, write := GraphQLAllowedCaps(agent.ModeAdvisory, agent.AgentCapabilities{}, body) + if allowed == tc.write || write != tc.write { + t.Errorf("allowed=%v write=%v, want allowed=%v write=%v", allowed, write, !tc.write, tc.write) + } + }) + } +} + +func TestGraphQLAutoMergeRefusal_SameLinePrefixes(t *testing.T) { + enabled := autoMergeOffFor("acme/frozen") + for name, q := range map[string]string{ + "fragment-first": `fragment F on Repository { id } ` + mergeMutationQuery, + "leading comma": `,` + mergeMutationQuery, + "leading BOM": "\uFEFF" + mergeMutationQuery, + } { + t.Run(name, func(t *testing.T) { + body := string(graphQLBody(t, q)) + body = body[:len(body)-1] + `,"variables":{"repo":"acme/frozen"}}` + if _, refused := GraphQLAutoMergeRefusal(enabled, []byte(body)); !refused { + t.Fatal("merge on an auto-merge-off repo was not refused") + } + }) + } +} diff --git a/src/crux/proxy/inference_no_route_test.go b/src/crux/proxy/inference_no_route_test.go new file mode 100644 index 0000000000..b2f7f25814 --- /dev/null +++ b/src/crux/proxy/inference_no_route_test.go @@ -0,0 +1,42 @@ +package proxy + +import ( + "bytes" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestInferenceNoRouteReason(t *testing.T) { + for _, tc := range []struct { + name string + setup func(p *GitHubProxy) + reason string + }{ + {name: "never set", setup: func(*GitHubProxy) {}, reason: "reason=never_set"}, + {name: "cleared", setup: func(p *GitHubProxy) { + p.inference.Set("scanner", &InferenceRoute{Backend: "litellm"}) + p.inference.Clear("scanner") + }, reason: "reason=cleared"}, + } { + t.Run(tc.name, func(t *testing.T) { + p := newInferenceTestProxy() + var logs bytes.Buffer + p.logger = slog.New(slog.NewTextHandler(&logs, nil)) + tc.setup(p) + req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(`{}`)) + req.Header.Set("x-api-key", "sk-hive-scanner") + w := httptest.NewRecorder() + p.inferenceTranslatorHandler().ServeHTTP(w, req) + if w.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502", w.Code) + } + out := logs.String() + if !strings.Contains(out, "inference no route") || !strings.Contains(out, tc.reason) { + t.Fatalf("log missing %q: %s", tc.reason, out) + } + }) + } +} diff --git a/src/crux/proxy/inference_route.go b/src/crux/proxy/inference_route.go index 4f35d29dec..654d49cf2d 100644 --- a/src/crux/proxy/inference_route.go +++ b/src/crux/proxy/inference_route.go @@ -128,11 +128,15 @@ var DefaultInferenceTools = []openaiTool{ type inferenceRouter struct { mu sync.RWMutex routes map[string]*InferenceRoute // agent name → route + // cleared remembers agents whose route was removed, so a later "no route" + // can be told apart from an agent that never had one. + cleared map[string]struct{} } func newInferenceRouter() *inferenceRouter { return &inferenceRouter{ - routes: make(map[string]*InferenceRoute), + routes: make(map[string]*InferenceRoute), + cleared: make(map[string]struct{}), } } @@ -140,12 +144,25 @@ func (ir *inferenceRouter) Set(agentName string, route *InferenceRoute) { ir.mu.Lock() defer ir.mu.Unlock() ir.routes[agentName] = route + delete(ir.cleared, agentName) } func (ir *inferenceRouter) Clear(agentName string) { ir.mu.Lock() defer ir.mu.Unlock() delete(ir.routes, agentName) + ir.cleared[agentName] = struct{}{} +} + +// MissReason says why Get(agentName) found no route: "cleared" when one was +// installed and later removed, "never_set" otherwise. +func (ir *inferenceRouter) MissReason(agentName string) string { + ir.mu.RLock() + defer ir.mu.RUnlock() + if _, ok := ir.cleared[agentName]; ok { + return "cleared" + } + return "never_set" } func (ir *inferenceRouter) Get(agentName string) *InferenceRoute { diff --git a/src/crux/proxy/linear_rules.go b/src/crux/proxy/linear_rules.go index cb8dce203a..daf6394718 100644 --- a/src/crux/proxy/linear_rules.go +++ b/src/crux/proxy/linear_rules.go @@ -371,14 +371,19 @@ func linearOperationStart(query string) (linearOpKind, int, bool) { // countGraphQLKeyword counts occurrences of an operation keyword appearing as a // standalone token at brace depth zero, ignoring occurrences inside strings and // comments. A substring count would miscount a field named `queryable` or the -// word "mutation" inside a comment body. +// word "mutation" inside a comment body. Returns -1 when the document cannot be +// scanned (an unterminated string), so callers can fail closed. func countGraphQLKeyword(query, kw string) int { n := 0 - scanGraphQLTokens(query, func(name string, depth int) { + ok := scanGraphQLTokensAt(query, func(name string, depth, _ int) bool { if depth == 0 && name == kw { n++ } + return true }) + if !ok { + return -1 + } return n } @@ -396,17 +401,11 @@ func indexGraphQLKeyword(query, kw string) int { return idx } -func scanGraphQLTokens(query string, fn func(name string, depth int)) { - scanGraphQLTokensAt(query, func(name string, depth, _ int) bool { - fn(name, depth) - return true - }) -} - // scanGraphQLTokensAt walks the document emitting each bare name token with the // brace depth it was found at, skipping strings and comments so their contents -// are never mistaken for tokens. fn returns false to stop the walk. -func scanGraphQLTokensAt(query string, fn func(name string, depth, off int) bool) { +// are never mistaken for tokens. fn returns false to stop the walk. Returns +// false only when the walk aborted on an unreadable (unterminated) literal. +func scanGraphQLTokensAt(query string, fn func(name string, depth, off int) bool) bool { depth := 0 for i := 0; i < len(query); { c := query[i] @@ -418,7 +417,7 @@ func scanGraphQLTokensAt(query string, fn func(name string, depth, off int) bool case c == '"': var ok bool if i, ok = skipGraphQLString(query, i); !ok { - return + return false } case c == '{': depth++ @@ -429,25 +428,32 @@ func scanGraphQLTokensAt(query string, fn func(name string, depth, off int) bool case isGraphQLNameStart(c): name := graphQLIdentRe.FindString(query[i:]) if !fn(name, depth, i) { - return + return true } i += len(name) default: i++ } } + return true } // skipGraphQLString advances past a string or block-string literal beginning at // i. Returns ok=false on an unterminated literal, which the callers treat as an -// unreadable document. +// unreadable document. Inside a block string `\"""` is an escaped triple quote, +// not the terminator. func skipGraphQLString(query string, i int) (int, bool) { if strings.HasPrefix(query[i:], `"""`) { - end := strings.Index(query[i+3:], `"""`) - if end < 0 { - return 0, false + for j := i + 3; j < len(query); j++ { + if strings.HasPrefix(query[j:], `\"""`) { + j += 3 + continue + } + if strings.HasPrefix(query[j:], `"""`) { + return j + 3, true + } } - return i + 3 + end + 3, true + return 0, false } i++ // opening quote for i < len(query) { diff --git a/src/crux/proxy/rules.go b/src/crux/proxy/rules.go index a40e7eb844..9f8db7751f 100644 --- a/src/crux/proxy/rules.go +++ b/src/crux/proxy/rules.go @@ -577,8 +577,6 @@ type graphQLRequest struct { Variables any `json:"variables"` } -var graphQLMutationRe = regexp.MustCompile(`(?m)^\s*mutation\b`) - // graphQLMergeMutationRe matches mutations that merge a pull request. Merging is // the highest-privilege write and must require the same mode as the REST // `PUT /pulls/{n}/merge` rule (ModeIssuesPRsMerge) — otherwise an ISSUES_ONLY @@ -804,7 +802,7 @@ func GraphQLAllowedCaps(mode agent.AgentMode, caps agent.AgentCapabilities, body } query := strings.TrimSpace(req.Query) - if !graphQLMutationRe.MatchString(query) { + if !isGitHubGraphQLMutation(query) { return true, false // read-only query } @@ -843,7 +841,7 @@ func GraphQLAutoMergeRefusal(enabled func(repo string) bool, body []byte) (strin if err := json.Unmarshal(body, &req); err != nil { return "", false } - if !graphQLMutationRe.MatchString(strings.TrimSpace(req.Query)) || !graphQLMergeMutationRe.MatchString(req.Query) { + if !isGitHubGraphQLMutation(req.Query) || !graphQLMergeMutationRe.MatchString(req.Query) { return "", false } if repo := repoFromGraphQLValue(req.Variables); repo != "" && !enabled(repo) { diff --git a/src/crux/proxy/upstream_chain.go b/src/crux/proxy/upstream_chain.go new file mode 100644 index 0000000000..dfd72f1762 --- /dev/null +++ b/src/crux/proxy/upstream_chain.go @@ -0,0 +1,84 @@ +package proxy + +import ( + "context" + "crypto/tls" + "net" + "net/netip" + "strings" + "time" + + "github.com/hivecommons/hive/crux/upstreamproxy" +) + +// SetUpstreamProxy installs the rule set deciding which upstream hosts this +// proxy chains through a corporate HTTP forward proxy (CONNECT) instead of +// dialling directly. nil (the default) keeps every upstream dial direct. The +// motivating case is a GitHub Enterprise host reachable only via an +// enterprise egress proxy: agents' git/gh traffic is iptables-redirected into +// this proxy, so this is the one place the chaining can happen for them. +func (p *GitHubProxy) SetUpstreamProxy(sel *upstreamproxy.Selector) { + p.upstream = sel + if sel != nil && p.logger != nil { + p.logger.Info("proxy: upstream proxy chaining enabled", + "proxy", redactedProxyURL(sel), "source", sel.Source(), "restricted", sel.Restricted()) + } +} + +func redactedProxyURL(sel *upstreamproxy.Selector) string { + u := sel.URL() + if u == nil { + return "" + } + return u.Scheme + "://" + u.Host +} + +// dialUpstreamTLS opens the TLS leg to hostport for a MITM'd flow, presenting +// serverName for SNI/verification. When the upstream selector names the host +// the TCP leg is a CONNECT tunnel through the upstream proxy; otherwise it is +// the direct SO_MARKed dial used before chaining existed. timeout bounds +// connect (+ CONNECT exchange) + TLS handshake together, as +// tls.DialWithDialer did. +func (p *GitHubProxy) dialUpstreamTLS(hostport, serverName string, timeout time.Duration) (net.Conn, error) { + proxyURL := p.upstream.ProxyFor(hostport) + if proxyURL == nil { + return tls.DialWithDialer(markDialer(timeout), "tcp", hostport, &tls.Config{ServerName: serverName}) + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + raw, err := upstreamproxy.Dial(ctx, markDialer(timeout).DialContext, proxyURL, hostport) + if err != nil { + return nil, err + } + tlsConn := tls.Client(raw, &tls.Config{ServerName: serverName}) + if err := tlsConn.HandshakeContext(ctx); err != nil { + _ = raw.Close() + return nil, err + } + return tlsConn, nil +} + +// dialUpstreamRaw opens the raw TCP leg of an uninspected tunnel (CONNECT +// passthrough or transparent-TLS passthrough) to hostport: through the +// upstream proxy when the selector names the host, otherwise with the +// destination-guarded direct dialer. On the chained path the proxy resolves +// the name, so the guard can only classify a literal IP destination; it does +// so before handing the target to the proxy. +func (p *GitHubProxy) dialUpstreamRaw(hostport string, timeout time.Duration) (net.Conn, error) { + proxyURL := p.upstream.ProxyFor(hostport) + if proxyURL == nil { + return p.tunnelDialer(timeout).Dial("tcp", hostport) + } + if !p.allowPrivateForward { + if host, _, err := net.SplitHostPort(hostport); err == nil { + if addr, err := netip.ParseAddr(strings.Trim(host, "[]")); err == nil { + if reason := plainForwardDenyReasonAddr(addr); reason != "" { + return nil, &plainForwardDenied{target: hostport, reason: reason} + } + } + } + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return upstreamproxy.Dial(ctx, markDialer(timeout).DialContext, proxyURL, hostport) +} diff --git a/src/crux/proxy/upstream_chain_test.go b/src/crux/proxy/upstream_chain_test.go new file mode 100644 index 0000000000..4c902d3483 --- /dev/null +++ b/src/crux/proxy/upstream_chain_test.go @@ -0,0 +1,261 @@ +package proxy + +import ( + "bufio" + "crypto/tls" + "crypto/x509" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/crux/upstreamproxy" +) + +// testConnectProxy is a minimal CONNECT-only forward proxy that records the +// targets it was asked to tunnel to. +type testConnectProxy struct { + ln net.Listener + targets atomic.Int32 + last atomic.Value +} + +func startTestConnectProxy(t *testing.T) *testConnectProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + cp := &testConnectProxy{ln: ln} + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go cp.serve(c) + } + }() + return cp +} + +func (cp *testConnectProxy) serve(c net.Conn) { + defer func() { _ = c.Close() }() + br := bufio.NewReader(c) + req, err := http.ReadRequest(br) + if err != nil || req.Method != http.MethodConnect { + _, _ = io.WriteString(c, "HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\n\r\n") + return + } + cp.targets.Add(1) + cp.last.Store(req.Host) + up, err := net.DialTimeout("tcp", req.Host, 5*time.Second) + if err != nil { + _, _ = io.WriteString(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + return + } + defer func() { _ = up.Close() }() + _, _ = io.WriteString(c, "HTTP/1.1 200 Connection established\r\n\r\n") + go func() { _, _ = io.Copy(up, br) }() + _, _ = io.Copy(c, up) +} + +func newChainedTestProxy(t *testing.T, cp *testConnectProxy, hosts []string) *GitHubProxy { + t.Helper() + sel, err := upstreamproxy.New("http://"+cp.ln.Addr().String(), hosts, nil, "test") + if err != nil { + t.Fatal(err) + } + p := newTestProxy() + p.allowPrivateForward = true // test origins are loopback listeners + p.SetUpstreamProxy(sel) + return p +} + +// A CONNECT passthrough for a host named by the selector must be tunnelled +// via the upstream proxy, end to end. +func TestHandleConnectDirectChainsThroughUpstreamProxy(t *testing.T) { + origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "via-proxy") + })) + defer origin.Close() + originHostPort := strings.TrimPrefix(origin.URL, "https://") + originHost, _, _ := net.SplitHostPort(originHostPort) + + cp := startTestConnectProxy(t) + p := newChainedTestProxy(t, cp, []string{originHost}) + + clientConn, proxyClient := net.Pipe() + defer clientConn.Close() + req, _ := http.NewRequest(http.MethodConnect, "https://"+originHostPort, nil) + req.Host = originHostPort + go func() { + p.handleConnectDirect(proxyClient, req) + proxyClient.Close() + }() + + br := bufio.NewReader(clientConn) + resp, err := http.ReadResponse(br, req) + if err != nil { + t.Fatalf("read CONNECT response: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("CONNECT status = %d, want 200", resp.StatusCode) + } + + pool := x509.NewCertPool() + pool.AddCert(origin.Certificate()) + tlsConn := tls.Client(clientConn, &tls.Config{RootCAs: pool, ServerName: originHost}) + if err := tlsConn.Handshake(); err != nil { + t.Fatalf("tls handshake over tunnel: %v", err) + } + if _, err := io.WriteString(tlsConn, "GET / HTTP/1.1\r\nHost: "+originHost+"\r\nConnection: close\r\n\r\n"); err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(tlsConn) + if !strings.Contains(string(body), "via-proxy") { + t.Fatalf("origin body not received through tunnel: %q", body) + } + if cp.targets.Load() != 1 { + t.Fatalf("upstream proxy saw %d CONNECTs, want 1", cp.targets.Load()) + } + if got := cp.last.Load(); got != originHostPort { + t.Fatalf("upstream proxy CONNECT target = %v, want %s", got, originHostPort) + } +} + +// Hosts outside the selector's rules keep the direct path: the upstream proxy +// must never see them. +func TestDialUpstreamRawNonMatchingHostStaysDirect(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + c, err := ln.Accept() + if err != nil { + return + } + _, _ = c.Write([]byte("direct")) + _ = c.Close() + }() + + cp := startTestConnectProxy(t) + p := newChainedTestProxy(t, cp, []string{"ghe.example.com"}) + + conn, err := p.dialUpstreamRaw(ln.Addr().String(), 2*time.Second) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + b, _ := io.ReadAll(conn) + if string(b) != "direct" { + t.Fatalf("got %q, want direct", b) + } + if cp.targets.Load() != 0 { + t.Fatalf("upstream proxy saw %d CONNECTs for a non-matching host", cp.targets.Load()) + } +} + +// The MITM upstream leg (dialUpstreamTLS) must complete a TLS handshake with +// the origin through the CONNECT tunnel when the host matches. +func TestDialUpstreamTLSChainsThroughUpstreamProxy(t *testing.T) { + origin := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + origin.StartTLS() + defer origin.Close() + originHostPort := strings.TrimPrefix(origin.URL, "https://") + originHost, _, _ := net.SplitHostPort(originHostPort) + + cp := startTestConnectProxy(t) + p := newChainedTestProxy(t, cp, []string{originHost}) + + conn, err := p.dialUpstreamTLS(originHostPort, originHost, 3*time.Second) + if err == nil { + // The handshake itself fails on cert verification against an unknown + // CA; reaching the handshake proves the tunnel carried TLS bytes. + conn.Close() + } + if cp.targets.Load() != 1 { + t.Fatalf("upstream proxy saw %d CONNECTs, want 1 (err=%v)", cp.targets.Load(), err) + } + if err != nil && !strings.Contains(err.Error(), "certificate") && !strings.Contains(err.Error(), "x509") { + t.Fatalf("unexpected non-TLS error through tunnel: %v", err) + } +} + +// A chained raw tunnel to a matching host is spliced through the upstream +// proxy, and a literal non-routable destination is refused before the proxy +// is ever asked (the guard cannot classify names the proxy resolves, so IP +// literals are the one thing it can still check). +func TestDialUpstreamRawChainsAndGuardsPrivateLiterals(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + c, err := ln.Accept() + if err != nil { + return + } + _, _ = c.Write([]byte("chained")) + _ = c.Close() + }() + originHostPort := ln.Addr().String() + originHost, _, _ := net.SplitHostPort(originHostPort) + + cp := startTestConnectProxy(t) + p := newChainedTestProxy(t, cp, []string{originHost, "10.0.0.0/8"}) + + conn, err := p.dialUpstreamRaw(originHostPort, 2*time.Second) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(conn) + conn.Close() + if string(b) != "chained" || cp.targets.Load() != 1 { + t.Fatalf("got %q via %d CONNECTs, want chained via 1", b, cp.targets.Load()) + } + + p.allowPrivateForward = false + _, err = p.dialUpstreamRaw("10.1.2.3:443", 2*time.Second) + var denied *plainForwardDenied + if !errors.As(err, &denied) { + t.Fatalf("private literal through proxy: err = %v, want plainForwardDenied", err) + } + if cp.targets.Load() != 1 { + t.Fatal("refused destination must not reach the upstream proxy") + } +} + +// When the upstream proxy refuses the CONNECT, dialUpstreamTLS surfaces the +// refusal instead of attempting a handshake on a dead tunnel. +func TestDialUpstreamTLSSurfacesProxyRefusal(t *testing.T) { + cp := startTestConnectProxy(t) + p := newChainedTestProxy(t, cp, []string{"ghe.corp.invalid"}) + // The test proxy dials the target itself; an unresolvable name yields 502. + _, err := p.dialUpstreamTLS("ghe.corp.invalid:443", "ghe.corp.invalid", 3*time.Second) + if err == nil || !strings.Contains(err.Error(), "refused") { + t.Fatalf("err = %v, want CONNECT refusal", err) + } +} + +func TestRedactedProxyURLDropsUserinfo(t *testing.T) { + sel, err := upstreamproxy.New("http://user:secret@proxy.corp.example:3128", nil, nil, "test") + if err != nil { + t.Fatal(err) + } + if got := redactedProxyURL(sel); got != "http://proxy.corp.example:3128" { + t.Fatalf("redactedProxyURL = %q", got) + } + if got := redactedProxyURL(nil); got != "" { + t.Fatalf("redactedProxyURL(nil) = %q", got) + } +} diff --git a/src/crux/proxy/upstream_timeout_test.go b/src/crux/proxy/upstream_timeout_test.go index 46e6b94c95..99b30956fe 100644 --- a/src/crux/proxy/upstream_timeout_test.go +++ b/src/crux/proxy/upstream_timeout_test.go @@ -37,6 +37,11 @@ func readGitHubProxySource(t *testing.T) string { if err != nil { t.Fatalf("glob github_proxy*.go: %v", err) } + chain, err := filepath.Glob(filepath.Join(filepath.Dir(file), "upstream_chain.go")) + if err != nil { + t.Fatalf("glob upstream_chain.go: %v", err) + } + paths = append(paths, chain...) var sb strings.Builder for _, path := range paths { if strings.HasSuffix(path, "_test.go") { @@ -63,9 +68,18 @@ func TestAllTLSUpstreamDialsAreBound(t *testing.T) { if strings.Contains(text, "tls.DialWithDialer(markDialer(0)") { t.Fatal("TLS upstream dials must not use an unbounded zero-timeout dialer") } + // Every TLS upstream dial funnels through dialUpstreamTLS, whose direct + // leg is the single bounded tls.DialWithDialer(markDialer(timeout)) and + // whose chained leg is bounded by a context carrying the same timeout. + if got := strings.Count(text, "tls.DialWithDialer("); got != 1 { + t.Fatalf("tls.DialWithDialer sites = %d, want 1 (dialUpstreamTLS)", got) + } + if !strings.Contains(text, "tls.DialWithDialer(markDialer(timeout)") { + t.Fatal("dialUpstreamTLS direct leg must use the bounded markDialer(timeout)") + } const wantBoundTLSUpstreamDials = 3 - if got := strings.Count(text, "tls.DialWithDialer(markDialer(upstreamDialTimeout)"); got != wantBoundTLSUpstreamDials { - t.Fatalf("bounded TLS upstream dials = %d, want %d", got, wantBoundTLSUpstreamDials) + if got := strings.Count(text, "upstreamDialTimeout)"); got < wantBoundTLSUpstreamDials { + t.Fatalf("dialUpstreamTLS callers passing upstreamDialTimeout = %d, want >= %d", got, wantBoundTLSUpstreamDials) } for _, want := range []string{ `p.logger.Error("transparent proxy upstream dial failed"`, diff --git a/src/crux/proxy/write_leak_gate_test.go b/src/crux/proxy/write_leak_gate_test.go index 677955ec76..b535d3d677 100644 --- a/src/crux/proxy/write_leak_gate_test.go +++ b/src/crux/proxy/write_leak_gate_test.go @@ -83,6 +83,9 @@ func TestSkipGraphQLString(t *testing.T) { {"block string with quotes inside", `"""a"b""" rest`, 0, 9, true}, {"block string with newline", "\"\"\"a\nb\"\"\"", 0, 9, true}, {"unterminated block string", `"""abc`, 0, 0, false}, + {"block string escaped triple quote", `"""a\"""b""" rest`, 0, 12, true}, + {"block string only escaped triple quote", `"""\""""""`, 0, 10, true}, + {"block string escaped terminator unterminated", `"""a\"""`, 0, 0, false}, {"offset start", `x "hi" y`, 2, 6, true}, } for _, tt := range tests { diff --git a/src/crux/proxy/write_surface_enforce.go b/src/crux/proxy/write_surface_enforce.go index 1d4dd4ff85..24466d6466 100644 --- a/src/crux/proxy/write_surface_enforce.go +++ b/src/crux/proxy/write_surface_enforce.go @@ -78,7 +78,7 @@ func WriteSurfaceGraphQLRefusal(enforced func(agent string) bool, agent string, return "", false } var req graphQLRequest - if err := json.Unmarshal(body, &req); err == nil && !graphQLMutationRe.MatchString(strings.TrimSpace(req.Query)) { + if err := json.Unmarshal(body, &req); err == nil && !isGitHubGraphQLMutation(req.Query) { return "", false } if !enforced(agent) { diff --git a/src/crux/proxy/write_surface_enforce_test.go b/src/crux/proxy/write_surface_enforce_test.go index a977478f68..cd82e2cbb0 100644 --- a/src/crux/proxy/write_surface_enforce_test.go +++ b/src/crux/proxy/write_surface_enforce_test.go @@ -89,6 +89,10 @@ func TestWriteSurfaceGraphQLRefusal(t *testing.T) { {"mutation", "scanner", `{"query":"mutation { addComment(input:{}) { clientMutationId } }"}`, true}, {"query", "scanner", `{"query":"query { viewer { login } }"}`, false}, {"shorthand query", "scanner", `{"query":"{ viewer { login } }"}`, false}, + {"fragment-first mutation", "scanner", `{"query":"fragment F on Repository { id } mutation { addComment(input:{}) { clientMutationId } }"}`, true}, + {"leading comma mutation", "scanner", `{"query":",mutation { addComment(input:{}) { clientMutationId } }"}`, true}, + {"leading BOM mutation", "scanner", "{\"query\":\"\uFEFFmutation { addComment(input:{}) { clientMutationId } }\"}", true}, + {"fragment-first query", "scanner", `{"query":"fragment F on Repository { id } query { viewer { login } }"}`, false}, {"not json fails closed", "scanner", `not json`, true}, {"unlisted lane mutation", "reviewer", `{"query":"mutation { x }"}`, false}, } diff --git a/src/crux/runs/provision.go b/src/crux/runs/provision.go index 9c8780a8a7..a9d760698a 100644 --- a/src/crux/runs/provision.go +++ b/src/crux/runs/provision.go @@ -10,6 +10,7 @@ import ( "github.com/hivecommons/hive/crux/api" "github.com/hivecommons/hive/crux/sandbox" + "github.com/hivecommons/hive/crux/upstreamproxy" ) // gitEnvAllowlist is what the clone may see of the service's environment. @@ -73,6 +74,10 @@ func execRunner(ctx context.Context, dir, name string, args ...string) ([]byte, // Never prompt and never carry a credential: the ad-hoc form is for // public repositories on the allowlist, so git gets the sandbox's // credential-free environment plus a no-prompt switch. - cmd.Env = append(sandbox.SanitizedEnv(os.Environ(), gitEnvAllowlist, nil), "GIT_TERMINAL_PROMPT=0") + env := append(sandbox.SanitizedEnv(os.Environ(), gitEnvAllowlist, nil), "GIT_TERMINAL_PROMPT=0") + // The sanitized environment drops HTTPS_PROXY; re-express the upstream + // forward-proxy rules as scoped git config so clones of hosts behind a + // corporate proxy still work. + cmd.Env = append(env, upstreamproxy.Default().GitEnv()...) return cmd.CombinedOutput() } diff --git a/src/crux/sandbox/openshell.go b/src/crux/sandbox/openshell.go new file mode 100644 index 0000000000..5da7dfe449 --- /dev/null +++ b/src/crux/sandbox/openshell.go @@ -0,0 +1,572 @@ +package sandbox + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "time" + + openshell "github.com/ai-openshell/sdk-go/openshell/v1" +) + +// OpenShell runtime (ADR-0026 phase 1, hivecommons/hive#14092). +// +// OpenShellLauncher implements Launcher on top of an OpenShell gateway: it +// creates a kernel-confined sandbox from the kick spec, runs the kick command +// in it, collects stdout/stderr/exit, and deletes the sandbox. The gateway, +// not hive, owns the pod, the Landlock/seccomp confinement and the egress +// proxy; hive's part is to hand it a spec and a network policy and to keep +// the same cleanup and timeout semantics the Job launcher has. +// +// Workspace transport is a shared PVC mounted through the gateway's +// Kubernetes driver config, exactly like kubejob: the SDK's file transport +// is single-file and optional on the gateway, so copy in/out was not viable. + +const ( + // OpenShellDefaultWorkspace is the OpenShell workspace (their term; not + // a Kubernetes namespace) sandboxes land in when none is configured. + OpenShellDefaultWorkspace = "default" + // OpenShellDefaultWorkspaceClaimMount is where hive sees the workspace + // claim; kubejob uses the same default. + OpenShellDefaultWorkspaceClaimMount = "/data" + // OpenShellWorkspaceMount is where the sandbox sees the per-kick + // workspace, matching the Podman and Job launchers. + OpenShellWorkspaceMount = "/workspace" + // OpenShellPolicyVersion is the policy schema version the SDK expects. + OpenShellPolicyVersion = 1 + // OpenShellExitCodeUnknown is reported when the sandbox ended without a + // readable exit code (gateway lost, kick timeout). Same value as + // kubejob.ExitCodeUnknown so callers treat both the same way. + OpenShellExitCodeUnknown = -1 + + // OpenShellPolicyProfileAnnotation carries sandbox.openshell.policy_profile + // on the created sandbox. + OpenShellPolicyProfileAnnotation = "hive.dev/policy-profile" + + openshellNamePrefix = "hive-kick-" + openshellNameMax = 63 + openshellRequestWait = 30 * time.Second + openshellReadyPoll = 2 * time.Second + openshellMaxOutput = 16 << 20 + openshellManagedLabel = "app.kubernetes.io/managed-by" + openshellManagedValue = "hive-openshell" + openshellHTTPSPort = 443 +) + +// OpenShellAlwaysAllowedHosts are the egress hosts every hive sandbox gets: +// the same GitHub hosts hive's MITM proxy (ADR-0002) brokers for a Podman +// sandbox. github.com is what `git fetch` talks to; api.github.com is +// restricted to read-only methods because the sandbox holds no token and +// hive does every write itself after the kick. +var OpenShellAlwaysAllowedHosts = []string{"github.com", "api.github.com"} + +// OpenShellDefaultBinaries are the in-image executables the egress policy +// is attached to when the operator names none. OpenShell matches network +// rules per real binary path, and a rule with no binaries allows nothing; +// these cover git, curl, node-based CLIs and the Python-based ones in the +// sandbox images hive ships. +var OpenShellDefaultBinaries = []string{ + "/usr/bin/git", + "/usr/bin/git-remote-https", + "/usr/lib/git-core/git-remote-https", + "/usr/bin/curl", + "/usr/bin/node", + "/usr/local/bin/node", + "/usr/bin/python3", + "/usr/local/bin/python3", + "/usr/local/bin/crux", +} + +// OpenShellOptions is the launcher's configuration; crux/agent builds it +// from config.SandboxOpenShellConfig. +type OpenShellOptions struct { + GatewayURL string + Workspace string + PolicyProfile string + Token string + InsecureTLS bool + Timeout time.Duration + WorkspaceClaim string + WorkspaceClaimMount string + AllowHosts []string + Binaries []string + RuntimeClassName string + NodeSelector map[string]string + Tolerations []OpenShellToleration +} + +// OpenShellToleration mirrors the subset of a Kubernetes toleration an +// operator writes by hand. +type OpenShellToleration struct { + Key, Operator, Value, Effect string +} + +// Validate reports the first configuration error a kick would hit. +func (o OpenShellOptions) Validate() error { + if strings.TrimSpace(o.GatewayURL) == "" { + return errors.New("openshell: gateway_url is required") + } + if _, _, err := openshellAddress(o.GatewayURL); err != nil { + return err + } + if strings.TrimSpace(o.WorkspaceClaim) == "" { + return errors.New("openshell: workspace_claim is required — the PVC hive's sandbox workspace root lives on, labelled openshell.ai/sandbox-attachable so the gateway may mount it") + } + for _, h := range o.AllowHosts { + if _, _, err := openshellHostPort(h); err != nil { + return err + } + } + return nil +} + +func (o OpenShellOptions) workspace() string { + if ws := strings.TrimSpace(o.Workspace); ws != "" { + return ws + } + return OpenShellDefaultWorkspace +} + +// openshellSandboxes and openshellExec are the slices of the SDK the +// launcher uses. They exist so tests can drive the launcher with a fake and +// so an SDK method rename shows up here, not in every call site. +type openshellSandboxes interface { + Create(ctx context.Context, workspace, name string, spec *openshell.SandboxSpec, opts ...openshell.CreateOption) (*openshell.Sandbox, error) + WaitReady(ctx context.Context, workspace, name string, opts ...openshell.WaitOptions) (*openshell.Sandbox, error) + Delete(ctx context.Context, workspace, name string, opts ...openshell.DeleteOptions) (*openshell.DeletionResult, error) +} + +type openshellExec interface { + Run(ctx context.Context, workspace, sandboxName string, command []string, opts ...openshell.ExecOptions) (*openshell.ExecResult, error) +} + +type openshellHealth interface { + Check(ctx context.Context) (*openshell.HealthResult, error) +} + +type openshellConfig interface { + GetGateway(ctx context.Context) (*openshell.GatewayConfig, error) +} + +type openshellClient interface { + Sandboxes() openshellSandboxes + Exec() openshellExec + Health() openshellHealth + Config() openshellConfig + Close() error +} + +// sdkClient adapts *openshell.Client to the narrow interfaces above. +type sdkClient struct{ c *openshell.Client } + +func (s sdkClient) Sandboxes() openshellSandboxes { return s.c.Sandboxes() } +func (s sdkClient) Exec() openshellExec { return s.c.Exec() } +func (s sdkClient) Health() openshellHealth { return s.c.Health() } +func (s sdkClient) Config() openshellConfig { return s.c.Config() } +func (s sdkClient) Close() error { return s.c.Close() } + +// OpenShellLauncher runs sandboxed kicks through an OpenShell gateway. +type OpenShellLauncher struct { + Options OpenShellOptions + + // newClient overrides the SDK client constructor in tests. + newClient func(OpenShellOptions) (openshellClient, error) + // now overrides the clock in tests. + now func() time.Time +} + +func (l *OpenShellLauncher) client() (openshellClient, error) { + if l.newClient != nil { + return l.newClient(l.Options) + } + return newOpenShellSDKClient(l.Options) +} + +func newOpenShellSDKClient(o OpenShellOptions) (openshellClient, error) { + addr, tls, err := openshellAddress(o.GatewayURL) + if err != nil { + return nil, err + } + cfg := openshell.Config{Address: addr, Timeout: openshellRequestWait} + if tls || o.InsecureTLS { + cfg.TLS = &openshell.TLSConfig{Insecure: o.InsecureTLS} + } + if o.Token != "" { + cfg.Auth = openshell.StaticToken(o.Token) + } + c, err := openshell.NewClient(cfg) + if err != nil { + return nil, fmt.Errorf("openshell: connecting to gateway %s: %w", addr, err) + } + return sdkClient{c: c}, nil +} + +// Run implements Launcher. +func (l *OpenShellLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, error) { + res := Result{ExitCode: OpenShellExitCodeUnknown} + if spec.ClineProvider != nil { + _ = spec.ClineProvider.Close() + return res, errors.New("openshell: scoped Cline provider files are Podman-only; the OpenShell runtime takes no credential files") + } + if err := l.Options.Validate(); err != nil { + return res, err + } + if strings.TrimSpace(spec.Image) == "" { + return res, errors.New("openshell: image is required") + } + if strings.TrimSpace(spec.Workspace) == "" { + return res, errors.New("openshell: workspace is required") + } + if len(spec.Command) == 0 { + return res, errors.New("openshell: command is required") + } + subPath, err := openshellWorkspaceSubPath(spec.Workspace, l.Options.WorkspaceClaimMount) + if err != nil { + return res, err + } + if l.Options.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, l.Options.Timeout) + defer cancel() + } + + client, err := l.client() + if err != nil { + return res, err + } + defer func() { _ = client.Close() }() + + ws := l.Options.workspace() + now := time.Now + if l.now != nil { + now = l.now + } + name := openshellSandboxName(spec.Name, now()) + sbSpec, err := l.sandboxSpec(spec, subPath) + if err != nil { + return res, err + } + createOpts := []openshell.CreateOption{openshell.WithLabels(map[string]string{openshellManagedLabel: openshellManagedValue})} + if p := strings.TrimSpace(l.Options.PolicyProfile); p != "" { + // The SDK (v0.1.8) has no API to select a gateway-side policy profile + // at create time; the name is recorded so gateway admission or an + // operator can act on it, and so the intent is visible on the sandbox. + createOpts = append(createOpts, openshell.WithAnnotations(map[string]string{OpenShellPolicyProfileAnnotation: p})) + } + if _, err := client.Sandboxes().Create(ctx, ws, name, sbSpec, createOpts...); err != nil { + return res, fmt.Errorf("openshell: creating sandbox %s: %w", name, err) + } + + // The caller's ctx may be dead already (kick timeout) or die between any + // two of these requests; cleanup falls back to a fresh, request-bounded + // context so a finished sandbox is still removed. + var cancels []context.CancelFunc + defer func() { + for _, c := range cancels { + c() + } + }() + live := func() context.Context { + if ctx.Err() == nil { + return ctx + } + fresh, cancel := context.WithTimeout(context.Background(), openshellRequestWait) + cancels = append(cancels, cancel) + return fresh + } + cleanup := func() { + _, _ = client.Sandboxes().Delete(live(), ws, name, openshell.DeleteOptions{AllowMissing: true}) + } + + if _, err := client.Sandboxes().WaitReady(ctx, ws, name, openshell.WaitOptions{PollInterval: openshellReadyPoll}); err != nil { + cleanup() + if ctx.Err() != nil { + return res, ctx.Err() + } + return res, fmt.Errorf("openshell: sandbox %s never became ready: %w", name, err) + } + + execOpts := openshell.ExecOptions{Env: l.execEnv(spec), WorkDir: spec.WorkDir, NoLoginShell: true} + if execOpts.WorkDir == "" { + execOpts.WorkDir = OpenShellWorkspaceMount + } + out, runErr := client.Exec().Run(ctx, ws, name, spec.Command, execOpts) + if out != nil { + res.Stdout = openshellCap(out.Stdout) + res.Stderr = openshellCap(out.Stderr) + res.ExitCode = out.ExitCode + } + cleanup() + if runErr != nil { + if ctx.Err() != nil { + // Kick timeout or cancel: the sandbox has been deleted above so + // the workload does not keep running on the cluster. + return res, ctx.Err() + } + return res, fmt.Errorf("openshell: running kick in sandbox %s: %w", name, runErr) + } + return res, nil +} + +// sandboxSpec maps the hive LaunchSpec onto the SDK's SandboxSpec. +func (l *OpenShellLauncher) sandboxSpec(spec LaunchSpec, subPath string) (*openshell.SandboxSpec, error) { + policy, err := OpenShellNetworkPolicy(l.Options.AllowHosts, l.Options.Binaries) + if err != nil { + return nil, err + } + mount := spec.WorkspaceMount + if mount == "" { + mount = OpenShellWorkspaceMount + } + if spec.NetworkNone { + policy.NetworkPolicies = map[string]openshell.NetworkPolicyRule{} + } + tpl := &openshell.SandboxTemplate{ + Image: spec.Image, + Labels: map[string]string{openshellManagedLabel: openshellManagedValue}, + Environment: l.execEnv(spec), + DriverConfig: l.driverConfig(subPath, mount, spec.ReadOnly), + } + if l.Options.RuntimeClassName != "" { + tpl.RuntimeClassName = l.Options.RuntimeClassName + } + return &openshell.SandboxSpec{Template: tpl, Policy: policy}, nil +} + +// execEnv is the environment the kick runs with: the allowlisted, +// credential-filtered subset of hive's own, plus the spec's explicit values — +// the same rule the Podman and Job launchers apply. +func (l *OpenShellLauncher) execEnv(spec LaunchSpec) map[string]string { + env := map[string]string{} + for _, entry := range SanitizedEnv(os.Environ(), spec.EnvAllowlist, spec.Env) { + if k, v, ok := strings.Cut(entry, "="); ok { + env[k] = v + } + } + return env +} + +// driverConfig is the gateway's Kubernetes driver passthrough: the shared +// workspace claim at a subPath, plus scheduling. The gateway only honours +// it when its `allow_driver_config` setting is on and the claim carries the +// `openshell.ai/sandbox-attachable` labels; that is a documented +// prerequisite rather than something hive can verify from here. +func (l *OpenShellLauncher) driverConfig(subPath, mount string, readOnly bool) map[string]any { + k8s := map[string]any{ + "volumes": []any{map[string]any{ + "name": "workspace", + "persistent_volume_claim": map[string]any{"claim_name": l.Options.WorkspaceClaim, "read_only": false}, + }}, + "containers": map[string]any{"agent": map[string]any{ + "volume_mounts": []any{map[string]any{ + "name": "workspace", + "mount_path": mount, + "sub_path": subPath, + "read_only": readOnly, + }}, + }}, + } + pod := map[string]any{} + if len(l.Options.NodeSelector) > 0 { + pod["node_selector"] = l.Options.NodeSelector + } + if len(l.Options.Tolerations) > 0 { + tols := make([]any, 0, len(l.Options.Tolerations)) + for _, t := range l.Options.Tolerations { + tol := map[string]any{} + for k, v := range map[string]string{"key": t.Key, "operator": t.Operator, "value": t.Value, "effect": t.Effect} { + if v != "" { + tol[k] = v + } + } + tols = append(tols, tol) + } + pod["tolerations"] = tols + } + if l.Options.RuntimeClassName != "" { + pod["runtime_class_name"] = l.Options.RuntimeClassName + } + if len(pod) > 0 { + k8s["pod"] = pod + } + return map[string]any{"kubernetes": k8s} +} + +// OpenShellNetworkPolicy derives the sandbox's egress policy from hive's +// always-allowed GitHub hosts plus the operator's extra hosts. github.com +// gets read-write TLS passthrough (git over HTTPS needs POST for +// fetch-pack even when it only reads); api.github.com is read-only so a +// sandbox that somehow obtained a token still could not mutate; extra hosts +// (model gateways, the hub) are read-write. Every rule is attached to the +// same binaries, because OpenShell allows nothing for an unlisted binary. +func OpenShellNetworkPolicy(allowHosts, binaries []string) (*openshell.SandboxPolicy, error) { + if len(binaries) == 0 { + binaries = OpenShellDefaultBinaries + } + bins := make([]openshell.PolicyNetworkBinary, 0, len(binaries)) + for _, b := range binaries { + if b = strings.TrimSpace(b); b != "" { + bins = append(bins, openshell.PolicyNetworkBinary{Path: b}) + } + } + rules := map[string]openshell.NetworkPolicyRule{ + "hive-git": { + Name: "hive-git", + Endpoints: []openshell.PolicyNetworkEndpoint{{ + Host: "github.com", Port: openshellHTTPSPort, + Access: openshell.NetworkAccessPresetReadWrite, + Enforcement: openshell.NetworkEnforcementModeEnforce, + }}, + Binaries: bins, + }, + "hive-github-api": { + Name: "hive-github-api", + Endpoints: []openshell.PolicyNetworkEndpoint{{ + Host: "api.github.com", Port: openshellHTTPSPort, + Access: openshell.NetworkAccessPresetReadOnly, + Enforcement: openshell.NetworkEnforcementModeEnforce, + }}, + Binaries: bins, + }, + } + seen := map[string]bool{"github.com": true, "api.github.com": true} + var extra []openshell.PolicyNetworkEndpoint + for _, h := range allowHosts { + host, port, err := openshellHostPort(h) + if err != nil { + return nil, err + } + key := host + ":" + strconv.Itoa(int(port)) + if seen[host] || seen[key] { + continue + } + seen[key] = true + extra = append(extra, openshell.PolicyNetworkEndpoint{ + Host: host, Port: port, + Access: openshell.NetworkAccessPresetReadWrite, + Enforcement: openshell.NetworkEnforcementModeEnforce, + }) + } + if len(extra) > 0 { + sort.Slice(extra, func(i, j int) bool { + if extra[i].Host != extra[j].Host { + return extra[i].Host < extra[j].Host + } + return extra[i].Port < extra[j].Port + }) + rules["hive-allow-hosts"] = openshell.NetworkPolicyRule{Name: "hive-allow-hosts", Endpoints: extra, Binaries: bins} + } + return &openshell.SandboxPolicy{ + Version: OpenShellPolicyVersion, + Filesystem: &openshell.FilesystemPolicy{ + IncludeWorkdir: true, + ReadWrite: []string{OpenShellWorkspaceMount, "/tmp"}, + }, + NetworkPolicies: rules, + }, nil +} + +// openshellAddress turns gateway_url into the SDK's host:port address and +// whether TLS is implied. Accepts `host:port`, `http://host:port` and +// `https://host[:port]`. +func openshellAddress(raw string) (addr string, tls bool, err error) { + raw = strings.TrimSpace(raw) + if !strings.Contains(raw, "://") { + if _, _, err := openshellHostPort(raw); err != nil || !strings.Contains(raw, ":") { + return "", false, fmt.Errorf("openshell: gateway_url %q must be host:port or an http(s) URL", raw) + } + return raw, false, nil + } + u, err := url.Parse(raw) + if err != nil || u.Host == "" { + return "", false, fmt.Errorf("openshell: gateway_url %q is not a valid URL", raw) + } + switch u.Scheme { + case "https", "grpcs": + tls = true + case "http", "grpc": + default: + return "", false, fmt.Errorf("openshell: gateway_url %q has unsupported scheme %q", raw, u.Scheme) + } + host := u.Host + if u.Port() == "" { + if tls { + host += ":443" + } else { + host += ":80" + } + } + return host, tls, nil +} + +// openshellHostName is a literal DNS name or IPv4 address: no wildcard, +// userinfo, query, fragment or second colon can ride along in a host entry. +var openshellHostName = regexp.MustCompile(`^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$`) + +// openshellHostPort parses an allow_hosts entry: `host` means 443. +func openshellHostPort(raw string) (string, uint32, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", 0, errors.New("openshell: allow_hosts entry is empty") + } + if strings.Contains(raw, "://") || strings.ContainsAny(raw, "/ ") { + return "", 0, fmt.Errorf("openshell: allow_hosts entry %q must be host or host:port, not a URL", raw) + } + host, portStr, found := strings.Cut(raw, ":") + if !openshellHostName.MatchString(host) { + return "", 0, fmt.Errorf("openshell: allow_hosts entry %q must be a literal host name — no wildcards, userinfo, query or fragment", raw) + } + if !found { + return host, openshellHTTPSPort, nil + } + port, err := strconv.ParseUint(portStr, 10, 16) + if err != nil || port == 0 || host == "" { + return "", 0, fmt.Errorf("openshell: allow_hosts entry %q has an invalid port", raw) + } + return host, uint32(port), nil +} + +// openshellWorkspaceSubPath turns hive's absolute workspace path into the +// path relative to the claim mount, which is what the volume mount's +// sub_path needs. Same rule as kubejob. +func openshellWorkspaceSubPath(workspace, claimMount string) (string, error) { + if claimMount == "" { + claimMount = OpenShellDefaultWorkspaceClaimMount + } + rel, err := filepath.Rel(filepath.Clean(claimMount), filepath.Clean(workspace)) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") { + return "", fmt.Errorf("openshell: workspace %q is not under the workspace claim mount %q — set agent_sandbox.workspace_dir under it, or openshell.workspace_claim_mount to where the claim is mounted", workspace, claimMount) + } + return filepath.ToSlash(rel), nil +} + +var openshellNameUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) + +// openshellSandboxName builds a unique, DNS-label-safe sandbox name from the +// launch name and a timestamp: hive-kick--. +func openshellSandboxName(launchName string, now time.Time) string { + base := strings.Trim(openshellNameUnsafe.ReplaceAllString(strings.ToLower(strings.TrimSuffix(launchName, "-sandbox")), "-"), "-") + if base == "" { + base = "agent" + } + suffix := fmt.Sprintf("-%d", now.Unix()) + room := openshellNameMax - len(openshellNamePrefix) - len(suffix) + if len(base) > room { + base = strings.TrimRight(base[:room], "-") + } + return openshellNamePrefix + base + suffix +} + +func openshellCap(b []byte) string { + if len(b) > openshellMaxOutput { + b = b[:openshellMaxOutput] + } + return string(b) +} diff --git a/src/crux/sandbox/openshell_integration_test.go b/src/crux/sandbox/openshell_integration_test.go new file mode 100644 index 0000000000..d87b063e5b --- /dev/null +++ b/src/crux/sandbox/openshell_integration_test.go @@ -0,0 +1,66 @@ +package sandbox + +import ( + "context" + "os" + "strings" + "testing" + "time" +) + +// TestOpenShellIntegration runs one real kick through an OpenShell gateway. +// It is skipped unless OPENSHELL_GATEWAY_URL is set, so the default test run +// needs no gateway; set OPENSHELL_WORKSPACE_CLAIM to the attachable PVC and +// OPENSHELL_IMAGE to a digest-pinned image the gateway can pull. The +// workspace path must live under OPENSHELL_WORKSPACE_CLAIM_MOUNT (default +// /data) on the machine running the test, i.e. this is meant to run from +// inside a hive pod or a debug pod with the claim mounted. +func TestOpenShellIntegration(t *testing.T) { + gateway := os.Getenv("OPENSHELL_GATEWAY_URL") + if gateway == "" { + t.Skip("OPENSHELL_GATEWAY_URL not set; skipping live gateway test") + } + claim := os.Getenv("OPENSHELL_WORKSPACE_CLAIM") + image := os.Getenv("OPENSHELL_IMAGE") + if claim == "" || image == "" { + t.Skip("OPENSHELL_WORKSPACE_CLAIM and OPENSHELL_IMAGE must be set with OPENSHELL_GATEWAY_URL") + } + mount := os.Getenv("OPENSHELL_WORKSPACE_CLAIM_MOUNT") + if mount == "" { + mount = OpenShellDefaultWorkspaceClaimMount + } + workspace, err := os.MkdirTemp(mount, "openshell-it-") + if err != nil { + t.Skipf("cannot create a workspace under %s: %v", mount, err) + } + defer os.RemoveAll(workspace) + + l := &OpenShellLauncher{Options: OpenShellOptions{ + GatewayURL: gateway, + Workspace: os.Getenv("OPENSHELL_WORKSPACE"), + Token: os.Getenv("OPENSHELL_TOKEN"), + InsecureTLS: os.Getenv("OPENSHELL_INSECURE_TLS") == "1", + WorkspaceClaim: claim, + WorkspaceClaimMount: mount, + Timeout: 5 * time.Minute, + }} + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute) + defer cancel() + start := time.Now() + res, err := l.Run(ctx, LaunchSpec{ + Image: image, + Name: "integration", + Workspace: workspace, + Command: []string{"sh", "-c", "echo hello > /workspace/out.txt && cat /workspace/out.txt"}, + }) + t.Logf("sandbox round trip: %s", time.Since(start)) + if err != nil { + t.Fatalf("Run: %v (stdout=%q stderr=%q)", err, res.Stdout, res.Stderr) + } + if res.ExitCode != 0 || !strings.Contains(res.Stdout, "hello") { + t.Fatalf("result = %+v", res) + } + if b, err := os.ReadFile(workspace + "/out.txt"); err != nil || !strings.Contains(string(b), "hello") { + t.Fatalf("workspace write did not round-trip through the shared claim: %v %q", err, b) + } +} diff --git a/src/crux/sandbox/openshell_logs.go b/src/crux/sandbox/openshell_logs.go new file mode 100644 index 0000000000..4913ebb4ed --- /dev/null +++ b/src/crux/sandbox/openshell_logs.go @@ -0,0 +1,274 @@ +package sandbox + +import ( + "context" + "errors" + "fmt" + "io" + "strconv" + "strings" + "time" + + openshell "github.com/ai-openshell/sdk-go/openshell/v1" +) + +// OpenShell sandbox log reader (hivecommons/hive#17645). +// +// An openshell-runtime agent has no tmux session in the hive pod, exactly +// like a job-runtime one, so the dashboard's read-only kick viewer needs a +// way to find the agent's live sandbox and follow its log. This is that +// seam: plain hive types in, plain text out, so pkg/dashboard never imports +// the OpenShell SDK and its tests drive a fake instead of a gateway. +// +// What the log is, and is not. GetLogs returns the gateway's in-memory +// buffer of supervisor, policy (OCSF) and tracing events for the sandbox +// (≤ 2,000 lines per sandbox, see openshellLogLinesPerRead). The kick +// itself runs as an exec session (OpenShellLauncher.Run → Exec().Run), and +// exec output reaches neither this buffer nor the container log — see +// "Accessing Logs" in the OpenShell docs. So this reader shows what the +// sandbox did around the kick (network and filesystem decisions, supervisor +// state), not the kick's own stdout, which the launcher collects from the +// exec result and the executor persists to the transcript when the kick +// ends. The dashboard says so in its status lines. + +// openshellLogLinesPerRead caps each GetLogs read. The gateway keeps at +// most 2,000 lines per sandbox, so asking for that many returns the whole +// buffer instead of the server's smaller default and keeps a chatty +// sandbox from losing lines between two polls. +const openshellLogLinesPerRead uint32 = 2000 + +// OpenShellPhaseDeleting is the Phase OpenShellSandboxInfo carries once the +// sandbox is being deleted or is already gone (NotFound). In the normal +// lifecycle that is how a kick ends: the launcher deletes the sandbox the +// moment Exec().Run returns. +const OpenShellPhaseDeleting = string(openshell.SandboxDeleting) + +// OpenShellSandboxInfo is what the viewer needs to know about a sandbox. +type OpenShellSandboxInfo struct { + Name string + Phase string + // Exited is true once the gateway reports an exit code for the + // sandbox's main process (Sandbox.Status.ExitCode). That is the + // sandbox workload, not the kick: the kick runs as an exec session and + // its exit code is only in the launcher's Result and the persisted + // transcript. In the normal lifecycle the launcher deletes the sandbox + // as soon as the kick ends, so a viewer usually sees NotFound rather + // than an exit code here. + Exited bool + ExitCode int +} + +// Finished reports whether the sandbox will produce no more output. +func (i OpenShellSandboxInfo) Finished() bool { + if i.Exited { + return true + } + switch openshell.SandboxPhase(i.Phase) { + case openshell.SandboxCompleted, openshell.SandboxError, openshell.SandboxStopped, openshell.SandboxDeleting: + return true + } + return false +} + +// OpenShellLogReader is the dashboard's view of an OpenShell gateway. +// *OpenShellLauncher implements it. +type OpenShellLogReader interface { + // LatestSandbox finds the newest live sandbox the launcher created for + // launchName (see LaunchName). found is false when none is running — + // the launcher deletes a sandbox as soon as its kick finishes, so that + // is the normal state between kicks. + LatestSandbox(ctx context.Context, launchName string) (info OpenShellSandboxInfo, found bool, err error) + // StreamLogs writes the sandbox's gateway log (supervisor, policy and + // tracing events — not the kick's exec output, see the package note + // above) to w one line at a time, polling the gateway every poll until + // the sandbox finishes, disappears or ctx ends, and returns the last + // state it observed. + StreamLogs(ctx context.Context, sandboxName string, poll time.Duration, w io.Writer) (OpenShellSandboxInfo, error) +} + +var _ OpenShellLogReader = (*OpenShellLauncher)(nil) + +// openshellLogSandboxes is the read slice of the SDK's SandboxInterface the +// log reader needs on top of openshellSandboxes. +type openshellLogSandboxes interface { + Get(ctx context.Context, workspace, name string) (*openshell.Sandbox, error) + ListAll(ctx context.Context, workspace string, opts ...openshell.ListOptions) ([]*openshell.Sandbox, error) + GetLogs(ctx context.Context, workspace, sandboxName string, opts ...openshell.LogOption) (*openshell.LogResult, error) +} + +func (l *OpenShellLauncher) logClient() (openshellClient, openshellLogSandboxes, error) { + if strings.TrimSpace(l.Options.GatewayURL) == "" { + return nil, nil, errors.New("openshell: gateway_url is not configured") + } + client, err := l.client() + if err != nil { + return nil, nil, err + } + sb, ok := client.Sandboxes().(openshellLogSandboxes) + if !ok { + _ = client.Close() + return nil, nil, errors.New("openshell: client cannot read sandbox logs") + } + return client, sb, nil +} + +// LatestSandbox implements OpenShellLogReader. +func (l *OpenShellLauncher) LatestSandbox(ctx context.Context, launchName string) (OpenShellSandboxInfo, bool, error) { + client, sb, err := l.logClient() + if err != nil { + return OpenShellSandboxInfo{}, false, err + } + defer func() { _ = client.Close() }() + all, err := sb.ListAll(ctx, l.Options.workspace(), openshell.ListOptions{LabelSelector: openshellManagedLabel + "=" + openshellManagedValue}) + if err != nil { + return OpenShellSandboxInfo{}, false, fmt.Errorf("openshell: listing sandboxes: %w", err) + } + var ( + best *openshell.Sandbox + bestUnix int64 + ) + for _, s := range all { + if s == nil || s.DeletionTimestamp != nil { + continue + } + unix, ok := openshellSandboxNameUnix(launchName, s.Name) + if !ok { + continue + } + if best == nil || unix > bestUnix { + best, bestUnix = s, unix + } + } + if best == nil { + return OpenShellSandboxInfo{}, false, nil + } + return openshellSandboxInfo(best), true, nil +} + +// openshellSandboxNameUnix reports whether name is one openshellSandboxName +// built for launchName, and the timestamp it carries. Matching by rebuilding +// the name keeps one naming rule: "deps" never matches "deps-bot"'s sandbox. +func openshellSandboxNameUnix(launchName, name string) (int64, bool) { + i := strings.LastIndexByte(name, '-') + if i < 0 || !strings.HasPrefix(name, openshellNamePrefix) { + return 0, false + } + unix, err := strconv.ParseInt(name[i+1:], 10, 64) + if err != nil { + return 0, false + } + if openshellSandboxName(launchName, time.Unix(unix, 0)) != name { + return 0, false + } + return unix, true +} + +func openshellSandboxInfo(s *openshell.Sandbox) OpenShellSandboxInfo { + info := OpenShellSandboxInfo{Name: s.Name, Phase: string(s.Status.Phase)} + if s.DeletionTimestamp != nil && info.Phase == "" { + info.Phase = OpenShellPhaseDeleting + } + if s.Status.ExitCode != nil { + info.Exited = true + info.ExitCode = int(*s.Status.ExitCode) + } + return info +} + +// StreamLogs implements OpenShellLogReader. GetLogs is a snapshot, not a +// stream, so following is a poll with WithLogSince; the gateway's "since" +// is inclusive, so lines at the last seen timestamp are de-duplicated. +func (l *OpenShellLauncher) StreamLogs(ctx context.Context, sandboxName string, poll time.Duration, w io.Writer) (OpenShellSandboxInfo, error) { + info := OpenShellSandboxInfo{Name: sandboxName} + client, sb, err := l.logClient() + if err != nil { + return info, err + } + defer func() { _ = client.Close() }() + if poll <= 0 { + poll = openshellReadyPoll + } + ws := l.Options.workspace() + var ( + since time.Time + atLast = map[string]bool{} + ) + drain := func() error { + opts := []openshell.LogOption{openshell.WithLogLines(openshellLogLinesPerRead)} + if !since.IsZero() { + opts = append(opts, openshell.WithLogSince(since)) + } + res, err := sb.GetLogs(ctx, ws, sandboxName, opts...) + if err != nil { + return err + } + if res == nil { + return nil + } + for _, line := range res.Lines { + if line.Timestamp.Before(since) { + continue + } + key := line.Source + "\x00" + line.Level + "\x00" + line.Target + "\x00" + line.Message + if line.Timestamp.Equal(since) { + if atLast[key] { + continue + } + } else { + since, atLast = line.Timestamp, map[string]bool{} + } + atLast[key] = true + if _, err := io.WriteString(w, formatOpenShellLogLine(line)); err != nil { + return err + } + } + return nil + } + ticker := time.NewTicker(poll) + defer ticker.Stop() + for { + logErr := drain() + s, err := sb.Get(ctx, ws, sandboxName) + switch { + case err != nil && openshell.IsNotFound(err): + // Deleted by the launcher after its kick: nothing more to read. + info.Phase = OpenShellPhaseDeleting + return info, nil + case err != nil: + return info, err + } + info = openshellSandboxInfo(s) + if logErr != nil && !openshell.IsNotFound(logErr) { + return info, logErr + } + if info.Finished() { + // Lines written between the last read and the exit. + if err := drain(); err != nil && !openshell.IsNotFound(err) { + return info, err + } + return info, nil + } + select { + case <-ctx.Done(): + return info, ctx.Err() + case <-ticker.C: + } + } +} + +func formatOpenShellLogLine(line openshell.LogLine) string { + var b strings.Builder + if !line.Timestamp.IsZero() { + b.WriteString(line.Timestamp.UTC().Format(time.RFC3339)) + b.WriteByte(' ') + } + if line.Source != "" { + b.WriteString("[" + line.Source + "] ") + } + if line.Level != "" { + b.WriteString(line.Level + " ") + } + b.WriteString(strings.TrimRight(line.Message, "\n")) + b.WriteByte('\n') + return b.String() +} diff --git a/src/crux/sandbox/openshell_logs_test.go b/src/crux/sandbox/openshell_logs_test.go new file mode 100644 index 0000000000..2da1939117 --- /dev/null +++ b/src/crux/sandbox/openshell_logs_test.go @@ -0,0 +1,392 @@ +package sandbox + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + openshell "github.com/ai-openshell/sdk-go/openshell/v1" + openshelltypes "github.com/ai-openshell/sdk-go/openshell/v1/types" +) + +// fakeOpenShellLogs scripts the read side of the gateway: a sandbox list, +// one GetLogs answer and one Get answer per poll. +type fakeOpenShellLogs struct { + mu sync.Mutex + list []*openshell.Sandbox + listErr error + selectors []string + logs [][]openshell.LogLine + // logErrs[i] is returned by the i-th GetLogs call when set. + logErrs []error + // nilLogs makes every GetLogs call answer (nil, nil). + nilLogs bool + since []time.Time + lines []uint32 + gets []*openshell.Sandbox + getErr error + logCalls int + getCalls int + closed int +} + +func (f *fakeOpenShellLogs) Sandboxes() openshellSandboxes { return f } +func (f *fakeOpenShellLogs) Exec() openshellExec { return nil } +func (f *fakeOpenShellLogs) Health() openshellHealth { return nil } +func (f *fakeOpenShellLogs) Config() openshellConfig { return nil } +func (f *fakeOpenShellLogs) Close() error { f.closed++; return nil } + +func (f *fakeOpenShellLogs) Create(context.Context, string, string, *openshell.SandboxSpec, ...openshell.CreateOption) (*openshell.Sandbox, error) { + return nil, nil +} + +func (f *fakeOpenShellLogs) WaitReady(context.Context, string, string, ...openshell.WaitOptions) (*openshell.Sandbox, error) { + return nil, nil +} + +func (f *fakeOpenShellLogs) Delete(context.Context, string, string, ...openshell.DeleteOptions) (*openshell.DeletionResult, error) { + return nil, nil +} + +func (f *fakeOpenShellLogs) ListAll(_ context.Context, _ string, opts ...openshell.ListOptions) ([]*openshell.Sandbox, error) { + f.mu.Lock() + defer f.mu.Unlock() + for _, o := range opts { + f.selectors = append(f.selectors, o.LabelSelector) + } + if f.listErr != nil { + return nil, f.listErr + } + return f.list, nil +} + +func (f *fakeOpenShellLogs) Get(context.Context, string, string) (*openshell.Sandbox, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.getErr != nil { + return nil, f.getErr + } + i := f.getCalls + f.getCalls++ + if i >= len(f.gets) { + i = len(f.gets) - 1 + } + return f.gets[i], nil +} + +func (f *fakeOpenShellLogs) GetLogs(_ context.Context, _, _ string, opts ...openshell.LogOption) (*openshell.LogResult, error) { + f.mu.Lock() + defer f.mu.Unlock() + cfg := openshelltypes.ApplyLogOptions(opts) + f.since = append(f.since, cfg.Since()) + f.lines = append(f.lines, cfg.Lines()) + i := f.logCalls + f.logCalls++ + if i < len(f.logErrs) && f.logErrs[i] != nil { + return nil, f.logErrs[i] + } + if f.nilLogs { + return nil, nil + } + if i >= len(f.logs) { + return &openshell.LogResult{}, nil + } + return &openshell.LogResult{Lines: f.logs[i]}, nil +} + +func openshellLogLauncher(f *fakeOpenShellLogs) *OpenShellLauncher { + return &OpenShellLauncher{ + Options: OpenShellOptions{GatewayURL: "gw:8080"}, + newClient: func(OpenShellOptions) (openshellClient, error) { return f, nil }, + } +} + +func osSandbox(name string, phase openshell.SandboxPhase) *openshell.Sandbox { + return &openshell.Sandbox{Name: name, Status: openshell.SandboxStatus{Phase: phase}} +} + +func TestOpenShellLatestSandboxPicksNewestForAgent(t *testing.T) { + launch := LaunchName("deps") + older := openshellSandboxName(launch, time.Unix(1700000000, 0)) + newer := openshellSandboxName(launch, time.Unix(1700000100, 0)) + deleting := osSandbox(openshellSandboxName(launch, time.Unix(1700000200, 0)), openshell.SandboxReady) + now := time.Now() + deleting.DeletionTimestamp = &now + f := &fakeOpenShellLogs{list: []*openshell.Sandbox{ + osSandbox(older, openshell.SandboxReady), + osSandbox(newer, openshell.SandboxProvisioning), + deleting, + osSandbox(openshellSandboxName(LaunchName("deps-bot"), time.Unix(1700000300, 0)), openshell.SandboxReady), + osSandbox("unrelated", openshell.SandboxReady), + }} + info, found, err := openshellLogLauncher(f).LatestSandbox(context.Background(), launch) + if err != nil || !found { + t.Fatalf("found=%v err=%v", found, err) + } + if info.Name != newer || info.Phase != string(openshell.SandboxProvisioning) || info.Finished() { + t.Fatalf("info = %+v, want the newest live sandbox %s", info, newer) + } + if len(f.selectors) != 1 || f.selectors[0] != openshellManagedLabel+"="+openshellManagedValue { + t.Fatalf("list must be restricted to hive's sandboxes, got %v", f.selectors) + } + if f.closed != 1 { + t.Fatalf("client closed %d times", f.closed) + } + + _, found, err = openshellLogLauncher(&fakeOpenShellLogs{}).LatestSandbox(context.Background(), launch) + if err != nil || found { + t.Fatalf("no sandbox: found=%v err=%v", found, err) + } +} + +func TestOpenShellLatestSandboxNeedsGateway(t *testing.T) { + l := &OpenShellLauncher{} + if _, _, err := l.LatestSandbox(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "gateway_url") { + t.Fatalf("err = %v", err) + } +} + +func TestOpenShellStreamLogsFollowsUntilExitWithoutDuplicates(t *testing.T) { + t1 := time.Date(2026, 11, 11, 6, 0, 1, 0, time.UTC) + t2 := t1.Add(time.Second) + t3 := t2.Add(time.Second) + code := int32(3) + done := osSandbox("sb", openshell.SandboxCompleted) + done.Status.ExitCode = &code + f := &fakeOpenShellLogs{ + logs: [][]openshell.LogLine{ + {{Timestamp: t1, Source: "sandbox", Level: "INFO", Message: "step 1"}, {Timestamp: t2, Source: "sandbox", Message: "step 2\n"}}, + // "since" is inclusive: the gateway repeats the t2 line. + {{Timestamp: t2, Source: "sandbox", Message: "step 2\n"}, {Timestamp: t3, Message: "step 3"}}, + {{Timestamp: t3, Message: "step 3"}}, + }, + gets: []*openshell.Sandbox{osSandbox("sb", openshell.SandboxReady), done}, + } + var out strings.Builder + info, err := openshellLogLauncher(f).StreamLogs(context.Background(), "sb", time.Millisecond, &out) + if err != nil { + t.Fatal(err) + } + want := t1.Format(time.RFC3339) + " [sandbox] INFO step 1\n" + + t2.Format(time.RFC3339) + " [sandbox] step 2\n" + + t3.Format(time.RFC3339) + " step 3\n" + if out.String() != want { + t.Fatalf("log =\n%s\nwant\n%s", out.String(), want) + } + if !info.Exited || info.ExitCode != 3 || !info.Finished() { + t.Fatalf("info = %+v", info) + } + if len(f.since) < 2 || !f.since[0].IsZero() || !f.since[1].Equal(t2) { + t.Fatalf("follow-up reads must ask only for new lines, got %v", f.since) + } + for i, n := range f.lines { + if n != openshellLogLinesPerRead { + t.Fatalf("read %d asked for %d lines, want the gateway's full %d-line buffer so none are lost between polls", i, n, openshellLogLinesPerRead) + } + } +} + +func TestOpenShellStreamLogsEndsWhenSandboxIsDeleted(t *testing.T) { + f := &fakeOpenShellLogs{ + logs: [][]openshell.LogLine{{{Message: "last words"}}}, + getErr: &openshell.StatusError{Code: openshell.ErrorNotFound, Message: "gone"}, + } + var out strings.Builder + info, err := openshellLogLauncher(f).StreamLogs(context.Background(), "sb", time.Millisecond, &out) + if err != nil { + t.Fatal(err) + } + if out.String() != "last words\n" || !info.Finished() || info.Exited { + t.Fatalf("out=%q info=%+v", out.String(), info) + } +} + +func TestOpenShellStreamLogsStopsWithContext(t *testing.T) { + f := &fakeOpenShellLogs{gets: []*openshell.Sandbox{osSandbox("sb", openshell.SandboxReady)}} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + done := make(chan error, 1) + go func() { + _, err := openshellLogLauncher(f).StreamLogs(ctx, "sb", time.Hour, &strings.Builder{}) + done <- err + }() + select { + case err := <-done: + if err == nil { + t.Fatal("a cancelled follow must report the context error") + } + case <-time.After(10 * time.Second): + t.Fatal("StreamLogs kept polling after the context ended") + } +} + +func TestOpenShellLogClientErrors(t *testing.T) { + boom := errors.New("dial failed") + l := &OpenShellLauncher{ + Options: OpenShellOptions{GatewayURL: "gw:8080"}, + newClient: func(OpenShellOptions) (openshellClient, error) { return nil, boom }, + } + if _, _, err := l.LatestSandbox(context.Background(), "x"); !errors.Is(err, boom) { + t.Fatalf("LatestSandbox err = %v, want %v", err, boom) + } + if _, err := l.StreamLogs(context.Background(), "sb", time.Millisecond, &strings.Builder{}); !errors.Is(err, boom) { + t.Fatalf("StreamLogs err = %v, want %v", err, boom) + } + + // A client whose Sandboxes() cannot read logs is closed and refused. + narrow := &fakeOpenShell{} + l.newClient = func(OpenShellOptions) (openshellClient, error) { return narrow, nil } + _, _, err := l.LatestSandbox(context.Background(), "x") + if err == nil || !strings.Contains(err.Error(), "cannot read sandbox logs") { + t.Fatalf("err = %v", err) + } + if !narrow.closed { + t.Fatal("a refused client must still be closed") + } +} + +func TestOpenShellLatestSandboxListError(t *testing.T) { + boom := errors.New("gateway down") + f := &fakeOpenShellLogs{listErr: boom} + _, found, err := openshellLogLauncher(f).LatestSandbox(context.Background(), "deps") + if found || !errors.Is(err, boom) || !strings.Contains(err.Error(), "listing sandboxes") { + t.Fatalf("found=%v err=%v", found, err) + } + if f.closed != 1 { + t.Fatalf("client closed %d times", f.closed) + } +} + +func TestOpenShellSandboxNameUnix(t *testing.T) { + launch := LaunchName("deps") + good := openshellSandboxName(launch, time.Unix(1700000000, 0)) + cases := []struct { + name string + want bool + }{ + {good, true}, + {"nodash", false}, + {"other-deps-1700000000", false}, + {openshellNamePrefix + "deps-notanumber", false}, + {openshellSandboxName(LaunchName("deps-bot"), time.Unix(1700000000, 0)), false}, + } + for _, c := range cases { + unix, ok := openshellSandboxNameUnix(launch, c.name) + if ok != c.want { + t.Errorf("%q: ok=%v want %v", c.name, ok, c.want) + } + if ok && unix != 1700000000 { + t.Errorf("%q: unix=%d", c.name, unix) + } + } +} + +func TestOpenShellSandboxInfoDeletingWithoutPhase(t *testing.T) { + now := time.Now() + s := &openshell.Sandbox{Name: "sb", DeletionTimestamp: &now} + info := openshellSandboxInfo(s) + if info.Phase != string(openshell.SandboxDeleting) || !info.Finished() || info.Exited { + t.Fatalf("info = %+v", info) + } + // A reported phase wins over the deletion marker. + s.Status.Phase = openshell.SandboxStopped + if got := openshellSandboxInfo(s); got.Phase != string(openshell.SandboxStopped) || !got.Finished() { + t.Fatalf("info = %+v", got) + } + if !(OpenShellSandboxInfo{Phase: string(openshell.SandboxError)}).Finished() { + t.Fatal("error phase must be finished") + } +} + +func TestOpenShellStreamLogsSkipsStaleLinesAndNilResults(t *testing.T) { + t1 := time.Date(2026, 11, 11, 6, 0, 1, 0, time.UTC) + t2 := t1.Add(time.Second) + f := &fakeOpenShellLogs{ + logs: [][]openshell.LogLine{ + {{Timestamp: t2, Message: "now"}}, + // The gateway replays an older line and the t2 line again. + {{Timestamp: t1, Message: "stale"}, {Timestamp: t2, Message: "now"}}, + }, + gets: []*openshell.Sandbox{osSandbox("sb", openshell.SandboxReady), osSandbox("sb", openshell.SandboxCompleted)}, + } + var out strings.Builder + info, err := openshellLogLauncher(f).StreamLogs(context.Background(), "sb", time.Millisecond, &out) + if err != nil { + t.Fatal(err) + } + if want := t2.Format(time.RFC3339) + " now\n"; out.String() != want { + t.Fatalf("log = %q, want %q", out.String(), want) + } + if !info.Finished() || info.Exited { + t.Fatalf("info = %+v", info) + } + + // A nil LogResult is an empty read, and poll <= 0 falls back to the default. + f = &fakeOpenShellLogs{nilLogs: true, gets: []*openshell.Sandbox{osSandbox("sb", openshell.SandboxCompleted)}} + out.Reset() + if _, err := openshellLogLauncher(f).StreamLogs(context.Background(), "sb", 0, &out); err != nil { + t.Fatal(err) + } + if out.Len() != 0 || f.logCalls != 2 { + t.Fatalf("out=%q logCalls=%d", out.String(), f.logCalls) + } +} + +type failingWriter struct{ err error } + +func (w failingWriter) Write([]byte) (int, error) { return 0, w.err } + +func TestOpenShellStreamLogsReportsErrors(t *testing.T) { + boom := errors.New("boom") + notFound := &openshell.StatusError{Code: openshell.ErrorNotFound, Message: "gone"} + + t.Run("get", func(t *testing.T) { + f := &fakeOpenShellLogs{getErr: boom} + if _, err := openshellLogLauncher(f).StreamLogs(context.Background(), "sb", time.Millisecond, &strings.Builder{}); !errors.Is(err, boom) { + t.Fatalf("err = %v", err) + } + }) + t.Run("logs", func(t *testing.T) { + f := &fakeOpenShellLogs{logErrs: []error{boom}, gets: []*openshell.Sandbox{osSandbox("sb", openshell.SandboxReady)}} + info, err := openshellLogLauncher(f).StreamLogs(context.Background(), "sb", time.Millisecond, &strings.Builder{}) + if !errors.Is(err, boom) || info.Phase != string(openshell.SandboxReady) { + t.Fatalf("err=%v info=%+v", err, info) + } + }) + t.Run("logs not found is not fatal while the sandbox lives", func(t *testing.T) { + f := &fakeOpenShellLogs{ + logErrs: []error{notFound, nil, nil}, + logs: [][]openshell.LogLine{nil, {{Message: "late"}}}, + gets: []*openshell.Sandbox{osSandbox("sb", openshell.SandboxReady), osSandbox("sb", openshell.SandboxCompleted)}, + } + var out strings.Builder + if _, err := openshellLogLauncher(f).StreamLogs(context.Background(), "sb", time.Millisecond, &out); err != nil { + t.Fatal(err) + } + if out.String() != "late\n" { + t.Fatalf("out = %q", out.String()) + } + }) + t.Run("final drain", func(t *testing.T) { + f := &fakeOpenShellLogs{logErrs: []error{nil, boom}, gets: []*openshell.Sandbox{osSandbox("sb", openshell.SandboxCompleted)}} + if _, err := openshellLogLauncher(f).StreamLogs(context.Background(), "sb", time.Millisecond, &strings.Builder{}); !errors.Is(err, boom) { + t.Fatalf("err = %v", err) + } + f = &fakeOpenShellLogs{logErrs: []error{nil, notFound}, gets: []*openshell.Sandbox{osSandbox("sb", openshell.SandboxCompleted)}} + if _, err := openshellLogLauncher(f).StreamLogs(context.Background(), "sb", time.Millisecond, &strings.Builder{}); err != nil { + t.Fatalf("a sandbox deleted during the final drain is not an error, got %v", err) + } + }) + t.Run("writer", func(t *testing.T) { + f := &fakeOpenShellLogs{ + logs: [][]openshell.LogLine{{{Message: "x"}}}, + gets: []*openshell.Sandbox{osSandbox("sb", openshell.SandboxReady)}, + } + if _, err := openshellLogLauncher(f).StreamLogs(context.Background(), "sb", time.Millisecond, failingWriter{boom}); !errors.Is(err, boom) { + t.Fatalf("err = %v", err) + } + }) +} diff --git a/src/crux/sandbox/openshell_preflight.go b/src/crux/sandbox/openshell_preflight.go new file mode 100644 index 0000000000..721557f3bf --- /dev/null +++ b/src/crux/sandbox/openshell_preflight.go @@ -0,0 +1,101 @@ +package sandbox + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + openshell "github.com/ai-openshell/sdk-go/openshell/v1" +) + +// OpenShell cluster-prerequisite preflight (ADR-0026 phase 4, +// hivecommons/hive#17644). Hive cannot see the node kernel from its pod, but +// the gateway reports whether it is up and which global settings it runs +// with; the three that decide whether a kick can work or is safe are checked +// here, read-only, and surfaced as boot/reload warnings. + +const ( + openshellPreflightTTL = 5 * time.Minute + + openshellSettingAllowDriverConfig = "allow_driver_config" + openshellSettingTelemetry = "telemetry" +) + +type openshellPreflightEntry struct { + at time.Time + warnings []string +} + +// OpenShellPreflighter checks gateways and remembers the answer for a TTL, so +// the dashboard polling the same warnings does not hit the gateway per call. +type OpenShellPreflighter struct { + mu sync.Mutex + cache map[string]openshellPreflightEntry + + newClient func(OpenShellOptions) (openshellClient, error) + now func() time.Time +} + +// DefaultOpenShellPreflighter backs the warnings AgentSandboxGateWarnings emits. +var DefaultOpenShellPreflighter = &OpenShellPreflighter{} + +// Check returns the warnings for one gateway. Never an error: an unreachable +// gateway is itself a warning. +func (p *OpenShellPreflighter) Check(ctx context.Context, opts OpenShellOptions) []string { + now := time.Now + if p.now != nil { + now = p.now + } + key := opts.GatewayURL + "\x00" + opts.Token + p.mu.Lock() + defer p.mu.Unlock() + if e, ok := p.cache[key]; ok && now().Sub(e.at) < openshellPreflightTTL { + return append([]string(nil), e.warnings...) + } + warnings := p.probe(ctx, opts) + if p.cache == nil { + p.cache = map[string]openshellPreflightEntry{} + } + p.cache[key] = openshellPreflightEntry{at: now(), warnings: warnings} + return append([]string(nil), warnings...) +} + +func (p *OpenShellPreflighter) probe(ctx context.Context, opts OpenShellOptions) []string { + gw := opts.GatewayURL + ctx, cancel := context.WithTimeout(ctx, openshellRequestWait) + defer cancel() + + newClient := newOpenShellSDKClient + if p.newClient != nil { + newClient = p.newClient + } + client, err := newClient(opts) + if err != nil { + return []string{fmt.Sprintf("openshell gateway %s is unreachable (%v); every kick on the openshell runtime will fail", gw, err)} + } + defer func() { _ = client.Close() }() + + health, err := client.Health().Check(ctx) + if err != nil { + return []string{fmt.Sprintf("openshell gateway %s is unreachable (%v); every kick on the openshell runtime will fail", gw, err)} + } + var out []string + if health == nil || !health.Healthy { + out = append(out, fmt.Sprintf("openshell gateway %s reports itself unhealthy; kicks on the openshell runtime may fail", gw)) + } + gwCfg, err := client.Config().GetGateway(ctx) + if err != nil || gwCfg == nil { + return append(out, fmt.Sprintf("openshell gateway %s settings could not be read (%v); allow_driver_config and telemetry are unverified", gw, err)) + } + if v, ok := gwCfg.Settings[openshellSettingAllowDriverConfig]; !ok || !v.BoolVal { + out = append(out, fmt.Sprintf("openshell gateway %s does not have allow_driver_config enabled; it will ignore the workspace volume mount and every kick will run without the workspace — set allow_driver_config = true on the gateway", gw)) + } + for k, v := range gwCfg.Settings { + if strings.Contains(strings.ToLower(k), openshellSettingTelemetry) && v.Type == openshell.SettingValueBool && v.BoolVal { + out = append(out, fmt.Sprintf("openshell gateway %s has telemetry enabled (%s); sandbox activity is reported off-cluster — disable it on the gateway if that is not intended", gw, k)) + } + } + return out +} diff --git a/src/crux/sandbox/openshell_test.go b/src/crux/sandbox/openshell_test.go new file mode 100644 index 0000000000..6f20cfc2d6 --- /dev/null +++ b/src/crux/sandbox/openshell_test.go @@ -0,0 +1,694 @@ +package sandbox + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" + + openshell "github.com/ai-openshell/sdk-go/openshell/v1" +) + +// fakeOpenShell is an in-memory stand-in for the SDK slices the launcher +// uses. It records the calls so tests can assert on the spec that reached +// the gateway and on cleanup, and lets each step fail or block on demand. +type fakeOpenShell struct { + created []fakeCreate + deleted []string + waitErr error + waitBlocks bool + runErr error + runBlocks bool + runResult *openshell.ExecResult + runCmd []string + runOpts openshell.ExecOptions + createErr error + closed bool + deleteCtxOK bool + healthErr error + unhealthy bool + gateway *openshell.GatewayConfig + gatewayErr error + healthCalls int + configCalls int +} + +func (f *fakeOpenShell) Check(context.Context) (*openshell.HealthResult, error) { + f.healthCalls++ + if f.healthErr != nil { + return nil, f.healthErr + } + return &openshell.HealthResult{Healthy: !f.unhealthy}, nil +} + +func (f *fakeOpenShell) GetGateway(context.Context) (*openshell.GatewayConfig, error) { + f.configCalls++ + return f.gateway, f.gatewayErr +} + +type fakeCreate struct { + workspace, name string + spec *openshell.SandboxSpec +} + +func (f *fakeOpenShell) Sandboxes() openshellSandboxes { return f } +func (f *fakeOpenShell) Exec() openshellExec { return f } +func (f *fakeOpenShell) Health() openshellHealth { return f } +func (f *fakeOpenShell) Config() openshellConfig { return f } +func (f *fakeOpenShell) Close() error { f.closed = true; return nil } + +func (f *fakeOpenShell) Create(_ context.Context, ws, name string, spec *openshell.SandboxSpec, _ ...openshell.CreateOption) (*openshell.Sandbox, error) { + if f.createErr != nil { + return nil, f.createErr + } + f.created = append(f.created, fakeCreate{ws, name, spec}) + return &openshell.Sandbox{Name: name}, nil +} + +func (f *fakeOpenShell) WaitReady(ctx context.Context, _, name string, _ ...openshell.WaitOptions) (*openshell.Sandbox, error) { + if f.waitBlocks { + <-ctx.Done() + return nil, ctx.Err() + } + if f.waitErr != nil { + return nil, f.waitErr + } + return &openshell.Sandbox{Name: name}, nil +} + +func (f *fakeOpenShell) Delete(ctx context.Context, _, name string, _ ...openshell.DeleteOptions) (*openshell.DeletionResult, error) { + // Cleanup must arrive on a live context even when the kick's has died. + f.deleteCtxOK = ctx.Err() == nil + f.deleted = append(f.deleted, name) + return &openshell.DeletionResult{}, nil +} + +func (f *fakeOpenShell) Run(ctx context.Context, _, _ string, cmd []string, opts ...openshell.ExecOptions) (*openshell.ExecResult, error) { + f.runCmd = cmd + if len(opts) > 0 { + f.runOpts = opts[0] + } + if f.runBlocks { + <-ctx.Done() + return nil, ctx.Err() + } + if f.runErr != nil { + return nil, f.runErr + } + if f.runResult != nil { + return f.runResult, nil + } + return &openshell.ExecResult{ExitCode: 0, Stdout: []byte("ok")}, nil +} + +func newTestOpenShell(fake *fakeOpenShell) *OpenShellLauncher { + return &OpenShellLauncher{ + Options: OpenShellOptions{ + GatewayURL: "openshell-gateway.openshell.svc:8080", + WorkspaceClaim: "hive-data", + AllowHosts: []string{"litellm.example.com", "hub.example.org:8443"}, + NodeSelector: map[string]string{"kernel": "6.12"}, + Tolerations: []OpenShellToleration{{Key: "sandbox", Operator: "Exists", Effect: "NoSchedule"}}, + }, + newClient: func(OpenShellOptions) (openshellClient, error) { return fake, nil }, + now: func() time.Time { return time.Unix(1700000000, 0) }, + } +} + +func testSpec() LaunchSpec { + return LaunchSpec{ + Image: "registry.example/dev:sha256-abc", + Name: "deps-sandbox", + Workspace: "/data/agents/sandbox/deps", + Command: []string{"crux", "turn", "--prompt-file", ".hive/prompt.md"}, + Env: map[string]string{"HIVE_AGENT": "deps", "GITHUB_TOKEN": "leak"}, + EnvAllowlist: []string{"PATH_NOT_SET_IN_TEST"}, + } +} + +func TestOpenShellRunSuccess(t *testing.T) { + fake := &fakeOpenShell{runResult: &openshell.ExecResult{ExitCode: 3, Stdout: []byte("out"), Stderr: []byte("err")}} + l := newTestOpenShell(fake) + res, err := l.Run(context.Background(), testSpec()) + if err != nil { + t.Fatalf("Run: %v", err) + } + if res.ExitCode != 3 || res.Stdout != "out" || res.Stderr != "err" { + t.Fatalf("result = %+v", res) + } + if len(fake.created) != 1 { + t.Fatalf("created = %d, want 1", len(fake.created)) + } + c := fake.created[0] + if c.workspace != OpenShellDefaultWorkspace { + t.Errorf("workspace = %q", c.workspace) + } + if c.name != "hive-kick-deps-1700000000" { + t.Errorf("name = %q", c.name) + } + if c.spec.Template.Image != "registry.example/dev:sha256-abc" { + t.Errorf("image = %q", c.spec.Template.Image) + } + if _, leaked := c.spec.Template.Environment["GITHUB_TOKEN"]; leaked { + t.Error("credential env reached the sandbox template") + } + if _, leaked := fake.runOpts.Env["GITHUB_TOKEN"]; leaked { + t.Error("credential env reached exec") + } + if fake.runOpts.Env["HIVE_AGENT"] != "deps" { + t.Errorf("exec env = %v, want HIVE_AGENT", fake.runOpts.Env) + } + if fake.runOpts.WorkDir != OpenShellWorkspaceMount { + t.Errorf("workdir = %q", fake.runOpts.WorkDir) + } + if strings.Join(fake.runCmd, " ") != "crux turn --prompt-file .hive/prompt.md" { + t.Errorf("cmd = %v", fake.runCmd) + } + if len(fake.deleted) != 1 || fake.deleted[0] != c.name { + t.Errorf("deleted = %v, want the sandbox removed after success", fake.deleted) + } + if !fake.closed { + t.Error("client not closed") + } +} + +func TestOpenShellRunDriverConfigMountsWorkspaceClaim(t *testing.T) { + fake := &fakeOpenShell{} + l := newTestOpenShell(fake) + if _, err := l.Run(context.Background(), testSpec()); err != nil { + t.Fatal(err) + } + dc := fake.created[0].spec.Template.DriverConfig + k8s, ok := dc["kubernetes"].(map[string]any) + if !ok { + t.Fatalf("driver config = %v", dc) + } + vols := k8s["volumes"].([]any) + pvc := vols[0].(map[string]any)["persistent_volume_claim"].(map[string]any) + if pvc["claim_name"] != "hive-data" { + t.Errorf("claim = %v", pvc) + } + mount := k8s["containers"].(map[string]any)["agent"].(map[string]any)["volume_mounts"].([]any)[0].(map[string]any) + if mount["sub_path"] != "agents/sandbox/deps" || mount["mount_path"] != OpenShellWorkspaceMount { + t.Errorf("mount = %v", mount) + } + pod := k8s["pod"].(map[string]any) + if pod["node_selector"].(map[string]string)["kernel"] != "6.12" { + t.Errorf("pod = %v", pod) + } + if tols := pod["tolerations"].([]any); len(tols) != 1 || tols[0].(map[string]any)["key"] != "sandbox" { + t.Errorf("tolerations = %v", tols) + } +} + +func TestOpenShellRunRejectsWorkspaceOutsideClaim(t *testing.T) { + fake := &fakeOpenShell{} + l := newTestOpenShell(fake) + spec := testSpec() + spec.Workspace = "/srv/elsewhere/deps" + _, err := l.Run(context.Background(), spec) + if err == nil || !strings.Contains(err.Error(), "workspace claim mount") { + t.Fatalf("err = %v", err) + } + if len(fake.created) != 0 { + t.Error("sandbox created despite invalid workspace") + } +} + +func TestOpenShellRunTimeoutDeletesSandbox(t *testing.T) { + fake := &fakeOpenShell{runBlocks: true} + l := newTestOpenShell(fake) + l.Options.Timeout = 50 * time.Millisecond + res, err := l.Run(context.Background(), testSpec()) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want deadline", err) + } + if res.ExitCode != OpenShellExitCodeUnknown { + t.Errorf("exit = %d, want unknown", res.ExitCode) + } + if len(fake.deleted) != 1 { + t.Fatalf("deleted = %v, want the running sandbox removed on timeout", fake.deleted) + } + if !fake.deleteCtxOK { + t.Error("cleanup used the dead kick context") + } +} + +func TestOpenShellRunWaitReadyFailureCleansUp(t *testing.T) { + fake := &fakeOpenShell{waitErr: errors.New("image pull backoff")} + l := newTestOpenShell(fake) + _, err := l.Run(context.Background(), testSpec()) + if err == nil || !strings.Contains(err.Error(), "never became ready") { + t.Fatalf("err = %v", err) + } + if len(fake.deleted) != 1 { + t.Errorf("deleted = %v", fake.deleted) + } +} + +func TestOpenShellRunRequiresImage(t *testing.T) { + fake := &fakeOpenShell{} + l := newTestOpenShell(fake) + spec := testSpec() + spec.Image = "" + if _, err := l.Run(context.Background(), spec); err == nil || !strings.Contains(err.Error(), "image is required") { + t.Fatalf("err = %v", err) + } + if len(fake.created) != 0 { + t.Error("sandbox created without an image") + } +} + +func TestOpenShellNetworkPolicy(t *testing.T) { + p, err := OpenShellNetworkPolicy([]string{"litellm.example.com", "hub.example.org:8443", "github.com", "litellm.example.com"}, nil) + if err != nil { + t.Fatal(err) + } + if p.Version != OpenShellPolicyVersion { + t.Errorf("version = %d", p.Version) + } + git := p.NetworkPolicies["hive-git"] + if len(git.Endpoints) != 1 || git.Endpoints[0].Host != "github.com" || git.Endpoints[0].Access != openshell.NetworkAccessPresetReadWrite { + t.Errorf("git rule = %+v", git) + } + api := p.NetworkPolicies["hive-github-api"] + if api.Endpoints[0].Access != openshell.NetworkAccessPresetReadOnly { + t.Errorf("api.github.com must be read-only: %+v", api) + } + if len(git.Binaries) != len(OpenShellDefaultBinaries) { + t.Errorf("binaries = %d, want defaults applied", len(git.Binaries)) + } + extra := p.NetworkPolicies["hive-allow-hosts"] + if len(extra.Endpoints) != 2 { + t.Fatalf("extra endpoints = %+v, want deduplicated hosts", extra.Endpoints) + } + if extra.Endpoints[0].Host != "hub.example.org" || extra.Endpoints[0].Port != 8443 || extra.Endpoints[1].Port != 443 { + t.Errorf("extra = %+v", extra.Endpoints) + } + for _, r := range p.NetworkPolicies { + for _, e := range r.Endpoints { + if e.Enforcement != openshell.NetworkEnforcementModeEnforce { + t.Errorf("%s/%s not enforced", r.Name, e.Host) + } + } + } + if p.Filesystem == nil || !p.Filesystem.IncludeWorkdir { + t.Error("filesystem policy must include the workdir") + } +} + +func TestOpenShellNetworkPolicyRejectsURLs(t *testing.T) { + if _, err := OpenShellNetworkPolicy([]string{"https://litellm.example.com"}, nil); err == nil { + t.Error("URL accepted as allow_hosts entry") + } + if _, err := OpenShellNetworkPolicy([]string{"host:99999"}, nil); err == nil { + t.Error("bad port accepted") + } +} + +func TestOpenShellNetworkNoneDropsEgress(t *testing.T) { + fake := &fakeOpenShell{} + l := newTestOpenShell(fake) + spec := testSpec() + spec.NetworkNone = true + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatal(err) + } + if n := len(fake.created[0].spec.Policy.NetworkPolicies); n != 0 { + t.Errorf("network rules = %d, want none", n) + } +} + +func TestOpenShellOptionsValidate(t *testing.T) { + cases := map[string]OpenShellOptions{ + "no gateway": {WorkspaceClaim: "c"}, + "no claim": {GatewayURL: "gw:8080"}, + "bad scheme": {GatewayURL: "ftp://gw", WorkspaceClaim: "c"}, + "bad host": {GatewayURL: "gw:8080", WorkspaceClaim: "c", AllowHosts: []string{"http://x"}}, + "bare scheme": {GatewayURL: "gw", WorkspaceClaim: "c"}, + } + for name, o := range cases { + if err := o.Validate(); err == nil { + t.Errorf("%s: want error", name) + } + } + ok := []string{"gw:8080", "http://gw:8080", "https://gw", "grpcs://gw:9443"} + for _, g := range ok { + if err := (OpenShellOptions{GatewayURL: g, WorkspaceClaim: "c"}).Validate(); err != nil { + t.Errorf("%s: %v", g, err) + } + } +} + +func TestOpenShellAddress(t *testing.T) { + cases := []struct { + in, addr string + tls bool + }{ + {"gw:8080", "gw:8080", false}, + {"http://gw:8080", "gw:8080", false}, + {"https://gw", "gw:443", true}, + {"https://gw:9443", "gw:9443", true}, + } + for _, c := range cases { + addr, tls, err := openshellAddress(c.in) + if err != nil || addr != c.addr || tls != c.tls { + t.Errorf("%s: %q %v %v", c.in, addr, tls, err) + } + } +} + +func TestOpenShellSandboxName(t *testing.T) { + now := time.Unix(1700000000, 0) + if got := openshellSandboxName("Deps Agent-sandbox", now); got != "hive-kick-deps-agent-1700000000" { + t.Errorf("name = %q", got) + } + long := openshellSandboxName(strings.Repeat("a", 100), now) + if len(long) > openshellNameMax || !strings.HasSuffix(long, "-1700000000") { + t.Errorf("long name = %q (%d)", long, len(long)) + } + if got := openshellSandboxName("---", now); got != "hive-kick-agent-1700000000" { + t.Errorf("empty base = %q", got) + } +} + +func TestOpenShellRunRefusesClineProvider(t *testing.T) { + spec, _ := clineLaunchFixture(t) + fake := &fakeOpenShell{} + l := newTestOpenShell(fake) + spec.Image = "registry.example/dev:sha256-abc" + spec.Workspace = "/data/agents/sandbox/deps" + if _, err := l.Run(context.Background(), spec); err == nil || !strings.Contains(err.Error(), "Podman-only") { + t.Fatalf("err = %v", err) + } + if len(fake.created) != 0 { + t.Error("sandbox created despite the Cline provider refusal") + } +} + +func TestOpenShellRunRejectsBadOptionsAndSpec(t *testing.T) { + fake := &fakeOpenShell{} + l := newTestOpenShell(fake) + l.Options.WorkspaceClaim = "" + if _, err := l.Run(context.Background(), testSpec()); err == nil || !strings.Contains(err.Error(), "workspace_claim is required") { + t.Fatalf("missing claim: err = %v", err) + } + + l = newTestOpenShell(fake) + spec := testSpec() + spec.Workspace = "" + if _, err := l.Run(context.Background(), spec); err == nil || !strings.Contains(err.Error(), "workspace is required") { + t.Fatalf("missing workspace: err = %v", err) + } + + spec = testSpec() + spec.Command = nil + if _, err := l.Run(context.Background(), spec); err == nil || !strings.Contains(err.Error(), "command is required") { + t.Fatalf("missing command: err = %v", err) + } + + l.Options.AllowHosts = []string{"https://not-a-host"} + if _, err := l.Run(context.Background(), testSpec()); err == nil || !strings.Contains(err.Error(), "must be host or host:port") { + t.Fatalf("bad allow_hosts: err = %v", err) + } + if len(fake.created) != 0 { + t.Error("a sandbox was created for an invalid launch") + } +} + +func TestOpenShellRunClientAndCreateErrors(t *testing.T) { + l := newTestOpenShell(&fakeOpenShell{}) + l.newClient = func(OpenShellOptions) (openshellClient, error) { return nil, errors.New("dial refused") } + if _, err := l.Run(context.Background(), testSpec()); err == nil || !strings.Contains(err.Error(), "dial refused") { + t.Fatalf("client error: err = %v", err) + } + + fake := &fakeOpenShell{createErr: errors.New("quota exceeded")} + l = newTestOpenShell(fake) + res, err := l.Run(context.Background(), testSpec()) + if err == nil || !strings.Contains(err.Error(), "creating sandbox") || !strings.Contains(err.Error(), "quota exceeded") { + t.Fatalf("create error: err = %v", err) + } + if res.ExitCode != OpenShellExitCodeUnknown { + t.Errorf("exit = %d, want unknown", res.ExitCode) + } + if len(fake.deleted) != 0 { + t.Errorf("deleted = %v, nothing to clean up when create failed", fake.deleted) + } + if !fake.closed { + t.Error("client not closed after a failed create") + } +} + +func TestOpenShellRunExecErrorCleansUpAndWraps(t *testing.T) { + fake := &fakeOpenShell{runErr: errors.New("stream reset")} + l := newTestOpenShell(fake) + _, err := l.Run(context.Background(), testSpec()) + if err == nil || !strings.Contains(err.Error(), "running kick in sandbox") || !strings.Contains(err.Error(), "stream reset") { + t.Fatalf("err = %v", err) + } + if len(fake.deleted) != 1 { + t.Errorf("deleted = %v, want the sandbox removed after an exec failure", fake.deleted) + } +} + +func TestOpenShellRunWaitReadyCanceledReturnsContextError(t *testing.T) { + fake := &fakeOpenShell{waitBlocks: true} + l := newTestOpenShell(fake) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := l.Run(ctx, testSpec()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want canceled", err) + } + if len(fake.deleted) != 1 || !fake.deleteCtxOK { + t.Errorf("deleted = %v ctxOK = %v, want cleanup on a live context", fake.deleted, fake.deleteCtxOK) + } +} + +func TestOpenShellRunPolicyProfileRuntimeClassAndOutputCap(t *testing.T) { + big := bytes.Repeat([]byte("x"), openshellMaxOutput+10) + fake := &fakeOpenShell{runResult: &openshell.ExecResult{ExitCode: 0, Stdout: big, Stderr: big}} + l := newTestOpenShell(fake) + l.Options.PolicyProfile = "strict" + l.Options.RuntimeClassName = "gvisor" + l.Options.Workspace = "team-a" + res, err := l.Run(context.Background(), testSpec()) + if err != nil { + t.Fatal(err) + } + if len(res.Stdout) != openshellMaxOutput || len(res.Stderr) != openshellMaxOutput { + t.Errorf("stdout %d stderr %d, want capped at %d", len(res.Stdout), len(res.Stderr), openshellMaxOutput) + } + if len(fake.created) != 1 { + t.Fatalf("created = %d", len(fake.created)) + } + c := fake.created[0] + if c.workspace != "team-a" { + t.Errorf("workspace = %q, want the configured OpenShell workspace", c.workspace) + } + if c.spec.Template.RuntimeClassName != "gvisor" { + t.Errorf("runtime class = %q", c.spec.Template.RuntimeClassName) + } + k8s, _ := c.spec.Template.DriverConfig["kubernetes"].(map[string]any) + pod, _ := k8s["pod"].(map[string]any) + if pod == nil || pod["runtime_class_name"] != "gvisor" { + t.Errorf("driver_config pod = %v, want runtime_class_name gvisor", pod) + } +} + +func TestOpenShellNetworkPolicySortsAllowHostsByPort(t *testing.T) { + policy, err := OpenShellNetworkPolicy([]string{"svc.example:9443", "svc.example:8443"}, nil) + if err != nil { + t.Fatal(err) + } + eps := policy.NetworkPolicies["hive-allow-hosts"].Endpoints + if len(eps) != 2 || eps[0].Port != 8443 || eps[1].Port != 9443 { + t.Errorf("endpoints = %+v, want sorted by port within a host", eps) + } +} + +func TestOpenShellAddressAndHostPortErrors(t *testing.T) { + for _, in := range []string{"", "gw", "gw:notaport", "ftp://gw:21", "http://", "http://%zz"} { + if _, _, err := openshellAddress(in); err == nil { + t.Errorf("openshellAddress(%q) accepted", in) + } + } + if addr, tls, err := openshellAddress("http://gw"); err != nil || addr != "gw:80" || tls { + t.Errorf("http default port: %q %v %v", addr, tls, err) + } + if addr, tls, err := openshellAddress("grpcs://gw:7443"); err != nil || addr != "gw:7443" || !tls { + t.Errorf("grpcs: %q %v %v", addr, tls, err) + } + for _, in := range []string{"", " ", "host:0", "host:70000", ":443", "host:abc", "host/path", "has space"} { + if _, _, err := openshellHostPort(in); err == nil { + t.Errorf("openshellHostPort(%q) accepted", in) + } + } +} + +func TestNewOpenShellSDKClientBuildsLazyClient(t *testing.T) { + // grpc.NewClient does not dial until the first RPC, so constructing the + // real SDK client against an unreachable address is hermetic. + for _, o := range []OpenShellOptions{ + {GatewayURL: "127.0.0.1:1"}, + {GatewayURL: "https://127.0.0.1:1", Token: "tok"}, + {GatewayURL: "127.0.0.1:1", InsecureTLS: true}, + } { + c, err := newOpenShellSDKClient(o) + if err != nil { + t.Fatalf("%+v: %v", o, err) + } + if c.Sandboxes() == nil || c.Exec() == nil { + t.Errorf("%+v: nil SDK service", o) + } + if err := c.Close(); err != nil { + t.Errorf("%+v: close: %v", o, err) + } + } + if _, err := newOpenShellSDKClient(OpenShellOptions{GatewayURL: "nope"}); err == nil { + t.Error("bad gateway_url accepted") + } + l := &OpenShellLauncher{Options: OpenShellOptions{GatewayURL: "127.0.0.1:1"}} + c, err := l.client() + if err != nil { + t.Fatal(err) + } + _ = c.Close() +} + +func preflightFixture(fake *fakeOpenShell) (*OpenShellPreflighter, *time.Time) { + clock := time.Unix(1700000000, 0) + return &OpenShellPreflighter{ + newClient: func(OpenShellOptions) (openshellClient, error) { return fake, nil }, + now: func() time.Time { return clock }, + }, &clock +} + +func goodGateway() *openshell.GatewayConfig { + return &openshell.GatewayConfig{Settings: map[string]openshell.SettingValue{ + "allow_driver_config": {Type: openshell.SettingValueBool, BoolVal: true}, + "telemetry_enabled": {Type: openshell.SettingValueBool, BoolVal: false}, + }} +} + +func TestOpenShellPreflightHealthyGatewayIsQuiet(t *testing.T) { + p, _ := preflightFixture(&fakeOpenShell{gateway: goodGateway()}) + if got := p.Check(context.Background(), OpenShellOptions{GatewayURL: "gw:8080"}); len(got) != 0 { + t.Errorf("warnings = %q, want none", got) + } +} + +func TestOpenShellPreflightWarnings(t *testing.T) { + bad := goodGateway() + bad.Settings["allow_driver_config"] = openshell.SettingValue{Type: openshell.SettingValueBool} + bad.Settings["telemetry_enabled"] = openshell.SettingValue{Type: openshell.SettingValueBool, BoolVal: true} + cases := []struct { + name string + fake *fakeOpenShell + want []string + }{ + {"unreachable", &fakeOpenShell{healthErr: errors.New("connection refused")}, []string{"unreachable"}}, + {"unhealthy", &fakeOpenShell{unhealthy: true, gateway: goodGateway()}, []string{"unhealthy"}}, + {"driver config off and telemetry on", &fakeOpenShell{gateway: bad}, []string{"allow_driver_config", "telemetry"}}, + {"settings unreadable", &fakeOpenShell{gatewayErr: errors.New("denied")}, []string{"could not be read"}}, + } + for _, c := range cases { + p, _ := preflightFixture(c.fake) + got := strings.Join(p.Check(context.Background(), OpenShellOptions{GatewayURL: "gw:8080"}), "\n") + for _, w := range c.want { + if !strings.Contains(got, w) { + t.Errorf("%s: warnings %q lack %q", c.name, got, w) + } + } + } +} + +func TestOpenShellPreflightIsCachedPerGatewayUntilTTL(t *testing.T) { + fake := &fakeOpenShell{gateway: goodGateway()} + p, clock := preflightFixture(fake) + opts := OpenShellOptions{GatewayURL: "gw:8080"} + p.Check(context.Background(), opts) + p.Check(context.Background(), opts) + if fake.healthCalls != 1 || fake.configCalls != 1 { + t.Fatalf("gateway hit %d/%d times, want once", fake.healthCalls, fake.configCalls) + } + p.Check(context.Background(), OpenShellOptions{GatewayURL: "other:8080"}) + if fake.healthCalls != 2 { + t.Errorf("a different gateway must be probed separately") + } + *clock = clock.Add(openshellPreflightTTL + time.Second) + p.Check(context.Background(), opts) + if fake.healthCalls != 3 { + t.Errorf("expired cache entry not refreshed") + } +} + +func TestOpenShellPreflightClientErrorIsAWarning(t *testing.T) { + p := &OpenShellPreflighter{newClient: func(OpenShellOptions) (openshellClient, error) { return nil, errors.New("bad address") }} + got := p.Check(context.Background(), OpenShellOptions{GatewayURL: "gw:8080"}) + if len(got) != 1 || !strings.Contains(got[0], "unreachable") { + t.Errorf("warnings = %q", got) + } +} + +// Egress-policy prover: every shipped example allow_hosts set must render to +// a policy that cannot reach more than hive intends. A config change that +// widens egress fails here. +var shippedOpenShellAllowHosts = [][]string{ + nil, + {"litellm.example.com", "hub.hivecommons.dev"}, // docs/sandbox-isolation.md + {"litellm.example.com", "hub.example.org:8443"}, +} + +func TestOpenShellEgressPolicyProver(t *testing.T) { + for _, hosts := range shippedOpenShellAllowHosts { + p, err := OpenShellNetworkPolicy(hosts, nil) + if err != nil { + t.Fatalf("%v: %v", hosts, err) + } + sawAPI := false + for name, r := range p.NetworkPolicies { + if len(r.Binaries) == 0 { + t.Errorf("%v: rule %s has no binaries and would allow nothing", hosts, name) + } + for _, e := range r.Endpoints { + if strings.ContainsAny(e.Host, "*?") || e.Host == "" { + t.Errorf("%v: rule %s has wildcard/empty host %q", hosts, name, e.Host) + } + if e.Port == 0 { + t.Errorf("%v: rule %s endpoint %s has no port", hosts, name, e.Host) + } + if e.Enforcement != openshell.NetworkEnforcementModeEnforce { + t.Errorf("%v: rule %s endpoint %s is not enforced", hosts, name, e.Host) + } + if e.Host == "api.github.com" { + sawAPI = true + if e.Access != openshell.NetworkAccessPresetReadOnly { + t.Errorf("%v: api.github.com access = %v, must be read-only", hosts, e.Access) + } + } + } + } + if !sawAPI { + t.Errorf("%v: api.github.com rule missing", hosts) + } + } +} + +func TestOpenShellEgressPolicyProverRejectsSmuggledHosts(t *testing.T) { + for _, h := range []string{ + "https://evil.example.com", "evil.example.com/path", "user@evil.example.com", + "user:pw@evil.example.com:443", "evil.example.com?x=1", "evil.example.com#frag", + "*.example.com", "*", "evil.example.com:443:80", "evil .example.com", ":443", "-evil.com", + } { + if _, err := OpenShellNetworkPolicy([]string{h}, nil); err == nil { + t.Errorf("allow_hosts entry %q accepted", h) + } + if err := (OpenShellOptions{GatewayURL: "gw:8080", WorkspaceClaim: "c", AllowHosts: []string{h}}).Validate(); err == nil { + t.Errorf("Validate accepted allow_hosts entry %q", h) + } + } +} diff --git a/src/crux/upstreamproxy/upstreamproxy.go b/src/crux/upstreamproxy/upstreamproxy.go new file mode 100644 index 0000000000..723dfa230b --- /dev/null +++ b/src/crux/upstreamproxy/upstreamproxy.go @@ -0,0 +1,502 @@ +// Package upstreamproxy decides which outbound connections must be chained +// through a corporate HTTP forward proxy, and opens them. +// +// The motivating deployment is a hive whose GitHub Enterprise host is reachable +// only through an enterprise egress proxy while everything else (github.com, +// model providers, the hub) is reached directly. Every :443 egress from a hive +// pod is iptables-redirected into the in-process MITM proxy (crux/proxy), which +// used to dial upstream directly; this package gives that proxy — and the +// control plane's own GitHub clients — a single rule set answering "does this +// host go via the upstream proxy, and if so which one?". +// +// Resolution order (Resolve): the hive config block `network.upstream_proxy` +// (url / hosts / no_proxy), each field overridable by HIVE_UPSTREAM_PROXY_URL, +// HIVE_UPSTREAM_PROXY_HOSTS and HIVE_UPSTREAM_PROXY_NO_PROXY; when no URL is +// configured that way the standard HTTPS_PROXY / HTTP_PROXY / NO_PROXY +// environment is honoured with http.ProxyFromEnvironment semantics (every host +// not excluded by NO_PROXY). Config wins over the standard environment. +package upstreamproxy + +import ( + "bufio" + "context" + "crypto/tls" + "encoding/base64" + "errors" + "fmt" + "net" + "net/http" + "net/netip" + "net/url" + "strings" + "sync" + "time" +) + +// Environment variable names. HIVE_UPSTREAM_PROXY_HOSTS and +// HIVE_UPSTREAM_PROXY_NO_PROXY are comma-separated lists of host patterns. +const ( + URLEnv = "HIVE_UPSTREAM_PROXY_URL" + HostsEnv = "HIVE_UPSTREAM_PROXY_HOSTS" + NoProxyEnv = "HIVE_UPSTREAM_PROXY_NO_PROXY" +) + +// Config is the operator-facing rule set (the `network.upstream_proxy` block). +type Config struct { + // URL is the forward proxy, e.g. "http://proxy.corp.example:3128". + // Userinfo, when present, is sent as Proxy-Authorization: Basic. + URL string + // Hosts lists the destination host patterns that must go via the proxy. + // Empty means every host not matched by NoProxy. Patterns: an exact host + // ("github.corp.example"), a wildcard ("*.corp.example" — subdomains + // only), a dotted suffix (".corp.example" — the domain and its + // subdomains), an IP literal, a CIDR, or "*". + Hosts []string + // NoProxy lists host patterns that are always dialled directly, with the + // same pattern grammar. Loopback is direct unless a Hosts rule names it + // explicitly (an IP literal or CIDR). + NoProxy []string +} + +// Selector answers ProxyFor for one resolved rule set. A nil *Selector is +// valid and means "no upstream proxy": every method on it reports direct. +type Selector struct { + proxyURL *url.URL + hosts []hostRule + noProxy []hostRule + source string +} + +// Resolve builds the selector from cfg overlaid with the process environment +// (getenv; nil means the environment is not consulted). It returns (nil, nil) +// when nothing configures an upstream proxy, and an error only for a +// malformed proxy URL or pattern — the caller logs it and runs direct. +func Resolve(cfg Config, getenv func(string) string) (*Selector, error) { + if getenv == nil { + getenv = func(string) string { return "" } + } + source := "config" + rawURL := strings.TrimSpace(cfg.URL) + hosts := cfg.Hosts + noProxy := cfg.NoProxy + if v := strings.TrimSpace(getenv(URLEnv)); v != "" { + rawURL = v + } + if v, ok := envList(getenv, HostsEnv); ok { + hosts = v + } + if v, ok := envList(getenv, NoProxyEnv); ok { + noProxy = v + } + if rawURL == "" { + // Standard environment fallback: HTTPS_PROXY (then HTTP_PROXY) applies + // to every host NO_PROXY does not exclude. + source = "env" + hosts = nil + for _, name := range []string{"HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"} { + if v := strings.TrimSpace(getenv(name)); v != "" { + rawURL = v + break + } + } + if rawURL == "" { + return nil, nil + } + if _, explicit := envList(getenv, NoProxyEnv); !explicit { + noProxy = nil + for _, name := range []string{"NO_PROXY", "no_proxy"} { + if v, ok := envList(getenv, name); ok { + noProxy = v + break + } + } + } + } + return New(rawURL, hosts, noProxy, source) +} + +// New builds a selector from already-resolved values. source labels where +// the rules came from ("config" or "env") for logging. +func New(rawURL string, hosts, noProxy []string, source string) (*Selector, error) { + u, err := parseProxyURL(rawURL) + if err != nil { + return nil, err + } + hostRules, err := parseRules(hosts) + if err != nil { + return nil, fmt.Errorf("upstream proxy hosts: %w", err) + } + noProxyRules, err := parseRules(noProxy) + if err != nil { + return nil, fmt.Errorf("upstream proxy no_proxy: %w", err) + } + if source == "" { + source = "config" + } + return &Selector{proxyURL: u, hosts: hostRules, noProxy: noProxyRules, source: source}, nil +} + +func parseProxyURL(raw string) (*url.URL, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, errors.New("upstream proxy url is empty") + } + if !strings.Contains(raw, "://") { + raw = "http://" + raw + } + u, err := url.Parse(raw) + if err != nil { + return nil, fmt.Errorf("upstream proxy url %q: %w", raw, err) + } + switch u.Scheme { + case "http", "https": + default: + return nil, fmt.Errorf("upstream proxy url %q: scheme %q not supported (http or https)", raw, u.Scheme) + } + if u.Hostname() == "" { + return nil, fmt.Errorf("upstream proxy url %q: missing host", raw) + } + if u.Port() == "" { + port := "80" + if u.Scheme == "https" { + port = "443" + } + u.Host = net.JoinHostPort(u.Hostname(), port) + } + return u, nil +} + +func envList(getenv func(string) string, name string) ([]string, bool) { + raw := getenv(name) + if strings.TrimSpace(raw) == "" { + return nil, false + } + var out []string + for _, p := range strings.Split(raw, ",") { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out, true +} + +// URL returns the configured proxy URL, or nil for a nil selector. +func (s *Selector) URL() *url.URL { + if s == nil { + return nil + } + return s.proxyURL +} + +// Source reports where the rules came from: "config", "env", or "" (none). +func (s *Selector) Source() string { + if s == nil { + return "" + } + return s.source +} + +// Restricted reports whether the selector names specific hosts (true) or +// applies to every non-excluded host (false). +func (s *Selector) Restricted() bool { + return s != nil && len(s.hosts) > 0 +} + +// ProxyFor returns the proxy a connection to hostport ("host" or +// "host:port") must be chained through, or nil to dial directly. +func (s *Selector) ProxyFor(hostport string) *url.URL { + if s == nil || s.proxyURL == nil { + return nil + } + host := normalizeHost(hostport) + if host == "" { + return nil + } + for _, r := range s.noProxy { + if r.matches(host) { + return nil + } + } + loopback := isLoopbackHost(host) + if len(s.hosts) == 0 { + if loopback { + return nil + } + return s.proxyURL + } + for _, r := range s.hosts { + if r.matches(host) && !(loopback && r.kind == ruleAll) { + return s.proxyURL + } + } + return nil +} + +// ProxyFunc adapts the selector to http.Transport.Proxy. A nil selector +// yields a function that always reports direct, so callers can assign it +// unconditionally. +func (s *Selector) ProxyFunc() func(*http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { + if req == nil || req.URL == nil { + return nil, nil + } + return s.ProxyFor(req.URL.Host), nil + } +} + +// GitEnv returns the environment entries that make a child `git` process +// honour the same rules, expressed as GIT_CONFIG_COUNT/KEY/VALUE entries so +// nothing else in the child (or the parent's Go clients) is affected: +// `http.proxy` when the selector applies to every host, otherwise one +// `http.https:///.proxy` entry per exact-host pattern. Wildcard, suffix, +// IP and CIDR patterns cannot be expressed per-URL in git config and are +// skipped (documented limitation). NoProxy is handed over as `no_proxy`. +func (s *Selector) GitEnv() []string { + if s == nil || s.proxyURL == nil { + return nil + } + var pairs [][2]string + if len(s.hosts) == 0 { + pairs = append(pairs, [2]string{"http.proxy", s.proxyURL.String()}) + } else { + for _, r := range s.hosts { + if r.kind == ruleExact { + pairs = append(pairs, [2]string{"http.https://" + r.host + "/.proxy", s.proxyURL.String()}) + } + } + } + if len(pairs) == 0 { + return nil + } + env := []string{fmt.Sprintf("GIT_CONFIG_COUNT=%d", len(pairs))} + for i, kv := range pairs { + env = append(env, + fmt.Sprintf("GIT_CONFIG_KEY_%d=%s", i, kv[0]), + fmt.Sprintf("GIT_CONFIG_VALUE_%d=%s", i, kv[1])) + } + if len(s.noProxy) > 0 { + raw := make([]string, 0, len(s.noProxy)) + for _, r := range s.noProxy { + raw = append(raw, r.raw) + } + env = append(env, "no_proxy="+strings.Join(raw, ",")) + } + return env +} + +// Dialer is the context dial function the CONNECT leg uses to reach the +// proxy itself (the caller supplies its SO_MARKed dialer). +type Dialer func(ctx context.Context, network, address string) (net.Conn, error) + +// connectResponseTimeout bounds how long the proxy may take to answer the +// CONNECT once the TCP leg is up; the caller's dial timeout bounds the leg. +const connectResponseTimeout = 15 * time.Second + +// Dial opens a TCP tunnel to target ("host:port") through proxyURL: it dials +// the proxy with dial, issues `CONNECT target`, and returns the connection +// positioned after the proxy's 200 response so the caller can run TLS (or +// splice bytes) over it exactly as it would over a direct connection. +func Dial(ctx context.Context, dial Dialer, proxyURL *url.URL, target string) (net.Conn, error) { + if proxyURL == nil { + return nil, errors.New("upstream proxy: nil proxy URL") + } + if _, _, err := net.SplitHostPort(target); err != nil { + return nil, fmt.Errorf("upstream proxy: target %q must be host:port: %w", target, err) + } + raw, err := dial(ctx, "tcp", proxyURL.Host) + if err != nil { + return nil, fmt.Errorf("upstream proxy %s: dial: %w", proxyURL.Host, err) + } + conn := raw + if proxyURL.Scheme == "https" { + tlsConn := tls.Client(raw, &tls.Config{ServerName: proxyURL.Hostname()}) + if err := tlsConn.HandshakeContext(ctx); err != nil { + _ = raw.Close() + return nil, fmt.Errorf("upstream proxy %s: tls: %w", proxyURL.Host, err) + } + conn = tlsConn + } + + req := &http.Request{ + Method: http.MethodConnect, + URL: &url.URL{Opaque: target}, + Host: target, + Header: http.Header{}, + } + if proxyURL.User != nil { + pass, _ := proxyURL.User.Password() + cred := base64.StdEncoding.EncodeToString([]byte(proxyURL.User.Username() + ":" + pass)) + req.Header.Set("Proxy-Authorization", "Basic "+cred) + } + + deadline := time.Now().Add(connectResponseTimeout) + if d, ok := ctx.Deadline(); ok && d.Before(deadline) { + deadline = d + } + _ = conn.SetDeadline(deadline) + if err := req.Write(conn); err != nil { + _ = conn.Close() + return nil, fmt.Errorf("upstream proxy %s: write CONNECT: %w", proxyURL.Host, err) + } + br := bufio.NewReader(conn) + resp, err := http.ReadResponse(br, req) + if err != nil { + _ = conn.Close() + return nil, fmt.Errorf("upstream proxy %s: read CONNECT response: %w", proxyURL.Host, err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + _ = conn.Close() + return nil, fmt.Errorf("upstream proxy %s: CONNECT %s refused: %s", proxyURL.Host, target, resp.Status) + } + _ = conn.SetDeadline(time.Time{}) + if br.Buffered() > 0 { + // The proxy sent bytes past its response header (unusual: the client + // speaks first in every protocol tunnelled here). Preserve them. + return &bufferedConn{Conn: conn, r: br}, nil + } + return conn, nil +} + +type bufferedConn struct { + net.Conn + r *bufio.Reader +} + +func (c *bufferedConn) Read(p []byte) (int, error) { return c.r.Read(p) } + +// ---- host rules ------------------------------------------------------------ + +type ruleKind int + +const ( + ruleAll ruleKind = iota + ruleExact + ruleSubdomains // *.example.com — subdomains only + ruleSuffix // .example.com — the domain and its subdomains + ruleIP + ruleCIDR +) + +type hostRule struct { + raw string + kind ruleKind + host string + ip netip.Addr + prefix netip.Prefix +} + +func parseRules(patterns []string) ([]hostRule, error) { + rules := make([]hostRule, 0, len(patterns)) + for _, p := range patterns { + raw := strings.TrimSpace(p) + if raw == "" { + continue + } + r, err := parseRule(raw) + if err != nil { + return nil, err + } + rules = append(rules, r) + } + return rules, nil +} + +func parseRule(raw string) (hostRule, error) { + r := hostRule{raw: raw} + p := strings.ToLower(raw) + if p == "*" { + r.kind = ruleAll + return r, nil + } + if strings.Contains(p, "/") { + prefix, err := netip.ParsePrefix(p) + if err != nil { + return r, fmt.Errorf("pattern %q: %w", raw, err) + } + r.kind, r.prefix = ruleCIDR, prefix.Masked() + return r, nil + } + if addr, err := netip.ParseAddr(strings.Trim(p, "[]")); err == nil { + r.kind, r.ip = ruleIP, addr.Unmap() + return r, nil + } + // Strip a :port a NO_PROXY-style entry may carry; ports are not matched. + if h, _, err := net.SplitHostPort(p); err == nil { + p = h + } + switch { + case strings.HasPrefix(p, "*."): + r.kind, r.host = ruleSubdomains, strings.TrimSuffix(p[2:], ".") + case strings.HasPrefix(p, "."): + r.kind, r.host = ruleSuffix, strings.TrimSuffix(p[1:], ".") + default: + r.kind, r.host = ruleExact, strings.TrimSuffix(p, ".") + } + if r.host == "" || strings.ContainsAny(r.host, "*/ ") { + return r, fmt.Errorf("pattern %q: not a host, wildcard, suffix, IP or CIDR", raw) + } + return r, nil +} + +func (r hostRule) matches(host string) bool { + switch r.kind { + case ruleAll: + return true + case ruleExact: + return host == r.host + case ruleSubdomains: + return strings.HasSuffix(host, "."+r.host) + case ruleSuffix: + return host == r.host || strings.HasSuffix(host, "."+r.host) + case ruleIP: + addr, err := netip.ParseAddr(host) + return err == nil && addr.Unmap() == r.ip + case ruleCIDR: + addr, err := netip.ParseAddr(host) + return err == nil && r.prefix.Contains(addr.Unmap()) + } + return false +} + +// normalizeHost lowercases hostport, strips a port and brackets, and drops a +// trailing dot. +func normalizeHost(hostport string) string { + h := strings.TrimSpace(hostport) + if host, _, err := net.SplitHostPort(h); err == nil { + h = host + } + h = strings.Trim(h, "[]") + return strings.TrimSuffix(strings.ToLower(h), ".") +} + +func isLoopbackHost(host string) bool { + if host == "localhost" || strings.HasSuffix(host, ".localhost") { + return true + } + addr, err := netip.ParseAddr(host) + return err == nil && addr.Unmap().IsLoopback() +} + +// ---- process-wide default -------------------------------------------------- + +var ( + defaultMu sync.RWMutex + defaultSelector *Selector +) + +// SetDefault installs the process-wide selector the hive resolved at boot; +// nil clears it. In-process git execs (GitEnv) read it so a clone of the GHE +// host from the control plane follows the same rules as the MITM proxy. +func SetDefault(s *Selector) { + defaultMu.Lock() + defaultSelector = s + defaultMu.Unlock() +} + +// Default returns the process-wide selector, or nil when none is installed. +func Default() *Selector { + defaultMu.RLock() + defer defaultMu.RUnlock() + return defaultSelector +} diff --git a/src/crux/upstreamproxy/upstreamproxy_test.go b/src/crux/upstreamproxy/upstreamproxy_test.go new file mode 100644 index 0000000000..f3d45b8623 --- /dev/null +++ b/src/crux/upstreamproxy/upstreamproxy_test.go @@ -0,0 +1,430 @@ +package upstreamproxy + +import ( + "bufio" + "context" + "crypto/tls" + "encoding/base64" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" +) + +func mapEnv(m map[string]string) func(string) string { + return func(k string) string { return m[k] } +} + +func TestResolveConfigHostsAndNoProxy(t *testing.T) { + sel, err := Resolve(Config{ + URL: "http://user:pw@proxy.example:3128", + Hosts: []string{"github.ibm.com", "*.github.ibm.com"}, + NoProxy: []string{"raw.github.ibm.com"}, + }, nil) + if err != nil || sel == nil { + t.Fatalf("Resolve: sel=%v err=%v", sel, err) + } + if sel.Source() != "config" || !sel.Restricted() { + t.Fatalf("source=%q restricted=%v", sel.Source(), sel.Restricted()) + } + cases := map[string]bool{ + "github.ibm.com": true, + "GITHUB.IBM.COM:443": true, + "api.github.ibm.com:443": true, + "raw.github.ibm.com": false, // no_proxy beats hosts + "github.com": false, + "api.github.com": false, + "localhost:8080": false, + "127.0.0.1": false, + } + for host, want := range cases { + if got := sel.ProxyFor(host) != nil; got != want { + t.Errorf("ProxyFor(%q) via proxy = %v, want %v", host, got, want) + } + } + if u := sel.URL(); u.Host != "proxy.example:3128" || u.User.Username() != "user" { + t.Fatalf("proxy URL = %v", u) + } +} + +func TestResolveEmptyHostsMeansAllButNoProxy(t *testing.T) { + sel, err := Resolve(Config{URL: "proxy.example:8080", NoProxy: []string{".hivecommons.dev", "10.0.0.0/8", "192.168.1.5"}}, nil) + if err != nil { + t.Fatal(err) + } + if sel.Restricted() { + t.Fatal("empty hosts must not be restricted") + } + if sel.URL().String() != "http://proxy.example:8080" { + t.Fatalf("scheme default: %s", sel.URL()) + } + for host, want := range map[string]bool{ + "github.ibm.com": true, + "api.anthropic.com": true, + "hive.hivecommons.dev": false, + "hivecommons.dev": false, + "10.20.30.40:443": false, + "192.168.1.5": false, + "192.168.1.6": true, + "[::ffff:10.1.2.3]:443": false, + } { + if got := sel.ProxyFor(host) != nil; got != want { + t.Errorf("ProxyFor(%q) = %v, want %v", host, got, want) + } + } +} + +func TestResolveEnvOverridesConfigAndFallsBackToStandardEnv(t *testing.T) { + // HIVE_* env overrides the file. + sel, err := Resolve(Config{URL: "http://file.example:1", Hosts: []string{"a.example"}}, mapEnv(map[string]string{ + URLEnv: "http://env.example:2", + HostsEnv: "b.example, c.example", + })) + if err != nil { + t.Fatal(err) + } + if sel.URL().Host != "env.example:2" || sel.ProxyFor("a.example") != nil || sel.ProxyFor("b.example") == nil { + t.Fatalf("env override not applied: url=%s", sel.URL()) + } + + // No hive config at all: standard HTTPS_PROXY/NO_PROXY apply to every host. + sel, err = Resolve(Config{}, mapEnv(map[string]string{ + "HTTPS_PROXY": "http://std.example:3128", + "NO_PROXY": "github.com,.githubusercontent.com", + })) + if err != nil { + t.Fatal(err) + } + if sel.Source() != "env" || sel.Restricted() { + t.Fatalf("source=%q restricted=%v", sel.Source(), sel.Restricted()) + } + if sel.ProxyFor("github.ibm.com") == nil || sel.ProxyFor("github.com") != nil || sel.ProxyFor("raw.githubusercontent.com") != nil { + t.Fatal("standard NO_PROXY semantics not honoured") + } + + // Hive config URL wins over the standard env. + sel, err = Resolve(Config{URL: "http://cfg.example:1", Hosts: []string{"github.ibm.com"}}, mapEnv(map[string]string{ + "HTTPS_PROXY": "http://std.example:3128", + })) + if err != nil { + t.Fatal(err) + } + if sel.URL().Host != "cfg.example:1" || sel.ProxyFor("github.com") != nil { + t.Fatalf("config did not win: %s", sel.URL()) + } + + // Nothing configured → nil selector, which is direct for everything. + sel, err = Resolve(Config{}, mapEnv(nil)) + if err != nil || sel != nil { + t.Fatalf("expected nil selector, got %v err=%v", sel, err) + } + if sel.ProxyFor("github.ibm.com") != nil || sel.GitEnv() != nil { + t.Fatal("nil selector must be direct") + } + if u, err := sel.ProxyFunc()(&http.Request{URL: &url.URL{Host: "github.ibm.com"}}); u != nil || err != nil { + t.Fatal("nil selector ProxyFunc must report direct") + } +} + +func TestResolveRejectsBadInput(t *testing.T) { + if _, err := Resolve(Config{URL: "socks5://p:1080"}, nil); err == nil { + t.Fatal("socks scheme accepted") + } + if _, err := Resolve(Config{URL: "http://p:3128", Hosts: []string{"bad host"}}, nil); err == nil { + t.Fatal("bad pattern accepted") + } + if _, err := Resolve(Config{URL: "http://p:3128", NoProxy: []string{"10.0.0.0/99"}}, nil); err == nil { + t.Fatal("bad CIDR accepted") + } +} + +func TestGitEnv(t *testing.T) { + sel, _ := Resolve(Config{URL: "http://p:3128", Hosts: []string{"github.ibm.com", "*.github.ibm.com"}, NoProxy: []string{"github.com"}}, nil) + got := strings.Join(sel.GitEnv(), "\n") + for _, want := range []string{ + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=http.https://github.ibm.com/.proxy", + "GIT_CONFIG_VALUE_0=http://p:3128", + "no_proxy=github.com", + } { + if !strings.Contains(got, want) { + t.Errorf("GitEnv missing %q in:\n%s", want, got) + } + } + all, _ := Resolve(Config{URL: "http://p:3128"}, nil) + if got := strings.Join(all.GitEnv(), "\n"); !strings.Contains(got, "GIT_CONFIG_KEY_0=http.proxy\n") { + t.Errorf("unrestricted GitEnv = %s", got) + } +} + +// connectProxy is a minimal CONNECT-capable forward proxy for tests. It +// records the Proxy-Authorization it saw and the targets it was asked for. +type connectProxy struct { + ln net.Listener + auth atomic.Value // string + targets atomic.Int32 + wantAuth string +} + +func startConnectProxy(t *testing.T, wantAuth string) *connectProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + p := &connectProxy{ln: ln, wantAuth: wantAuth} + p.auth.Store("") + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go p.serve(c) + } + }() + t.Cleanup(func() { _ = ln.Close() }) + return p +} + +func (p *connectProxy) serve(c net.Conn) { + defer func() { _ = c.Close() }() + br := bufio.NewReader(c) + req, err := http.ReadRequest(br) + if err != nil || req.Method != http.MethodConnect { + _, _ = io.WriteString(c, "HTTP/1.1 400 Bad Request\r\n\r\n") + return + } + p.auth.Store(req.Header.Get("Proxy-Authorization")) + if p.wantAuth != "" && req.Header.Get("Proxy-Authorization") != p.wantAuth { + _, _ = io.WriteString(c, "HTTP/1.1 407 Proxy Authentication Required\r\n\r\n") + return + } + p.targets.Add(1) + up, err := net.DialTimeout("tcp", req.Host, 5*time.Second) + if err != nil { + _, _ = io.WriteString(c, "HTTP/1.1 502 Bad Gateway\r\n\r\n") + return + } + defer func() { _ = up.Close() }() + _, _ = io.WriteString(c, "HTTP/1.1 200 Connection established\r\n\r\n") + done := make(chan struct{}, 2) + go func() { _, _ = io.Copy(up, br); done <- struct{}{} }() + go func() { _, _ = io.Copy(c, up); done <- struct{}{} }() + <-done +} + +func TestDialTunnelsTLSThroughCONNECT(t *testing.T) { + origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "hello via "+r.Host) + })) + defer origin.Close() + originHost := strings.TrimPrefix(origin.URL, "https://") + + wantAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte("alice:s3cret")) + px := startConnectProxy(t, wantAuth) + proxyURL, _ := url.Parse("http://alice:s3cret@" + px.ln.Addr().String()) + + dialer := &net.Dialer{Timeout: 5 * time.Second} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + raw, err := Dial(ctx, dialer.DialContext, proxyURL, originHost) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer func() { _ = raw.Close() }() + + pool := origin.Client().Transport.(*http.Transport).TLSClientConfig.RootCAs + tlsConn := tls.Client(raw, &tls.Config{ServerName: "127.0.0.1", RootCAs: pool}) + if err := tlsConn.HandshakeContext(ctx); err != nil { + t.Fatalf("TLS over tunnel: %v", err) + } + req, _ := http.NewRequest(http.MethodGet, "https://"+originHost+"/x", nil) + if err := req.Write(tlsConn); err != nil { + t.Fatal(err) + } + resp, err := http.ReadResponse(bufio.NewReader(tlsConn), req) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 || !strings.HasPrefix(string(body), "hello via") { + t.Fatalf("origin reply: %d %q", resp.StatusCode, body) + } + if px.auth.Load().(string) != wantAuth { + t.Fatalf("proxy saw auth %q", px.auth.Load()) + } + if px.targets.Load() != 1 { + t.Fatalf("proxy CONNECT count = %d", px.targets.Load()) + } +} + +func TestDialSurfacesProxyRefusal(t *testing.T) { + px := startConnectProxy(t, "Basic needed") + proxyURL, _ := url.Parse("http://" + px.ln.Addr().String()) + dialer := &net.Dialer{Timeout: 5 * time.Second} + _, err := Dial(context.Background(), dialer.DialContext, proxyURL, "example.invalid:443") + if err == nil || !strings.Contains(err.Error(), "407") { + t.Fatalf("expected 407 refusal, got %v", err) + } +} + +func TestParseProxyURLDefaults(t *testing.T) { + cases := map[string]string{ + "proxy.corp.example:3128": "http://proxy.corp.example:3128", + "http://proxy.corp.example": "http://proxy.corp.example:80", + "https://proxy.corp.example": "https://proxy.corp.example:443", + "http://u:p@proxy.corp.example:1": "http://u:p@proxy.corp.example:1", + } + for in, want := range cases { + sel, err := New(in, nil, nil, "config") + if err != nil { + t.Fatalf("New(%q): %v", in, err) + } + if got := sel.URL().String(); got != want { + t.Errorf("New(%q).URL() = %s, want %s", in, got, want) + } + if sel.Source() != "config" { + t.Errorf("Source() = %q", sel.Source()) + } + } + for _, bad := range []string{"", " ", "socks5://p:1080", "http://", "http://:3128"} { + if _, err := New(bad, nil, nil, "config"); err == nil { + t.Errorf("New(%q) accepted", bad) + } + } +} + +func TestNilSelectorAccessors(t *testing.T) { + var s *Selector + if s.URL() != nil || s.Source() != "" || s.Restricted() || s.GitEnv() != nil || s.ProxyFor("ghe.corp.example:443") != nil { + t.Fatal("nil selector must report no proxy") + } + if u, err := s.ProxyFunc()(nil); u != nil || err != nil { + t.Fatalf("ProxyFunc(nil req) = %v, %v", u, err) + } + if u, err := s.ProxyFunc()(&http.Request{}); u != nil || err != nil { + t.Fatalf("ProxyFunc(req without URL) = %v, %v", u, err) + } +} + +func TestProxyFuncUsesRequestHost(t *testing.T) { + sel, err := New("http://p:3128", []string{"ghe.corp.example"}, nil, "config") + if err != nil { + t.Fatal(err) + } + req, _ := http.NewRequest(http.MethodGet, "https://ghe.corp.example/api/v3", nil) + if u, _ := sel.ProxyFunc()(req); u == nil || u.Host != "p:3128" { + t.Fatalf("ProxyFunc(ghe) = %v", u) + } + req, _ = http.NewRequest(http.MethodGet, "https://api.github.com/", nil) + if u, _ := sel.ProxyFunc()(req); u != nil { + t.Fatalf("ProxyFunc(api.github.com) = %v, want direct", u) + } +} + +func TestSetDefaultAndDefault(t *testing.T) { + t.Cleanup(func() { SetDefault(nil) }) + if Default() != nil { + t.Fatal("Default() must start nil") + } + sel, _ := New("http://p:3128", nil, nil, "env") + SetDefault(sel) + if Default() != sel { + t.Fatal("Default() did not return the installed selector") + } + SetDefault(nil) + if Default() != nil { + t.Fatal("SetDefault(nil) must clear") + } +} + +func TestDialErrorPaths(t *testing.T) { + ctx := context.Background() + dial := (&net.Dialer{Timeout: time.Second}).DialContext + if _, err := Dial(ctx, dial, nil, "ghe.corp.example:443"); err == nil { + t.Fatal("nil proxy URL accepted") + } + u, _ := url.Parse("http://127.0.0.1:1") + if _, err := Dial(ctx, dial, u, "ghe.corp.example"); err == nil || !strings.Contains(err.Error(), "host:port") { + t.Fatalf("target without port: %v", err) + } + refusing := func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("boom") } + if _, err := Dial(ctx, refusing, u, "ghe.corp.example:443"); err == nil || !strings.Contains(err.Error(), "dial: boom") { + t.Fatalf("proxy dial failure: %v", err) + } + + // A proxy that answers garbage instead of an HTTP status line. + ln, _ := net.Listen("tcp", "127.0.0.1:0") + defer ln.Close() + go func() { + c, err := ln.Accept() + if err != nil { + return + } + _, _ = io.WriteString(c, "not http\r\n\r\n") + _ = c.Close() + }() + gu, _ := url.Parse("http://" + ln.Addr().String()) + if _, err := Dial(ctx, dial, gu, "ghe.corp.example:443"); err == nil || !strings.Contains(err.Error(), "read CONNECT response") { + t.Fatalf("garbage response: %v", err) + } +} + +// Bytes the proxy sends right after its 200 must not be lost. +func TestDialPreservesBytesAfterConnectResponse(t *testing.T) { + ln, _ := net.Listen("tcp", "127.0.0.1:0") + defer ln.Close() + go func() { + c, err := ln.Accept() + if err != nil { + return + } + br := bufio.NewReader(c) + if _, err := http.ReadRequest(br); err != nil { + return + } + _, _ = io.WriteString(c, "HTTP/1.1 200 Connection established\r\n\r\nearly-bytes") + _ = c.Close() + }() + u, _ := url.Parse("http://" + ln.Addr().String()) + conn, err := Dial(context.Background(), (&net.Dialer{Timeout: time.Second}).DialContext, u, "ghe.corp.example:443") + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if _, ok := conn.(*bufferedConn); !ok { + t.Fatalf("expected bufferedConn, got %T", conn) + } + b, _ := io.ReadAll(conn) + if string(b) != "early-bytes" { + t.Fatalf("got %q", b) + } +} + +// An https:// proxy URL wraps the proxy leg in TLS before CONNECT: against +// a proxy presenting an untrusted certificate the failure is a TLS error on +// the proxy leg, not a plaintext CONNECT reaching the handler. +func TestDialHTTPSProxyLegIsTLS(t *testing.T) { + var handled atomic.Int32 + tlsProxy := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handled.Add(1) + })) + defer tlsProxy.Close() + + u, _ := url.Parse(tlsProxy.URL) + dial := (&net.Dialer{Timeout: time.Second}).DialContext + _, err := Dial(context.Background(), dial, u, "ghe.corp.example:443") + if err == nil || !strings.Contains(err.Error(), "tls") { + t.Fatalf("https proxy with untrusted CA: err = %v, want tls error", err) + } + if handled.Load() != 0 { + t.Fatal("CONNECT must not reach the proxy handler when the TLS leg fails") + } +} diff --git a/src/deploy/README.md b/src/deploy/README.md index 98b8c4d738..c6705aa0d6 100644 --- a/src/deploy/README.md +++ b/src/deploy/README.md @@ -19,6 +19,7 @@ Use it indirectly through the dashboard terminal link. If a terminal pane says i - `entrypoint.sh` — container startup, config layering, proxy/agent setup, and long-lived process supervision. - `k8s/` — the Kustomize base (`kustomization.yaml`) with namespace, deployment, service, PVC, Secret, ConfigMap and RBAC manifests, plus the custom error backend (`error-pages.yaml`), the hub backup CronJob (`backup-cronjob.yaml`) and the node-prep DaemonSet (`node-prep/`), which are applied separately. The OpenShift Route lives in `kustomize/overlays/openshift/`. - `inference/` — sample in-cluster OpenAI-compatible inference deployment and RBAC. +- `openshell.md` — cluster-side notes for the opt-in `sandbox.runtime: openshell` (ADR-0026): prerequisites, the upstream Helm install, the gateway settings and PVC labels hive's runtime depends on, and how to verify. No manifests are vendored. - `docker-compose.architect.yaml`, `hive-quickstart.yaml`, `hive-level*.yaml`, `architect-only.yaml`, `hive.yaml` — example deployment/configuration manifests. - `blue-green-deploy.sh`, `bootstrap-lxc.sh`, `create-lxc.sh` — operational scripts for **Docker Compose** deployments (LXC bootstrap and a blue-green Compose upgrade). Not "non-Kubernetes" generally: none of the three shells out to `podman` or reads `HIVE_DEPLOY_RUNTIME`, so none of them applies to the Quadlet stack in `quadlet/` below. See [Deployment helper scripts](../docs/deployment-scripts.md) for each script's scope and where a Podman operator should go instead. - `quadlet/` — Podman Quadlet units for the standalone deployment (ADR-0017). `hive.container` and `hive-data.volume` start the Hive service under `systemd`, rootful or rootless; `hive.network` and `hive-gateway.container` put the authenticating gateway in front of it with the same published-port split as the Compose stack — 3001 published, 7681 never. See [the operator guide](../docs/podman-standalone-quadlet.md). Docker Compose remains the default runtime and is unaffected. diff --git a/src/deploy/inference/vllm-deployment.yaml b/src/deploy/inference/vllm-deployment.yaml index b4de56f980..37456df78a 100644 --- a/src/deploy/inference/vllm-deployment.yaml +++ b/src/deploy/inference/vllm-deployment.yaml @@ -36,7 +36,7 @@ spec: spec: initContainers: - name: model-download - image: curlimages/curl:8.13.0@sha256:d43bdb28bae0be0998f3be83199bfb2b81e0a30b034b6d7586ce7e05de34c3fd + image: docker.io/curlimages/curl:8.13.0@sha256:d43bdb28bae0be0998f3be83199bfb2b81e0a30b034b6d7586ce7e05de34c3fd command: - /bin/sh - -c diff --git a/src/deploy/openshell.md b/src/deploy/openshell.md new file mode 100644 index 0000000000..7653c1ad9f --- /dev/null +++ b/src/deploy/openshell.md @@ -0,0 +1,101 @@ +# Installing the OpenShell gateway for `sandbox.runtime: openshell` + +Cluster-side notes for the opt-in OpenShell sandbox runtime +([ADR-0026](../docs/adr/0026-sandbox-runtime-selection.md), epic +[#17642](https://github.com/hivecommons/hive/issues/17642); operator view in +[sandbox-isolation.md](../docs/sandbox-isolation.md#running-sandboxed-kicks-in-an-openshell-sandbox)). +Hive installs nothing: the gateway is a cluster-admin install, once per cluster, +and hive only talks to it. This page points at the upstream install and lists +the settings hive's runtime depends on. Upstream is the source of truth for the +install itself — [NVIDIA/OpenShell → docs/kubernetes/setup](https://github.com/NVIDIA/OpenShell/blob/main/docs/kubernetes/setup.mdx) +and the [gateway configuration reference](https://github.com/NVIDIA/OpenShell/blob/main/docs/how-it-works/gateways/configuration.mdx) — +and no manifests are vendored here; pin the chart version you install and +record it alongside the hive config. + +## Prerequisites to check before installing + +| Requirement | Why | How to check | +|---|---|---| +| Node kernel **≥ 6.2** with Landlock ABI ≥ 3 | OpenShell's sandbox qualification fails below it | `kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kernelVersion}'`. Of the clusters hive runs on, LKE (Debian 13, 6.12) qualifies; OKE UEK7 (5.15) and RHCOS 9.x (5.14) do not. | +| A CNI that **enforces** egress `NetworkPolicy` (Calico, Cilium) | the gateway proxies egress; the pod must not be able to go around it. Flannel accepts policies and ignores them | create a deny-all policy in a scratch namespace and confirm a connection is actually refused, not just that the object exists | +| Cluster-admin for the install | CRDs, the gateway's RBAC, PKI secrets | — | +| A **ReadWriteMany** claim for hive's data when sandboxes may land on other nodes | the workspace is shared by mounting the same PVC | same caveat as the Job runtime | + +## Install + +```shell +helm upgrade --install openshell \ + oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell --create-namespace \ + --set server.drivers.kubernetes.allowDriverConfig=true \ + --set-string 'server.env.OPENSHELL_TELEMETRY_ENABLED=false' +``` + +Then, for the workspace hive will use (hive's `agent_sandbox.openshell.namespace`, +default `default`), install the workspace chart with driver config allowed so +the gateway may read PVC metadata for admission: + +```shell +helm upgrade --install openshell-workspace \ + oci://ghcr.io/nvidia/openshell/openshell-workspace \ + --version \ + --namespace \ + --set gateway.allowDriverConfig=true +``` + +Settings hive's runtime depends on, in gateway terms: + +- **`allow_driver_config = true`** under `[openshell.drivers.kubernetes]` (Helm: + `server.drivers.kubernetes.allowDriverConfig`). Hive passes the workspace + mount as caller `driver_config`; with this off every kick fails at create. +- **Resource admission labels** on hive's data claim. The gateway only mounts a + caller-selected PVC that carries, by default: + + ```yaml + metadata: + labels: + openshell.ai/sandbox-attachable: "true" + openshell.ai/sandbox-attachable-workspace: "" + ``` + + `kubectl label pvc hive-data openshell.ai/sandbox-attachable=true openshell.ai/sandbox-attachable-workspace=default` + on the spoke's namespace is enough for the defaults. +- **`OPENSHELL_TELEMETRY_ENABLED=false`** on the gateway. Hive's threat model + treats sandbox metadata as spoke-private; keep upstream telemetry off. +- **A bearer token** for hive. Create one per the upstream access-control page + and hand it to the hive pod as an environment variable (a Secret-backed + `env` entry on the hive Deployment); name that variable in + `agent_sandbox.openshell.token_env`. The token never goes in `hive.yaml`. +- **Digest-pinned images**: the agent's `sandbox.image` should be + `registry/image@sha256:…`. The gateway's own supervisor and sandbox runtime + images are pinned by the chart version. +- **Workspace mode**: `shared` with `server.sandboxNamespace` set to the + spoke's namespace keeps the sandbox pods, the hive pod and the claim in one + namespace, which is what the shared PVC needs. `operator` mode (one + pre-provisioned namespace per workspace) also works; mount the claim there. + +## Verify + +1. `kubectl -n openshell get pods` — the gateway is `Ready`. +2. Point hive at it (`gateway_url: https://openshell-gateway.openshell.svc:`; + `insecure_tls: true` only while the chart's self-signed PKI is in use and + the CA is not yet trusted by the hive image) and restart or reload; the + boot log must show no `AgentSandboxGateWarnings` line about `gateway_url` + or `workspace_claim`. +3. From a debug pod with the claim mounted at `/data`, run the env-gated test: + `OPENSHELL_GATEWAY_URL=… OPENSHELL_WORKSPACE_CLAIM=hive-data OPENSHELL_IMAGE= go test ./crux/sandbox -run TestOpenShellIntegration -v`. + It writes through `/workspace`, reads the file back on the hive side and + logs the round-trip latency. +4. Kick an openshell-runtime agent and read the sandbox log: a `POST` to + `api.github.com` must be **denied** and `git fetch` must succeed. Sandboxes + are labelled `app.kubernetes.io/managed-by=hive-openshell` and are deleted + when the kick ends; one that is still there after a finished kick is a bug. + +## Not covered + +Hive does not preflight any of the above ([#17644](https://github.com/hivecommons/hive/issues/17644)) +and has no Podman-driver path for standalone hosts yet. The hub provisioner +does not install the gateway for hosted spokes; that is a follow-up once the +prototype numbers ([#17643](https://github.com/hivecommons/hive/issues/17643)) +are in. diff --git a/src/docs/adr/0026-sandbox-runtime-selection.md b/src/docs/adr/0026-sandbox-runtime-selection.md index 3ef7e1ba2c..3a6e062fde 100644 --- a/src/docs/adr/0026-sandbox-runtime-selection.md +++ b/src/docs/adr/0026-sandbox-runtime-selection.md @@ -1,6 +1,6 @@ # ADR-0026: Sandbox runtime selection -Status: Proposed +Status: Accepted (phase 1 implemented, runtime opt-in) — #17642, hivecommons/hive#17646 ## Context @@ -25,7 +25,10 @@ own documentation on 2026-10-25: **Not done in this record:** the prototype run on a hive spoke (#14092 step 2). Startup latency, image and boot size, and proxy and attribution behaviour are **not measured** here. [Prototype protocol](#prototype-protocol) lists what to -measure. This record stays `Proposed` until those numbers exist. +measure; #17643 tracks running it. The runtime shipped as opt-in before those +numbers existed (see [Implementation notes](#implementation-notes-phase-1)): +nothing changes for an operator who does not select it, and the measurement +needs the code to exist. ### The contract a runtime must satisfy @@ -130,7 +133,7 @@ the agent. ## Decision -**Proposed:** adopt **OpenShell** as a third `sandbox.runtime` value +**Accepted:** adopt **OpenShell** as a third `sandbox.runtime` value (`openshell`), behind the existing `sandbox.Launcher` interface. Use the Kubernetes driver on hosted spokes and the Podman driver on standalone hosts. The existing `podman` and `job` runtimes stay as they are, and the default @@ -190,6 +193,31 @@ moves to `Accepted`: | Attribution | The refused write appears in runtime logs with the sandbox and kick name | | Broker path intact | The kick's commits arrive as a PR through `pushbroker`, unchanged | +## Implementation notes (phase 1) + +Phase 1 landed in hivecommons/hive#17646 (epic #17642) as +`sandbox.runtime: openshell`, `src/crux/sandbox/openshell.go`, configured +under `agent_sandbox.openshell` — see +[sandbox-isolation.md](../sandbox-isolation.md#running-sandboxed-kicks-in-an-openshell-sandbox) +for the operator view and [deploy/openshell.md](../../deploy/openshell.md) for +the cluster side. Writing it against the real SDK moved four assumptions in +this record: + +| This record assumed | What the SDK (v0.1.8) offers | What phase 1 does | +|---|---|---| +| Go SDK at `github.com/NVIDIA/OpenShell/sdk/go` | that path is deprecated upstream in favour of `github.com/ai-openshell/sdk-go` | pins `github.com/ai-openshell/sdk-go v0.1.8` (tagged). +3 modules per go.mod (`sdk-go`, `x/oauth2`, `x/sync`), grpc 1.83.2 → 1.85.0, no CGO, ~3.6 MB on the `hive` binary | +| Workspace copied in and out through the SDK | `Files().Upload` is single-file and the default transport is optional on the gateway | the hive data PVC is mounted at a `sub_path` through the gateway's Kubernetes `driver_config`, exactly as the `job` runtime shares it. Needs `allow_driver_config = true` on the gateway and the claim labelled `openshell.ai/sandbox-attachable` | +| `policy_profile` selects a gateway-side policy | no create-time policy-profile API; OpenShell "profiles" are credential-provider profiles | the policy is derived by hive (`OpenShellNetworkPolicy`); `policy_profile` is recorded on the sandbox as the annotation `hive.dev/policy-profile` for gateway admission to act on | +| A static egress allowlist to translate | none exists: ADR-0002 redirects all `:443` to the proxy and decides per request | rules for `github.com` (read-write, git fetch-pack) and `api.github.com` (read-only), plus `agent_sandbox.openshell.allow_hosts`; attached to an overridable default binary set because OpenShell allows nothing for an unlisted binary | + +The preconditions above are still the operator's to verify; phase 1 refuses a +kick when `gateway_url` or `workspace_claim` is missing and warns at boot, but +cannot see the kernel, the CNI or the gateway settings from inside the hive +pod (#17644 adds a gateway preflight). The kernel survey for the clusters hive +runs on, read on 2026-11-10: LKE (`lke648397`, `hive-ci`) run 6.12 on Debian 13 +with Calico and qualify; OKE UEK7 (5.15), and the RHCOS 9.x OpenShift clusters +(5.14) do not. + ## Consequences - Hive gains a sandbox boundary that is checked before the agent runs, without diff --git a/src/docs/adr/README.md b/src/docs/adr/README.md index fd9143592e..8c5addf4f4 100644 --- a/src/docs/adr/README.md +++ b/src/docs/adr/README.md @@ -83,7 +83,7 @@ sequencing; ADR-0019/0025 give their current action/mode contract first. - [ADR-0023: Hive may merge PRs whose author could merge them](0023-trusted-author-automerge.md) - [ADR-0024: Provider credential lifecycle](0024-provider-credential-lifecycle.md) — **Proposed**, not a credential-pool or refresh-service setup guide. - [ADR-0025: One stage engine derives and writes every `stage/*` label](0025-central-stage-engine.md) -- [ADR-0026: Sandbox runtime selection](0026-sandbox-runtime-selection.md) (Proposed) +- [ADR-0026: Sandbox runtime selection](0026-sandbox-runtime-selection.md) (Accepted — phase 1 implemented, runtime opt-in) ### v6-only ADRs (`0100`+) diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index aa50320e49..3db53d93a8 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -33,17 +33,17 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)); also `hub_build` (the hub's short SHA) and, once a stdio `hive-admin-mcp` client has called `/api/version`, `/api/status` or `/api/status/summary` with its `hive-admin-mcp/+` User-Agent, `last_seen_stdio_build` and `last_seen_stdio_at` (RFC3339) so the dashboard can flag build skew ([#12318](https://github.com/hivecommons/hive/issues/12318)) | `pkg/dashboard/api.go:46` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:47` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:77` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1827` | -| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET`, signature mismatch, undecodable payload — and never changes the overall verdict | `pkg/dashboard/server.go:1828` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1829` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1835` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); every your-turn issue and PR row carries the critical path (#13646): `waits_on[]` (`{kind,ref}`, kind one of `pr`, `issue`, `external`, `base-check` (ref = the check red on the base branch), `approval`, `merge`; ref `#N` in the same repository or `owner/repo#N`), `unblocks` (transitive count of your-turn items waiting on it, cycle-guarded), `may_conflict_with[]` (PR numbers of the same repository sharing a closing issue or a known changed file; never an edge), `action` (`merge`, `release-hold`, `rebase`, `rerun` or `approve`), `action_ref`, `root_cause` and `blocker` (the root cause it collapses under); `repos[].yourTurn` has `issues`, `prs` (your turn, `CLOSE` excluded), `root_issues`, `root_prs`, `root_cause_count`, `order[]` (`{kind,repo,number}` by unblocks then age), `groups[]` (`{blocker,blocker_title,dependents[]}`) and `main_red` (`{check,since,fix,fix_your_turn}`); the top-level `yourTurn` sums the counts | `pkg/dashboard/server.go:1840` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers, with the hive-wide `yourTurn` root-cause counts the phone widget reads (#13646); honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1841` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1844` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1845` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1846` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1847` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1848` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1833` | +| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET`, signature mismatch, undecodable payload — and never changes the overall verdict | `pkg/dashboard/server.go:1834` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1835` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1841` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); every your-turn issue and PR row carries the critical path (#13646): `waits_on[]` (`{kind,ref}`, kind one of `pr`, `issue`, `external`, `base-check` (ref = the check red on the base branch), `approval`, `merge`; ref `#N` in the same repository or `owner/repo#N`), `unblocks` (transitive count of your-turn items waiting on it, cycle-guarded), `may_conflict_with[]` (PR numbers of the same repository sharing a closing issue or a known changed file; never an edge), `action` (`merge`, `release-hold`, `rebase`, `rerun` or `approve`), `action_ref`, `root_cause` and `blocker` (the root cause it collapses under); `repos[].yourTurn` has `issues`, `prs` (your turn, `CLOSE` excluded), `root_issues`, `root_prs`, `root_cause_count`, `order[]` (`{kind,repo,number}` by unblocks then age), `groups[]` (`{blocker,blocker_title,dependents[]}`) and `main_red` (`{check,since,fix,fix_your_turn}`); the top-level `yourTurn` sums the counts | `pkg/dashboard/server.go:1846` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers, with the hive-wide `yourTurn` root-cause counts the phone widget reads (#13646); honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1847` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1850` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1851` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1852` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1853` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1854` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | @@ -86,7 +86,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:48` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1922` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1928` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:91` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:92` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:93` | @@ -114,9 +114,9 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1849` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1850` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1851` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1855` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1856` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1857` | ## Configuration @@ -199,7 +199,7 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:392` | | `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:393` | | `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:394` | -| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:572` | +| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:573` | | `GET` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Get | `pkg/dashboard/api.go:233` | | `PUT` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Set | `pkg/dashboard/api.go:234` | @@ -208,8 +208,8 @@ History point limits, timestamp cutoffs, incremental cost reads and ETag/304 beh | `PUT` | `/api/config/governor/watchdog` | Owner only | Governor Watchdog | `pkg/dashboard/api.go:243` | | `GET` | `/api/config/escalation` | Owner only | Escalation Config Get | `pkg/dashboard/api.go:246` | | `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:247` | -| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:568` | -| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:569` | +| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:569` | +| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:570` | | `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:254` | | `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:257` | | `PUT` | `/api/config/governor/features` | Owner only | Governor Features; `writebackKinds` replaces the whole `writeback.kinds` map (a kind sent as `null` or `{}` is dropped). Wrapped by the admin MCP `governor.feature_settings` and `writeback.kind_policy` operations | `pkg/dashboard/api.go:263` | @@ -320,7 +320,7 @@ Read the result from `GET /api/kick/{agent}/status`, which returns `status` of ` | `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/` label; `{"held": false}` removes exactly the labels causing the hold (`hive-pause/` and/or generic hold labels) and never removes `hive/` provenance. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-.json` with audit-backed, provenance-only, and ambiguous legacy `hive/` classifications. | `pkg/dashboard/api.go:367`, `pkg/dashboard/api_repo_hold.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458"}`. On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision ` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR only (409 otherwise). The close variants post a comment naming the operator (and the superseding PR when `superseded_by` is given) and close the item as not planned. Never merges. Every decision writes an audit-log entry (`repo_item_decision_*`). | `pkg/dashboard/api.go:368`, `pkg/dashboard/api_repo_item_decision.go` | | `POST` / `GET` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated` | Same as `…/hold` (owner or repo write), signed-in GitHub user; `decisions.delegate.enabled` (403 when off) | *Let hive decide*: enqueue a delegated-decision job in which the hive decides the parked item itself as the owner's delegate under `pkg/delegatedecision.PolicyText` and records it as a human decision (`/hive decision` comment with the `` marker, park labels off, `triage/accepted`, queue approval on a PR head). Body `{"type": "issue"|"pr", "reason", "caption", "sentinel_reason"}`. `POST` answers 202 with the job (`{id, status: "running"|"done"|"failed", decision: {verdict, rule, text, rationale, actions, keeps_park}, notes, error}`); 409 with the running job while one is in flight; `GET` reads it, from memory while this process holds the job and otherwise from the operator-action ledger (`from_ledger: true`; status, requester, start, verdict and rule, with the outcome's words in `notes` or `error`), so the last result outlives the 30-minute in-memory eviction and a restart. A job still running when the hive restarts is never resumed: the next process marks its ledger record `failed` (*interrupted by a hive restart before it finished, and not resumed …*) with a `repo_item_decision_delegated_failed` audit entry, and the operator asks again. Rails: never merges, refuses items wearing `hold`, never grants secrets/permissions (`needs-owner` + `hold` instead), recommends only on hive policy/governance questions (keeps 085). Audited as `repo_item_decision_delegated[_failed]`. See [dashboard-sections.md](dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:329`, `pkg/dashboard/api_repo_item_decide_delegated.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:591`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:592`, `pkg/dashboard/api_repo_item_accept_triage.go` | | `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:383`, `pkg/dashboard/api_acmm_eval.go` | | `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:384` | | `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](acmm-advisor.md) | `pkg/dashboard/api.go:385` | @@ -656,7 +656,7 @@ always resolved server-side from the validated token. | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:389` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:390` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:391` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1856` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1862` | ## Hub SaaS 102 routes — see [api-reference/hub-saas.md](api-reference/hub-saas.md). @@ -680,34 +680,34 @@ always resolved server-side from the validated token. | `GET` | `/api/hub/clusters` | Hub auth | List Clusters | `pkg/hub/saas.go:531` | | `GET` | `/api/hub/image-pulls` | Hub auth | Per-Release Image Pull Series | `pkg/hub/saas.go:377` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1790` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1791` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1725` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1726` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1727` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1728` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1729` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1737` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1738` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1739` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1740` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1799` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1800` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1731` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1732` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1733` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1734` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1735` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1746` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1747` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1748` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1749` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1748` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1758` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1759` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1760` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1761` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1762` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1763` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1764` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1765` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1768` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1771` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1772` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1779` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1780` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1781` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1798` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1803` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1805` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1757` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1767` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1768` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1769` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1770` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1771` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1772` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1773` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1774` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1777` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1780` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1781` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1788` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1789` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1790` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1807` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1812` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1814` | diff --git a/src/docs/api-reference/agents-and-controls.md b/src/docs/api-reference/agents-and-controls.md index b62508d26a..1a4a401e8d 100644 --- a/src/docs/api-reference/agents-and-controls.md +++ b/src/docs/api-reference/agents-and-controls.md @@ -37,6 +37,13 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:340` | | `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:341` | +| `GET` | `/api/benchmarks/export` | Owner only | Anonymized per-PR [benchmark records](../benchmarks.md) (schema `hivecommons/benchmarks` `result.schema.json`) built from the evidence bundles updated since `since` (RFC3339 or `YYYY-MM-DD`; default 90 days; `400` when malformed) plus `hive_hash`, `publish`, `generated_at` and `count`. `records` is empty when `benchmarks.publish: false`; repo/org names appear only for GitHub-public repos when `benchmarks.repo_consent` is on | `pkg/dashboard/api.go:566` | +| `GET` | `/api/agents/{name}/log` | Dashboard auth/session | Agent Full Log | `pkg/dashboard/api.go:139` | +| `GET` | `/api/agents/{name}/terminal-urls` | Dashboard auth/session | Agent Terminal URLs | `pkg/dashboard/api.go:143` | +| `GET` | `/api/agents/{name}/kicks` | Dashboard auth/session | Agent Kick Log List | `pkg/dashboard/api.go:146` | +| `GET` | `/api/agents/{name}/kicks/{id}` | Dashboard auth/session | Agent Kick Log Get | `pkg/dashboard/api.go:147` | +| `GET` | `/agents/{name}/kicks` | Dashboard auth/session | Agent Kick History Page (HTML) | `pkg/dashboard/api.go:148` | +| `POST` | `/api/agents/{name}/login-code` | Owner only | Agent Login Code | `pkg/dashboard/api.go:170` | ### Kick is asynchronous diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 549ee37564..200a9bf176 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1849` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1850` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1851` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1855` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1856` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1857` | diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index 889a4091e0..b691d42ac6 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -85,7 +85,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:392` | | `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:393` | | `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:394` | -| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:572` | +| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:573` | | `GET` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Get | `pkg/dashboard/api.go:233` | | `PUT` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Set | `pkg/dashboard/api.go:234` | @@ -94,8 +94,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/governor/watchdog` | Owner only | Governor Watchdog | `pkg/dashboard/api.go:243` | | `GET` | `/api/config/escalation` | Owner only | Escalation Config Get | `pkg/dashboard/api.go:246` | | `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:247` | -| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:568` | -| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:569` | +| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:569` | +| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:570` | | `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:254` | | `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:257` | | `PUT` | `/api/config/governor/features` | Owner only | Governor Features | `pkg/dashboard/api.go:263` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 9d9adda891..26ea1bb477 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -51,7 +51,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `GET` | `/api/upstream-watch` | Owner only | Upstream-watch divergence: per watched fork the upstream followed, the watermark and last-run time, surfaced/ported/dismissed/skipped counts, and the recent upstream refs with their upstream URL, diff URL, fork issue number and state. Read-only — it reads the config and the watch state file, never GitHub. Also carries `interval`, `project_repos` and each repo's configured entry (`config`) for the editor. | `pkg/dashboard/api.go:324` | | `PUT` | `/api/config/upstream-watch` | Owner only | Upstream Watch Config Update: edit `upstream_watch` from the Features tab. Body `{enabled?, interval?, repos?: {: {upstream?, sources?, pr_labels?, max_issues_per_run?, label?} \| null}}` — absent keys are unchanged, `null` removes a repo, keys are stored as the bare repo name. Validated with the loader's rules before the config changes (400 on failure), then persisted and audited as `config_upstream_watch`. | `pkg/dashboard/api.go:325` | -| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:582` | +| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:583` | | `GET` | `/api/widget` | Dashboard auth/session | Widget | `pkg/dashboard/api.go:131` | | `GET` | `/api/pane/{agent}` | Dashboard auth/session | Pane | `pkg/dashboard/api.go:136` | | `GET` | `/api/role` | Dashboard auth/session | Role | `pkg/dashboard/api.go:150` | @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:389` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:390` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:391` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1856` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1862` | ### Activity observation provenance diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index 5fedb04511..3b086177d8 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:46` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:47` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:77` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1827` | -| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET`, signature mismatch, undecodable payload — and never changes the overall verdict | `pkg/dashboard/server.go:1828` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1829` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1835` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `code_scanning` `{enabled, repos[], notices[]}` is the code scanning sweep's picture, absent while the sweep is off everywhere — per repo `{repo, security_url, checked_at, open_alerts, by_severity{critical,high,medium,low,note}, qualifying, below_threshold, excluded, tracked, tracked_closed, untracked, open_issues, filed, updated, closed, truncated, notice, notice_kind, error}`, where `tracked` is open alerts listed on an open hive issue and `notice_kind` is `missing_permission` (the App lacks Code scanning alerts: Read) or `not_enabled` (see `docs/code-scanning.md`); `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot; `ghRateLimits.agent_caps` `{fleet_cap, fleet_cap_set, window_s, agents: [{agent, cap, source: agent|base_agent|fleet, base_agent, calls}]}` is the GitHub API budget panel's per-agent proxy cap (0 = no cap) and the calls the proxy counted against it in the current sliding hour (`calls` is null for an uncapped agent, which the proxy does not count) | `pkg/dashboard/server.go:1840` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1841` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1844` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1845` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1846` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1847` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1848` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1833` | +| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET`, signature mismatch, undecodable payload — and never changes the overall verdict | `pkg/dashboard/server.go:1834` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1835` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1841` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `code_scanning` `{enabled, repos[], notices[]}` is the code scanning sweep's picture, absent while the sweep is off everywhere — per repo `{repo, security_url, checked_at, open_alerts, by_severity{critical,high,medium,low,note}, qualifying, below_threshold, excluded, tracked, tracked_closed, untracked, open_issues, filed, updated, closed, truncated, notice, notice_kind, error}`, where `tracked` is open alerts listed on an open hive issue and `notice_kind` is `missing_permission` (the App lacks Code scanning alerts: Read) or `not_enabled` (see `docs/code-scanning.md`); `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot; `ghRateLimits.agent_caps` `{fleet_cap, fleet_cap_set, window_s, agents: [{agent, cap, source: agent|base_agent|fleet, base_agent, calls}]}` is the GitHub API budget panel's per-agent proxy cap (0 = no cap) and the calls the proxy counted against it in the current sliding hour (`calls` is null for an uncapped agent, which the proxy does not count) | `pkg/dashboard/server.go:1846` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1847` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1850` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1851` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1852` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1853` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1854` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index a7ca0e57f6..9d134d49f7 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -32,40 +32,47 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/access-denied` | Hub handler-specific | Access Denied | `pkg/hub/saas.go:370` | | `GET` | `/api/hub/clusters` | Hub auth | List Clusters | `pkg/hub/saas.go:531` | | `GET` | `/api/hub/image-pulls` | Hub auth | Per-Release Image Pull Series | `pkg/hub/saas.go:377` | +| `POST` | `/api/benchmarks/ingest` | Per-hive heartbeat bearer | Spoke push target for anonymized [benchmark records](../benchmarks.md): body `{hive_id, records[]}`; the hub recomputes the hive hash from the authenticated id, rejects records whose `source.hive_id` differs (`401` bad bearer, `403` unregistered hive, `400` malformed), merges accepted records by id into `/data/benchmarks//.json` and returns `202 {accepted, rejected, files}` | `pkg/hub/benchmarks.go` | +| `GET` | `/api/hub/benchmarks` | Public | Benchmark publisher status: `last_ingest_at`, `last_publish_at`, `last_publish_files`, `last_error`, `pending_files`, `hive_count`, `record_count` and per-hive-hash `hives{}` counts; the hub commits pending month files hourly to `hivecommons/benchmarks` (`HIVE_BENCHMARKS_REPO`) via its App identity | `pkg/hub/benchmarks.go` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1790` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1791` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1725` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1726` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1727` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1728` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1729` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1737` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1738` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1739` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1740` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1799` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1800` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1731` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1732` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1733` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1734` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1735` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1746` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1747` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1748` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1749` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1748` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1758` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1759` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1760` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1761` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1762` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1763` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1764` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1765` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1768` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1771` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1772` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1779` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1780` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1781` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1798` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1803` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1805` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1757` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1767` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1768` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1769` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1770` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1771` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1772` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1773` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1774` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1777` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1780` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1781` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1788` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1789` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1790` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1807` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1812` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1814` | | `GET` | `/activity` | Signed-in user | Fleet activity page ([#16063](https://github.com/hivecommons/hive/issues/16063)): read-only, data-free shell that polls `/api/hub/activity` every 15 s with `If-None-Match`. No session → `307` to `/login`; blocked user → `403`. Own Content-Security-Policy (scripts and styles from `'self'` only, `script-src-attr`/`style-src-attr` `'none'`), `Cache-Control: no-cache` + content `ETag`. See [Hub Fleet activity page](../hub-fleet-activity.md) | `pkg/hub/activity_page.go` | | `GET` | `/activity/public` | Public | Public-safe Fleet activity page ([#15966](https://github.com/hivecommons/hive/issues/15966)): the same data-free shell as `/activity`, served without a session; on this path the page polls `/api/hub/activity/public` with `credentials: 'omit'`. Source of the `twitch.tv/hivecommonsfleet` broadcast. Same CSP and cache headers as `/activity`. See [Hub Fleet activity page](../hub-fleet-activity.md#public-safe-view) | `pkg/hub/activity_public.go` | | `GET` | `/activity/assets/{name}` | Public | The shared tracker assets the Fleet activity page loads, byte-identical: the row asset (`pkg/webassets/trackerrow`, [#16062](https://github.com/hivecommons/hive/issues/16062)) and the `am.*` URL-view and display-playlist helpers (`pkg/webassets/trackerview`, [#16064](https://github.com/hivecommons/hive/issues/16064)); only names in `trackerrow.Files` or `trackerview.Files`, else `404`. `Cache-Control: no-cache` + content `ETag` | `pkg/hub/activity_page.go` | | `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of at most 1,000 events, none held longer than 24 hours (ageing out is not counted in `events_dropped`; when full, the oldest event of the hive holding the most events is dropped, counted in that hive's `events_dropped`, so one hive cannot evict the others' events). Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/activity/public` | Public | Public-safe activity view ([#15966](https://github.com/hivecommons/hive/issues/15966)): the `/api/hub/activity` shape for anyone, never reading a cookie, token or `Authorization` header (credentials do not change the response). Only hives the registry lists as public on github.com, and only items and events not marked `private`; `totals`, per-hive `truncated` and `events_dropped`, hive owners and per-user grants are omitted. Same filters (`400` on an invalid one). The body is built at most once per 2 s per parsed filter and shared by every anonymous caller, so rows, `generated_at` and the `ETag` change at most every 2 s; query strings that parse to the default view share one entry. Toggling a hive's visibility drops the cache at once. `ETag` + `If-None-Match` → `304`; `Cache-Control: no-cache` | `pkg/hub/activity_public.go` | +| `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:34` | +| `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:35` | +| `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:36` | +| `POST` | `/api/marketplace/refresh` | Hub admin | On-demand ingest of the CNCF corpus (+ `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`). Without `?repo=` starts a background run → `202 {accepted, repos}` (`409` when one is already running); `?repo=owner/name` runs that repository synchronously → `200 {accepted, repo, result}` | `pkg/hub/marketplace.go:37` | +| `GET` | `/api/marketplace/status` | Public | `{entries, byTier, etag, generatedAt, corpusRepos, trackedRepos, failingRepos, refreshing, schedule, lastRefresh, lastCheckedAt}` — last refresh summary and counts by tier | `pkg/hub/marketplace.go:38` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index 0647f6358c..3fb6bea551 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -25,7 +25,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:378`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:379` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:381` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:591`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:592`, `pkg/dashboard/api_repo_item_accept_triage.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — ``, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:369`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | | `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:383`, `pkg/dashboard/api_acmm_eval.go` | | `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:384` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index f076a52e02..dfb00a2edb 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:48` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1922` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1928` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:91` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:92` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:93` | diff --git a/src/docs/assets/cncf-artwork/README.md b/src/docs/assets/cncf-artwork/README.md new file mode 100644 index 0000000000..73df41fad6 --- /dev/null +++ b/src/docs/assets/cncf-artwork/README.md @@ -0,0 +1,13 @@ +# CNCF artwork: stacked Hive Commons logo + +Source: `projects/hive-commons/stacked/` in cncf/artwork at commit `bffd512a8b625f15cf5f8e0427469380b5768e90`. + +Changes: + +- The "Hive Commons" wordmark paths are removed; the sticker disc (including the curved `hivecommons.dev` text) is untouched. +- The canvas is resized to the icon geometry (`viewBox="0 0 512 512"`, `scale(0.5)`), so the sticker fills it. +- PNGs are 512×512 RGBA renders of the new SVGs. + +The layout mirrors `cncf/artwork`. A maintainer still needs to open the matching cncf/artwork PR with these files. + +License: artwork © Hive Commons maintainers, CC-BY-4.0 (as stated in cncf/artwork#685). diff --git a/src/docs/assets/cncf-artwork/hive-commons/stacked/black/hive-commons-stacked-black.png b/src/docs/assets/cncf-artwork/hive-commons/stacked/black/hive-commons-stacked-black.png new file mode 100644 index 0000000000..36996d9f61 Binary files /dev/null and b/src/docs/assets/cncf-artwork/hive-commons/stacked/black/hive-commons-stacked-black.png differ diff --git a/src/docs/assets/cncf-artwork/hive-commons/stacked/black/hive-commons-stacked-black.svg b/src/docs/assets/cncf-artwork/hive-commons/stacked/black/hive-commons-stacked-black.svg new file mode 100644 index 0000000000..7ebca5366d --- /dev/null +++ b/src/docs/assets/cncf-artwork/hive-commons/stacked/black/hive-commons-stacked-black.svg @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/docs/assets/cncf-artwork/hive-commons/stacked/color/hive-commons-stacked-color.png b/src/docs/assets/cncf-artwork/hive-commons/stacked/color/hive-commons-stacked-color.png new file mode 100644 index 0000000000..8142347c38 Binary files /dev/null and b/src/docs/assets/cncf-artwork/hive-commons/stacked/color/hive-commons-stacked-color.png differ diff --git a/src/docs/assets/cncf-artwork/hive-commons/stacked/color/hive-commons-stacked-color.svg b/src/docs/assets/cncf-artwork/hive-commons/stacked/color/hive-commons-stacked-color.svg new file mode 100644 index 0000000000..e32e882fa2 --- /dev/null +++ b/src/docs/assets/cncf-artwork/hive-commons/stacked/color/hive-commons-stacked-color.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/docs/assets/cncf-artwork/hive-commons/stacked/white/hive-commons-stacked-white.png b/src/docs/assets/cncf-artwork/hive-commons/stacked/white/hive-commons-stacked-white.png new file mode 100644 index 0000000000..7de6c2aed0 Binary files /dev/null and b/src/docs/assets/cncf-artwork/hive-commons/stacked/white/hive-commons-stacked-white.png differ diff --git a/src/docs/assets/cncf-artwork/hive-commons/stacked/white/hive-commons-stacked-white.svg b/src/docs/assets/cncf-artwork/hive-commons/stacked/white/hive-commons-stacked-white.svg new file mode 100644 index 0000000000..c9d4ce99b5 --- /dev/null +++ b/src/docs/assets/cncf-artwork/hive-commons/stacked/white/hive-commons-stacked-white.svg @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/docs/benchmarks.md b/src/docs/benchmarks.md new file mode 100644 index 0000000000..be9cf68661 --- /dev/null +++ b/src/docs/benchmarks.md @@ -0,0 +1,101 @@ +# Benchmarks: publishing per-PR agent/model/runtime outcomes + +Every hive already seals one [review evidence bundle](review-evidence.md) per +PR head. Those bundles record which agent and model opened the PR, which +backend (harness) ran it, how the review swarm scored it, whether CI was green +at merge, when it was opened and merged, and whether it was reverted. The +`benchmarks` feature turns that evidence into **anonymized per-PR benchmark +records** and publishes them to the public leaderboard at + (data repo +[`hivecommons/benchmarks`](https://github.com/hivecommons/benchmarks)), so the +fleet can compare models, harnesses and sandbox runtimes on real work rather +than on vendor evals. Design discussion: hivecommons/hive#17694. + +## What is published + +One record per PR per role, conforming to +[`schema/result.schema.json`](https://github.com/hivecommons/benchmarks/blob/main/schema/result.schema.json) +(vendored in `pkg/evidence/benchmarks/testdata/`): + +| Field | Source | +|---|---| +| `source.hive_id` | **Salted hash** of the hive id: `sha256("hivecommons-benchmarks/v1/" + hive_id)[:16]`. The hive id itself never leaves the spoke. | +| `source.hive_version`, `source.cluster` | Build version and `HIVE_CLUSTER_ID`. | +| `agent.name`, `agent.role` (`author` / `reviewer`), `agent.harness` | Bundle `author.agent`, `author.backend`; reviewer records come from the dominant `(backend, model)` among the swarm verdicts. | +| `model.provider`, `model.name` | Bundle `author.model`; the provider is inferred from the model name prefix. | +| `sandbox_runtime` | Stamped on new bundles from the agent's effective crux sandbox (`podman`, `kubernetes-job`, …) or `none` when agents run in-process. Old bundles fall back to the current config. | +| `task.kind` | Stamped from the agent's lane (`docs`, `security-fix`, `ci-repair`, `release`, `dependency-update`, `triage`, `refactor`), else `unknown`. Reviewer records are `pr-review`. | +| `task.acmm_level` | `policy.acmm_level` at review time. | +| `outcome.*` | `pr_opened`, `pr_merged`, `merged_at`, `time_to_merge_s`, `ci_status_at_merge`, `ci_green_first_try`, `review_verdict`, `reverted`, `closed_unmerged`, `human_interventions` (human approvals / label changes). | +| `quality.*` | Swarm verdict counts and confidence. | + +**What never leaves the spoke:** the hive id, repository and organization +names (unless the repository is public — see below), PR numbers, commit SHAs, +agent prompts, diff content, reviewer comments, human logins, and cost/token +figures (not present in bundles today). + +## Privacy model: default on, anonymized + +| `hive.yaml` key | Default | Effect | +|---|---|---| +| `benchmarks.publish` | `true` | Publish anonymized records (hashed hive id, no repo/org). Set `false` to **opt out** entirely: `/api/benchmarks/export` returns zero records and the hourly push to the hub stops. | +| `benchmarks.repo_consent` | `true` | Attach `task.repo` / `source.org` **only for repositories GitHub reports as public** (visibility is checked through the App client and cached for 24h; unknown ⇒ treated as private). Public-repo activity is already public, so this adds no new disclosure. Set `false` to anonymize even public repos. Private repos are never named regardless of this setting. | + +Both toggles are in **Settings → Features → Benchmarks** in the dashboard, +which also shows this hive's hash so an operator can find their row in the +leaderboard's per-hive table. + +## Cadence: hub-driven, hourly + +``` +spoke (/data/evidence) --hourly POST /api/benchmarks/ingest--> hub (/data/benchmarks//.json) + | + +--hourly Contents-API commit--> hivecommons/benchmarks data/fleet//.json + | + +--GitHub Pages rebuild on push--> hivecommons.dev/benchmarks +``` + +1. **Spoke → hub (push, hourly).** Three minutes after start and then every + hour, the spoke converts bundles updated since its persisted watermark + (`/data/benchmarks-publish.json`), caps a batch at 2000 records and POSTs + them to `/api/benchmarks/ingest` with the same per-hive heartbeat + bearer used for NPS and task-status pushes. The hub recomputes the hash + from the authenticated hive id and rejects any record claiming a different + `source.hive_id`, so one spoke cannot write into another's rows. Push was + chosen over hub-pull because the hub holds no credential for spokes and + most spoke dashboards are cluster-private. +2. **Hub → GitHub (hourly).** The hub merges records by id into month files + under `/data/benchmarks/`, marks changed files pending, and once an hour + (ten minutes after start, at most 60 files per run) writes them to + `hivecommons/benchmarks` at `data/fleet//.json` through the + Contents API using the hub's GitHub App identity (the installation needs + `contents: write` on the data repo). Identical content is skipped. +3. **Pages** rebuilds `data/index.json` on every push, so the site lags the + spoke by at most ~2 hours. + +Until the hub publisher is rolled out, the data repo's +`scripts/backfill-from-cluster.sh` pulls `/data/evidence` straight from spoke +pods and a nightly workflow rebuilds the index. + +## Endpoints + +| Endpoint | Where | Auth | Purpose | +|---|---|---|---| +| `GET /api/benchmarks/export?since=` | spoke | owner | Schema-valid records for bundles updated since `since` (default 90 days). Empty when `benchmarks.publish: false`. | +| `GET /api/status` → `benchmarks{}` | spoke | owner | `hive_hash`, `publish`, `repo_consent`, `last_push_at`, `last_push_count`, `total_pushed`, `last_error`, `watermark_at`, `site_url`. | +| `POST /api/benchmarks/ingest` | hub | per-hive heartbeat bearer | Spoke push target. | +| `GET /api/hub/benchmarks` | hub | public status | Last ingest/publish time, pending files, per-hive-hash record counts. | + +## Hub environment + +| Variable | Default | Purpose | +|---|---|---| +| `HIVE_BENCHMARKS_REPO` | `hivecommons/benchmarks` | Target data repo (`owner/name`). | +| `HIVE_BENCHMARKS_PUBLISH_DISABLED` | unset | Any value stops the GitHub publisher; ingest still stores records locally. | + +## Code map + +- `pkg/evidence/benchmarks` — record types, `HiveHash`, `Convert` (bundles → records), schema golden test. +- `pkg/dashboard/api_benchmarks.go` — export endpoint, status block, hourly pusher, Settings panel data. +- `pkg/hub/benchmarks.go` — ingest, month-file store, hourly GitHub publisher, status endpoint. +- `pkg/evidence` `Author.runtime` / `Author.task_kind` — stamped at bundle write via `github.EvidenceAgentContext`. diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index c98e896da6..d5e7bd53e5 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -155,3 +155,654 @@ result at the next integer version under `name` (default: the repository slug). `cmd/workflow-survey` now compiles every corpus repository's lifecycle model (`lifecycle.json` alongside `model.json`) and reports pass/refusal counts by diagnostic code. + +## Workflows tab (RFC phase 3b) + +Settings → **Workflows** is the spoke's home for models (`static-src/js/config-workflows-library.js`, +`css/workflows.css`). It has two sub-views and a detail page, all built from `GET /api/workflows` +and `GET /api/workflows/{name}/{version}`: + +- **Library** — one row per `name@version`: provenance badge (`built-in` for the virtual + hive-default and any read-only entry, else `meta.provenance`), compile badge (`compiles`, + `compiles · N warnings`, `does not compile`, with the diagnostics as the tooltip), source, + states / columns, and the repositories bound to it. Built-in rows sort first. Owners get + Duplicate (saves a copy under a new name at version 1, provenance `local`) and Delete; Delete is + withheld while anything is bound (the API returns 409 anyway) and the row says `bound` instead. + Export shows the model JSON in-app (copy to clipboard) for everyone. +- **Detail** — facts, the diagram, diagnostics, citations and the raw model/profile JSON. + `workflowDiagramSvg(model, profile)` is a pure function: the compiled profile's columns are lanes + in display order, each state is a chip carrying its padded stage number, transitions are bezier + edges (same-lane edges bow to the right, backward edges are styled `back`), inferred states are + dashed, the initial state and terminal column are marked. Duplicate and dangling transitions are + dropped rather than drawn. +- **Ingest** — pick a configured repository (or type `owner/repo`), optionally a name (default: the + slug), then Preview or Ingest-and-save; both call `POST /api/workflows/ingest`. The result shows + the compile outcome, files read, pipelines found, and opens the saved entry. + +The tab never binds: binding stays on Settings → Repos so the per-repository decision sits next +to the repository's other policy. The Activity Monitor's model badge deep-links here via +`openWorkflowLibraryEntry(name, version)`. Every HTML builder is a pure function of its inputs so +`workflows_library_ui_test.go` runs them under node without a DOM. Editing is phase 4. + +## Simulate (RFC phase 3c) + +"Paste or pick a real issue/PR and see which state the model puts it in and why, using the same +`Facts` the engine would use" — the operator's confidence check before binding. Nothing is +written; any logged-in role may simulate. + +`POST /api/workflows/{name}/{version}/simulate` takes + +```json +{ "kind": "issue" | "pr", "facts": { ...SimulateFacts... }, "repo": "owner/repo", "number": 123 } +``` + +`facts` is a lowerCamel JSON mirror of `stageengine.Facts` (`SimulateFacts` in +`pkg/dashboard/api_workflows_simulate.go`): `labels`, `open`, `merged`, `draft`, `origin`, +`author{login,kind,selfAuthored,role}`, `park{holds,needsHuman,…}`, `issue{bucket,…}` and +`pr{checks,review,dco,conflicting,changesRequested,humanApprover,lane,…}`. Enumerations use the +engine's name helpers so a response can be pasted back as a request: `author.kind` +hive|clanker|human|unknown, `pr.checks` green|red|pending|unknown, `pr.review` +approved|changes_requested|pending|unavailable|requires_human, `pr.dco` yes|no|unknown +(`stageengine.ParseAuthorKind/ParseChecks/ParseReview/ParseDCO` are the inverses of the name +helpers). `park.holds` left null is derived from the labels with `stagelabels.Holds`. + +The handler loads and compiles the stored model exactly as `GET` does, converts the facts, and +calls `stageengine.Explain(profile, kind, facts)`, which returns the same `Decision` as +`DeriveFromModel(...).Derive` plus a trace: + +```json +{ + "decision": { "state": "accepted", "column": "Accepted", "stage": 20, "label": "stage/020-accepted", + "reason": "rule accepted", "abstain": false, "covered": true, "terminal": false }, + "trace": [ { "id": "requested", "state": "requested", "evaluated": true, "matched": false, + "predicates": [ { "fact": "label", "op": "has", "value": "techdocs/request", "holds": true, "known": true }, + { "fact": "label", "op": "not", "value": "techdocs/accepted", "holds": false, "known": true, "actual": "techdocs/accepted" } ] }, + { "id": "accepted", "state": "accepted", "evaluated": true, "matched": true, "predicates": [ … ] }, + { "id": "pr-review", "state": "review", "kind": "pr", "evaluated": false, "matched": false, "skipped": "kind pr" }, + { "id": "later", "state": "done", "evaluated": false, "matched": false } ], + "facts": { …normalized echo… }, "filled": [], "unknown": [], "notes": [] +} +``` + +Trace semantics: for `derivation: rules` every rule appears in order; rules for the other item +kind are `skipped`; rules are `evaluated` until the first `matched` one, each with its predicates' +`holds`/`known` verdicts and the actual value when it differs; rules after the match are listed +with `evaluated: false`. The uncovered case has no matched rule and the decision carries +`abstain: true, covered: false, reason "workflow: not covered…"`. For `derivation: builtin` +(hive-default and ingested models) the trace is a single `{builtin: true, state, reason}` entry, +because the built-in derivers are not rule lists. `column` is looked up from the profile and +`stage` from the state's `lifecycle.number`; a built-in decision of Stage None (for example a +draft PR) returns `covered: true` with an empty `state` and the deriver's reason. + +Built-in PR derivation reads lane facts the engine's lane observers normally supply. When the +request carries no `pr.lane`, `stageengine.SimulatedLane` synthesizes one from the other facts +(self-authored / trusted-author / contributor lane; reason from merged, closed, draft, conflict, +changes requested, holds, needs-human, red or pending checks, missing review, untrusted role) and +the response says so in `notes`. Give `pr.lane{lane,reason}` explicitly to override it. + +Best-effort fill: `repo` + `number` without `facts` fetches the item through the hive's GitHub +client (`github.Client.WorkflowItemSnapshot`: the issue, the PR when it is one, its mergeable +state and the head commit's check summary) and fills `labels`, `open`, `merged`, `draft`, +`author.login`, `author.role` (from `role/*` labels), `park.holds`, `origin`, and for PRs +`pr.checks`, `pr.dco`, `pr.conflicting`, `pr.headSHA`, `pr.baseRef`. `filled` lists what came +from the forge and `unknown` what stayed at its default — `author.kind`, `pr.review`, +`issue.bucket` and the claim/covering-PR facts need the engine's own observers and are not +fetched; edit them in the form before running. `kind` may be omitted and is taken from the item; +a mismatch is a 400. Without a wired GitHub client the endpoint answers 422 rather than guessing. + +The UI (`renderWorkflowSimulate`, `renderWorkflowSimulateResult` in +`config-workflows-library.js`) lives in a collapsed **Simulate** section of the model detail page: +a `owner/repo#123` picker with Fetch facts, an issue/PR radio, and a facts form generated from the +declarative `WF_SIM_FIELDS` table (so the builder, the form reader and +`workflows_simulate_ui_test.go` share one definition). Run posts the form, shows the decision chip +(state · column · stage · reason or **Not covered**), the trace as an ordered list with the +matched rule highlighted and each predicate as `fact op value → ✓/✗/?`, and +`wfSimMarkDiagram` tags the resulting state's chip in the diagram (`.wf-dg-state.simulated`). + +## Versions, diff, confirm and label regeneration (RFC phase 4, backend) + +Phase 4's definition of done is "the TechDocs model can be changed from the diagram and the change +regenerates labels and commands". The backend half gives the editor the primitives for that; the +diagram UI lands separately on top of it. + +### Structural diff — `workflowmodel.Diff` + +`Diff(a, b *Model) ModelDiff` is pure and deterministic: spec only (metadata never counts), +every list sorted by key, the same pair always yields the same JSON. It compares five keyed +collections plus `spec.meta`: + +| Set | Key | Fields compared | +|---|---|---| +| `columns` | column id | label, hint, states, terminal | +| `states` | state id (regions flattened, with `parent`) | label, owner, type, parent, guardRef, pipelines, origin, column, number, kinds, waiting, replies, confidence | +| `transitions` | `from>to` (`from>to@on` when the pair occurs more than once) | on, guardRef, gateRef, note | +| `labels` | label name | color, description, state | +| `rules` | rule id, else `rules[i]` | kind, state, when (rendered predicates), reason | +| `meta` | — | version, author, description, license, source, provenance, derivation mode | + +Each set is `{added, removed, changed:[{key, fields:[{field, from, to}]}]}`; `summary` is one +line (`+2 states, −1 transition, 3 changed`; `no changes` when `empty`). Lists are rendered as +comma-joined strings so a field change is always a flat string pair. + +### Confirm inferred → local — `workflowmodel.Confirm` + +`Confirm(m, by)` returns a deep copy with every `confidence: inferred` element (states at any +depth, events, gates) marked confirmed, `meta.provenance = local`, `meta.version` cleared so the +store assigns the next integer version, and `metadata.annotations["workflow.hivecommons.dev/confirmed-by"]` +set to the caller. `meta.source` is kept: the model still says where it was ingested from. The +input is never modified; `Inferred(m)` counts what is left to confirm. The store gained +`NextVersion(name)` (largest integer version + 1, never a taken one) for this. + +### API + +| Route | Role | Answer | +|---|---|---| +| `GET /api/workflows/{name}/versions` | any | `{versions: []Entry}` newest save first; hive-default returns its one virtual entry; 404 unknown | +| `GET /api/workflows/{name}/diff?from=V1&to=V2[&fromName=&toName=]` | any | `{from: Entry, to: Entry, diff: ModelDiff}`; 400 when `from`/`to` missing, 404 when either side is unknown. `fromName`/`toName` let a local model be diffed against `hive-default` | +| `POST /api/workflows/{name}/{version}/confirm` | owner | 201 `{entry, confirmed, from}` — the confirmed copy saved as the next version; 409 for hive-default (read-only); audited `workflow_confirm` | +| `GET /api/workflows/{name}/{version}/labels` | any | `{name, version, hash, prefix, labels: []LabelDef{name,color,description,state}, commands: [{state, verbs}], stages: [{state, number, label}]}` — the regeneration preview; 400 with diagnostics for a draft that does not compile | + +`labels` is `Profile.LabelDefs(stagelabels.StageColor)`: the model's `spec.labels.entries` with +the stage colour filled in where an entry names none, the state's label as description where an +entry has none, and descriptions clamped to GitHub's limit. `commands` is `Profile.Replies` (the +human-gate verbs per state) in column order. `stages` is the built-in stage label each numbered +state maps onto today, since the engine still writes the built-in family by stage number. + +### Label regeneration on bind + +`automerge.Engine.ensureStageLabels` provisions the built-in family once per repository per +process. It now also asks `stageengine.Options.WorkflowFor` (the P2 lookup cmd/hive wires from +the bindings and the store) for the profile bound to the repository; for a bound, non-hive-default +profile it creates or recolours that profile's `LabelDefs` in the same pass, after the built-in +family, reusing the one label list call. The memo is per repository *and* profile hash +(`StageTracker.ensuredWorkflow`), so a rebinding, an unbind, or a new version of the bound model +changes the hash and the next pass re-lists and provisions only what is missing. No reload hook is +needed: the dashboard's binding edit mutates the live config, `WorkflowFor` reads it on every call, +and the hash comparison does the rest. An unbound repository (or one bound to hive-default) takes +exactly the pre-P4 path: one list, the built-in family, no model labels. + +### Known gaps (phase 4 backend) + +- Labels are only ever created or recoloured, never deleted: unbinding or removing an entry leaves + the label in the repository (same policy as the built-in family). Removal needs an explicit + operator action, not a sweep side effect. +- The engine derives through the built-in stage table by number; a model's own `labels.entries` + are provisioned but not yet written onto items. Writing model labels is the engine half of + phase 4 and follows the diagram editor. +- A model without a `labels` block regenerates nothing (TechDocs as shipped); the preview still + shows its commands and the built-in stages its numbered states land on. + +## Editor (RFC phase 4, UI) + +The owner's brief was "the editor needs to be a good design tool too": a design surface in the +spirit of a lightweight draw.io for state machines, not a form over the JSON. Settings → Workflows +grows an **Editor** nav item (`config-workflows-editor.js`, `workflows-editor.css`, registered +right after the library module), and the library rows and detail page grow **Edit** (owner) or +**Edit a copy** (hive-default and other built-ins, which are read-only); an ingest result gains +**Open in editor** so an ingested draft can be reviewed on the canvas before it is saved. + +**Layout.** Toolbar · left templates palette · centre SVG canvas · right inspector · bottom +tabbed drawer (Problems · YAML · Diff · Simulate). With no model loaded the Editor shows a start +screen: New blank model, Duplicate hive-default, Open from Library, From Ingest, Marketplace +(disabled placeholder for phase 7). + +**Canvas.** Columns are vertical lanes: `+ Column` adds one, double-click the header renames it, +drag the header horizontally to reorder, drag its right edge to resize (snapped to an 8px grid, +`view.laneWidths`), terminal lanes are shaded and dashed. States are chips inside lanes: drag +within or between lanes (rewrites `columns[].states` and `lifecycle.column`), double-click to +rename the label, Delete/Backspace removes the selection and its transitions (confirmed when more +than one), marquee on empty canvas, shift-click to extend, arrow keys nudge order. Each chip has +a right-edge port; dragging it onto another chip creates a transition with `on: ''`, selects it +and focuses the inspector's `on` field; clicking an edge selects it. Pan with space+drag, middle button or wheel; zoom with +ctrl/⌘+wheel or the toolbar ± / Fit; a minimap appears when the model outgrows the viewport. +**Auto-layout** orders lanes by BFS depth from the initial state (terminal lanes last), drops +unassigned states into the nearest lane by depth, and sorts each lane by stage number then +label. Compile diagnostics appear as badges on chips (error red, warning amber, tooltip text); +inferred states are dashed with a one-click ✓ **Confirm** that sets `confidence: local` locally, +and a toolbar **Confirm all inferred** that, for a stored ingested version, also calls +`POST /api/workflows/{name}/{version}/confirm` after saving. The initial state carries a marker. + +**Edge layer: quiet by default, detail on focus.** A real model (hive-default has 22 states and +~90 transitions) turns a naive one-bezier-per-transition drawing into a knot, so the canvas and +the read-only library diagram share one pure router (`wfRouteEdges` in +`config-workflows-library.js`) and a set of rules: + +- *No labels by default.* Every edge renders its event as a floating pill, but CSS shows it only + while that edge is hovered or selected. A toolbar **Labels** toggle shows labels for the + exceptional events only — the model's most common `on` (e.g. `stage-change`, detected by + `wfEdgeCommonEvent`) is never worth labelling and stays quiet. +- *Orthogonal routing through gutters, bundled per lane pair.* Edges leave the source chip's + right port, run into the gutter between the lanes, drop vertically in a channel reserved for + that (from-lane → to-lane) pair and enter the target's left port with rounded corners + (`wfOrthoPath`). Edges that share a lane pair share one vertical trunk, so the Actionable → + Claimed bundle is one line that fans out at both ends. Same-lane edges hug the lane's right + margin, stacked by distance. Edges that skip lanes run below the lanes in a corridor row; + backward edges between adjacent lanes run down the shared gutter (both ports face it), and + backward edges that skip lanes route around the *outside* above the lane headers, dashed and + muted. `wfEdgeChannels` counts the corridor rows so the layout reserves that space. +- *Focus mode.* Hovering or selecting a state dims every edge not incident to it + (`--wf-edge-dim`), lifts the incident edges into a front layer in the accent colour with + arrowheads (`fill="context-stroke"`), and outlines the neighbouring chips; hovering an edge + highlights it and both end chips and reveals its label. The hover handler + (`wfEdgeFocusHover`) is DOM-only and shared by the editor canvas and the library diagram. +- *Fan-out collapse.* More than four transitions from one state into the same lane collapse to + one trunk with a **+N** badge at the source port; hovering expands the fan, and in the editor + clicking the badge pins it open (`view.fansOpen`). + +Ports, drag-to-connect, arrowheads and the initial/terminal styling are unchanged; every colour, +opacity and size is a token so the style ratchet stays green. + +**Pure core.** Everything that mutates or lays out the model is a pure function over plain JSON +(never mutates its input, deterministic), so the undo stack is a snapshot list and the core is +unit-tested under node (`workflows_editor_ui_test.go`): `wfEdNewModel`, `wfEdNormalize`, +`wfEdAddColumn` / `wfEdMoveColumn` / `wfEdRenameColumn` / `wfEdRemoveColumn(model, id, +moveStatesTo)`, `wfEdAddState` / `wfEdMoveState` / `wfEdRenameState` (rewrites transitions, +initial, label entries, derivation rules, lanes and agent grants) / `wfEdRemoveState`, +`wfEdAddTransition` / `wfEdRemoveTransition`, `wfEdUpdate(model, path, value)` with paths like +`spec.states[id=open].lifecycle.number` (empty value deletes), `wfEdUniqueId` (slug + `-N`), +`wfEdAutoLayout`, `wfEdLayout(model, profile, view)` → lanes/chips/edges/fans geometry (edges via the shared `wfRouteEdges` / `wfOrthoPath` / `wfEdgeChannels` router), +`wfEdLaneAt` / `wfEdSlotAt` for drops, `wfEdLocalProblems` (mirrors the compiler's refusals — +`state-no-exit`, `state-no-column`, `unknown-column`, `unknown-state`, `column-empty`, +`no-terminal-column`, `duplicate-stage-number`, `unknown-event`, `inferred-element` — for +instant feedback before the server answers), `wfEdDiff(a, b)` (the client twin of +`workflowmodel.Diff`: keyed change sets for columns, states, transitions, labels and rules plus +meta field changes and a summary), `wfEdSvg(model, layout, state)` and `wfEdMinimapSvg`. + +**Runtime.** `_wfEd = {model, saved, history[], future[], selection, view{panX,panY,zoom, +laneWidths}, compile{busy,profile,diags}, drawer, sim, yaml, …}`. `wfEdCommit(next)` pushes a +deep-copy snapshot (cap 200), bumps a revision, re-renders only the affected regions and +schedules a live compile: 300 ms after the last change the model is posted to +`POST /api/workflows/compile` and the toolbar pill reads `compiles` / `N warnings` / `does not +compile`. Pointer interactions attach once per render (`wfEdAttach(root)`, idempotent) to the +canvas root; every other control is a delegated `data-action`. Cmd/Ctrl-Z, Shift-Cmd-Z and Cmd-S +work when the editor has focus. Leaving the editor with unsaved changes asks via `hiveConfirm`. + +**Drawers.** *Problems* lists diagnostics (click selects the element). *YAML* is a textarea +bound two-way through the new `POST /api/workflows/yaml` (`{model}` → `{yaml}` and `{yaml}` → +`{model, error?}`; model→YAML refreshes while the tab is visible and the textarea is not focused; +Apply parses back and commits; parse errors show inline). *Diff* shows the client-side +`wfEdDiff` of the working copy against the last save as grouped lists, and, for a stored +version, the stored-versions diff (`GET …/diff?from&to`) against any earlier version. *Simulate* +uses the library's declarative `WF_SIM_FIELDS` form and posts `{model, kind, facts}` to the new +`POST /api/workflows/simulate`, which compiles the unsaved body and runs `stageengine.Explain` +(422 with diagnostics when the model does not compile); the decided state is highlighted on the +canvas (`.wf-ed-sim-hit`, others dimmed) and the reason and trace render below the form. + +**Persistence.** *Save* posts the model with `meta.version: ''`, which `handleWorkflowSave` now +resolves through `store.NextVersion(name)`, then reloads the library and reopens the saved entry +in the editor. *Save as…* prompts a new name (version 1, provenance local). *Revert* restores the +last saved snapshot. hive-default opens read-only with an *Edit a copy* button. + +**Export.** SVG (the canvas serialized with the CSS custom-property values resolved via +`getComputedStyle`, so it is self-contained), PNG (drawn to a canvas and shown in a dialog with +open-in-new-tab — no forced download), model JSON and model YAML, all through +`hiveExportViewer`. + +**Deferred.** Editing events, gates, roles and derivation rules (they round-trip untouched and +are editable through the YAML drawer); nested regions beyond flattening; marketplace (phase 7); +keyboard-only lane reorder; collaborative/locking semantics; server-side layout persistence +(lane widths live in the session). + +## Ingest: intake, triage, roles and branch rules (RFC phase 5) + +`ingest.Lifecycle` now reads the repository's process conventions next to its pipelines and adds +an *intake half* to the draft: how work enters, who triages it, who approves a change and what the +forge requires before merge. `ReadTree` keeps the files below in addition to the CI definitions +(`ingest.IsIntakePath`); the dashboard seam (`WorkflowRepoFiles`) is unchanged. Everything this +half adds is `confidence: inferred` and cites where it came from; the compiler warns once per +element and a human confirms them in the editor exactly as for the pipeline half. Trees with none +of these inputs produce the byte-identical model as before (`TestLifecyclePipelineOnlyUnchanged`). + +### Heuristics + +| Input | Read as | Proposes | +|---|---|---| +| `.github/ISSUE_TEMPLATE/*.{yml,yaml}` (issue forms), `*.md` and `.github/ISSUE_TEMPLATE.md` (front matter) | `name`, `labels` (list or comma string) | One `requested-` state per distinct label set in an **Intake** column after Open; event `issue-opened-with-template:` = `labels has ` (the template's first label that is not a triage label and not already bound); transitions open→requested, requested→triage (or →actionable when no triage label exists) and requested→closed; a `labels.entries` row per template label pointing at the state. | +| `.github/ISSUE_TEMPLATE/config.yml` | `blank_issues_enabled` | Annotation `workflow.hivecommons.dev/blank-issues: disabled` when false (every issue arrives through a template). | +| Labels seen in templates, `.github/settings.yml` (probot `labels:`), `.github/labels.yml` (sequence, `labels:` wrapper or mapping), `.github/labeler.yml` (top-level keys), `CONTRIBUTING.md` (backticked labels, or hyphen/slash labels at word boundaries) | Triage vocabulary: `needs-triage`/`triage/needed`/`status/needs-triage`/…; `triage/accepted`/`triage-accepted`/`accepted`; `triage/declined`/`wontfix`/`invalid`/`not-a-bug`; conventions `good first issue`, `help wanted` | A `triage` state in a **Triage** column after Intake; events `triage-requested` (`labels has `), `triage-accepted` (`labels has `, or `labels lacks ` when there is no accepted label), `triage-declined`; gate `triage-decision` released by `approver` when one exists, else `maintainer`, with reply `/label `; transitions open→triage, triage→actionable (gated), triage→closed; label rows `needs → triage`, `accepted → actionable`, `declined → closed`; convention labels as plain rows. | +| PR template (`.github/PULL_REQUEST_TEMPLATE.md`, `PULL_REQUEST_TEMPLATE.md`, `docs/…`, `.github/PULL_REQUEST_TEMPLATE/*.md`) | `- [ ]` / `* [ ]` / `[x]` items | `checks` entries `pr-template-` of kind `manual`, `source: path:line`. Duplicates by id are dropped. | +| `CODEOWNERS` (root, `.github/`, `docs/`) | Every owner on every rule (users and `org/team`) | Role `approver` of kind `members` (params `members`, `source: path:line,…`). | +| root `OWNERS`, `OWNERS_ALIASES` (Prow) | `approvers`, `reviewers`, `filters[".*"]`, aliases | Roles `approver` and `reviewer`; the review gate's reply is `/approve`. | +| `MAINTAINERS`, `MAINTAINERS.md` (root, `.github/`, `docs/`) | `@handle` and `github.com/handle` spellings (plain list, markdown table or prose; names and e-mail addresses are not identities) | The hive-default permission-based `maintainer` role becomes kind `members` listing those handles (`source: path:line`); when no `approver` role exists, the review gate is released by `maintainer`. | +| `kind/*`, `priority/*`, `type/*`, `area/*`, `severity/*`, `size/*` labels declared in `labels.yml`, `settings.yml`, `labeler.yml`, templates or backticked in `CONTRIBUTING.md` | Classification vocabulary triage applies | Plain `labels.entries` rows (no state), sorted by name, so the triage gate's vocabulary is visible in the editor. | +| Any `approver` role | | Gate `review` released by `approver` (condition `pr.review is approved`) attached to the hive-default edges from CI / Review into the admitted Merge-queue states (`awaiting-merge-admission`, `ready-for-human-merge`, `pre-merge-soak`, `merge-queue`), so the compiler's "gate with no owner role" refusal is satisfied by a real role. | +| Branch rules: `GET /repos/{o}/{r}/branches/{b}/protection`, falling back to `GET /repos/{o}/{r}/rules/branches/{b}` (repository and organisation rulesets); or probot `.github/settings.yml` `branches[].protection` when the API gave nothing | required status checks, required approving reviews, code-owner review, dismiss stale reviews, enforce admins, linear history, signed commits | `checks` entries `check-` of kind `ci`, `required: true`; guard `required-checks` (params `contexts`) on the same merge edges; `review.minApprovals: N`; gate `codeowner-review`; bookkeeping effects on `merged` (`enforce-admins`, `require-linear-history`, `require-signed-commits`, `dismiss-stale-reviews`) with the source in their params. | + +The forge half arrives as a synthetic file at `ingest.BranchRulesPath` (`.hive/branch-rules.json`, +`ingest.BranchRules`): `cmd/hive`'s `workflowRepoFiles` fills it through `Client.FetchBranchRules` +when ingesting by `repo`, so pasted `files` and the survey corpus can carry the same reading. +`ingest.LifecycleOptions` accepts the rules explicitly, overriding the file. + +### Citations + +Every element cites `path:line` for file-derived facts (`Citation.Path`/`Line`; `checks[].source`, +`guards[].source`, `roles[].params.source` and `bookkeeping[].params.source` carry the same +`path:line` string) and `api:` — for example `api:repos/cncf/techdocs/rules/branches/main` +— for facts read from the forge. `meta.source` becomes `issue-templates` (or +`github-actions+issue-templates`) once a template was read. + +### What `cncf/techdocs` proposes + +With its current `main` (no issue templates yet; `.github/settings.yml` declares `needs-triage`, +`triage/accepted`, `triage/declined`, `good first issue`, `help wanted`; the org ruleset requires no +checks or approvals): columns `open · triage · actionable · claimed · pr-in-progress · ci-review · +merge-queue · merged`, the `triage` state citing `.github/settings.yml:129/132/135`, the three +triage events, the `triage-decision` gate released by `maintainer`, and the five label rows. The +intake form on the pending `build/03-intake` branch (`assessment-request.yml`, labels `Docs analysis` ++ `needs-triage`) adds `intake[requested-documentation-assessment-request]` before Triage — the +fixture under `pkg/workflowmodel/ingest/testdata/techdocs` carries both. + +### Limitations + +- Heuristics prefer precision: a template without labels, or whose labels are all triage labels or + already bound to another intake state, yields a `issue-template-no-labels` / `issue-template-merged` + finding rather than a state. Label names are matched from a fixed table; a repository's own + triage word (`status: new`) is not recognised. +- Only the root `OWNERS` is read (nested owners files scope by path, which the model does not + express); CODEOWNERS owners are all `approver`, since the file has no reviewer notion. +- `codeowner-review` is informational: the engine does not observe which files a PR touches. +- The engine still derives through the built-in stage table; nothing lands in Intake or Triage + until the label-driven engine half of phase 4 writes model labels onto items. +- Branch protection needs `administration:read`; without it the rulesets endpoint is used, which + omits legacy protection settings such as `enforce_admins`. + +## Label GC and model commands (RFC phase 4, follow-up) + +Two of the phase-4 backend gaps close here: stale model labels are now reclaimed, and a bound +model's `replies` reach the human gate as commands. + +### Label garbage collection on unbind / rebind + +`ensureStageLabels` already provisions a bound model's `LabelDefs` after the built-in family. It +now also remembers what it provisioned and reconciles on change +(`automerge_stage_labels_gc.go`): + +- **Record.** After each provisioning pass the engine writes one JSON record per repository, + `workflows/provisioned/__.json` under `data.workflows_dir` + (`workflowmodel.ProvisionedLabels{repo, model name@version, hash, labels[]}`, through the + `workflowmodel.ProvisionedStore` interface the workflow store implements). The record is the + hive's memory of which labels *it* created or adopted by recolouring for a model; a label the + repository already had for its own reasons is never in it. +- **Diff.** When the profile hash differs from the record's — a rebind, a new version of the bound + model, or an unbind (lookup returns nil or hive-default) — the engine computes + `previous − current` model labels. +- **Delete, only when safe.** A stale label is deleted only if it is in the record, is not in the + built-in family (`stagelabels.LabelDefsFor` plus the retired spellings in `stagelabels.Renames`), + and no open issue or PR wears it (one `label:"x" is:open` search per label; a failed search keeps + the label and logs it). Anything kept stays in the record so the next pass retries it. Every + deletion is audited as `workflow_label_removed` with the repository, label and the model key it + belonged to. +- **Knob.** `workflow.label_gc` (`WorkflowConfig.LabelGC`, default `true`). `false` keeps every + label and logs what would have gone; the record is still maintained, so turning it back on + reclaims what accumulated. The knob is yaml-only (no Settings control; allowlisted in + `settings_parity_allowlist.txt`). +- **Zero change for unbound repositories.** With no `ProvisionedStore` wired, or for a repository + that was never bound, nothing is remembered and nothing is deleted; the pre-P4 path is unchanged + and its tests are byte-identical. + +### Model replies as human-gate commands + +A model's `replies[state]` lists the verbs a human may answer the gate with. They now flow through +three places, all for a repository bound to a non-default model only: + +- **Lookup.** `Profile.RepliesForNumber(stage)` returns the verbs for the state at a stage number; + `Profile.ReplyForNumber(stage, verb)` resolves one to a `ReplyEffect{Verb, From, To, Label}` — + the transition the model declares with that verb as its `on` event (if any), the target state, + and the label `labels.entries` maps to that state (if any). `NormalizeReply` folds case, inner + whitespace and a leading slash so "Approve Analysis", "approve analysis" and "/approve analysis" + are one verb. +- **Comments.** The `/hive` parser (`parseUnparkCommandFor`) tries the model's verbs first — + multi-word, longest match wins, trailing punctuation trimmed, the rest of the line is the + decision text — then the built-ins. With no verbs it *is* the old parser, so `/hive approve` on + an unbound repository is exactly what it was. On a parked issue the sweep answers a model verb + with `applyIssueModelReply`: the parking labels are cleared as for `/hive approve` + (`approved-direction` added, `needs-human` lifted), the mapped label is applied when there is one + (audited `hive_label_applied` with `reply=`), and one reply comment is posted that begins + with the same `Decision:` sentence the decision history reads, then says what the model did + ("moves the item from `a` to `b`; `label` applied" / "…the model maps no label to that state" / + "is recorded; the model declares no transition for it"). Answered replies are idempotency + watermarks like the built-ins. cmd/hive passes the stage engine's `WorkflowFor` to the un-park + sweep so comments see the same binding the labeller does. +- **Dashboard.** The tracker payload's `workflows.repos[].replies` maps stage number → + verbs (hive-default's block carries none). `stageTrackerHumanGateVerbs` appends them after the + built-in verbs, deduped by label, as `wf:` actions; choosing one prompts for an optional + note and POSTs `{"decision":"model-reply","reply":"","comment":…}` to + `/api/repos/{repo}/items/{n}/decision`. The handler resolves the bound profile + (`workflowProfileForRepo`), refuses unbound repositories and verbs the model does not offer at + the item's state with 409, PRs with 400, and otherwise runs the same `applyIssueModelReply` + path, audited `repo_item_decision_model_reply`. + +### Known gaps (phase 4 follow-up) + +- Model replies act on **parked issues** only. A verb declared for a PR state (TechDocs' + `draft-review: approve / request changes`) is not offered by the pill or honoured in comments; PRs + keep GitHub's native review signal. Un-parked issue states with replies are likewise not + answered — the reply needs a gate to answer. +- The pill offers model verbs only where it already shows a human gate; a model state with replies + but no `needs-human` / needs-decision label on the item shows nothing extra. +- Label GC trusts the provisioned record: labels created by an earlier hive build (before the + record existed) are adopted into the record on the next provisioning pass only while the model + still declares them; labels a model dropped *before* this change shipped are never reclaimed + automatically. + +## Ingest from cluster and from written specs (RFC phase 6) + +Two more ways into a draft model, beside the repository ingest of phase 3a. Both end in the +same place — a `WorkflowModel` with `meta.provenance: ingested-unreviewed` that has been through +`Compile` — and both are owner-only, audited as `workflow_ingest`, and advertised by +`GET /api/workflows/ingest/capabilities` (`{cluster, clusterReason, contexts[], spec: {llm, model}}`) +so the Ingest view can grey out what the spoke cannot do. + +### Tekton from a live cluster — `pkg/workflowmodel/ingest/cluster` + +The hive has no Kubernetes client library; everything that talks to a cluster shells out to +`kubectl`, and this package does the same through a `Runner` seam (`Kubectl{Kubeconfig}` in +production, a table-driven fake in tests). `Fetch` runs +`kubectl [--context C] get -o json (-n NS | --all-namespaces) [-l SELECTOR]` for +`pipelines`, `tasks` and `pipelineruns` in `tekton.dev` and `eventlisteners`, `triggers`, +`triggertemplates` and `triggerbindings` in `triggers.tekton.dev`, keeps the newest +`RunsPerPipeline` runs per pipeline (default 3, max 20), serialises every object to YAML under +`cluster///.yaml` and hands that file set to the **existing** file-based +Tekton ingester. There is one Tekton parser; citations in the resulting model point at those +virtual paths. + +Runs never become lifecycle elements. The last run per pipeline is reported in the response +(`runs[]: {pipeline, namespace, run, status, completed}`), recorded on the model as an +`observed` annotation (`workflow.hivecommons.dev/observed-run/`) and as a +`CodeObservedRun` coverage finding — so a reviewer sees "this pipeline last Succeeded" without +the model claiming anything it did not read from a definition. + +Failure modes map to explicit errors and statuses: no Tekton CRDs → `ErrNoCRD` (422), RBAC +denied → `ErrForbidden` (403), unknown context → `ErrNoContext` (404), no namespace and no +`allNamespaces` → `ErrNamespaceRequired` (400), no kubectl/kubeconfig on the spoke → +`ErrNoRunner` (422), caps exceeded → `ErrTooLarge` (413). Caps: 25 namespaces, 20 runs per +pipeline, 500 objects after run trimming, 8 MiB of YAML, 60 s per kubectl call. + +``` +POST /api/workflows/ingest/cluster {context?, namespace | allNamespaces, selector?, runsPerPipeline?, repository?, save?, name?} +→ {model, compiled, diagnostics, profile, files, pipelines, source: "cluster:/", runs[], namespaces[], entry?} +``` + +### Written spec — `pkg/workflowmodel/ingest/spec` + +Input is markdown or plain text up to 256 KiB — pasted, read from a repository's default branch +through the forge client, or fetched from an `https` URL on a public host (the contribute SSRF +guard applies). `Split` cuts it into heading-delimited sections with line ranges; a section's +reference is `spec:#L-L` and every citation in the output uses one. + +**Stage 1 — deterministic, no LLM.** `Extract` reads numbered and bulleted step lists into +states in order; a small hint table maps words such as *triage*, *intake*, *request*, *review*, +*approve*, *merge*, *publish*, *deploy* onto column hints; *if / when / after / unless* +sentences within a step become transition guards; role nouns (*maintainer*, *reviewer*, *TOC*, +*author*, …) become roles, and a step that names a role becomes a gate released by it. The +skeleton always compiles: `Extract` is the fallback for everything below. + +**Stage 2 — LLM refinement, optional.** When a model gateway is configured (the same reviewer +endpoint the owner-delegate decider and chat grounding use, through a `TurnRunner` with an +8 000-token completion budget) and the caller leaves `useLLM` on, `Ingest` prompts it with the +spec, the section references, the skeleton, a schema derived by reflection from the Go types, +the fact vocabulary and the compiler's refusal rules (`prompt.go`, `schema.go`), asking for one +Model JSON object. The answer is accepted only if it survives, in order: + +1. `json.Unmarshal` with `DisallowUnknownFields` into `workflowmodel.Model`; +2. `Mark` — apiVersion/kind/name forced, `meta.version "1"`, `meta.source spec:`, + `meta.provenance ingested-unreviewed`, **every** state, event and gate set to + `origin: ingested` and `confidence: inferred`, every citation checked against the section + references (unknown ones replaced by a whole-document reference and reported as + `spec-llm-citation-replaced`), `pipelines`/`sources`/`agents` stripped; +3. `Compile`. + +A failure at any step is fed back once (`MaxAttempts = 2`) as diagnostics appended to the +prompt; a second failure falls back to the skeleton and reports `spec-llm-fell-back`. When the +gateway's answer is used, the model carries the annotation +`workflow.hivecommons.dev/llm-assisted: "LLM-assisted (model )"` so the editor can show it. + +The boundary this keeps: **the LLM is consulted only here, only to draft, and only through the +`Completer` seam of this package**. Nothing in derivation, compile, the engine or the other +ingesters imports a gateway. Its output cannot carry `confidence: confirmed`, cannot cite +anything but the spec, cannot add pipelines, and cannot bypass the compiler; the worst it can do +is be ignored. + +``` +POST /api/workflows/ingest/spec {text | repo+path | url, useLLM?, repository?, save?, name?} +→ {model, compiled, diagnostics, profile, source: "spec:", llm: {used, model, attempts, accepted, error?}, sections[], entry?} +``` + +`useLLM` defaults to "a gateway is configured"; asking for it without one is a 422 rather than a +silent deterministic run. + +### Ingest view + +`static-src/js/config-workflows-ingest-sources.js` appends two collapsible panels to the Ingest +view through one guarded hook at the end of `renderWorkflowIngest`: *From a cluster* (context +from the capabilities route's list or a text field, namespace, selector, all-namespaces, the +observed-run list) and *From a written spec* (paste, URL or repository file; a *Use AI +assistance* checkbox rendered only when the spoke reports a gateway; the LLM note and section +list in the result). Both reuse the repository ingest's result block, diagnostics and diagram. + +## Bundles and marketplace (RFC phase 7) + +Phase 7 lets a model leave the hive it was authored on. Three pieces: a signed +**bundle** format (`pkg/workflowmodel/bundle`), an immutable **catalogue** on the +hub (`pkg/hub/marketplace*.go`), and the spoke's **import / publish / browse** +surface (`pkg/dashboard/api_workflows_marketplace.go`, Settings → Workflows → +Marketplace). + +### Bundle — `pkg/workflowmodel/bundle` + +``` +{ apiVersion: "workflow.hivecommons.dev/v1alpha1-bundle", kind: "WorkflowBundle", + model: , profile: , hash: , createdAt, createdBy, tier } +``` + +- `Export(m, by, now)` compiles the model and refuses on compile errors; the hash is + `CanonicalHash(m)` — sha256 of the model's canonical JSON (sorted keys, no + insignificant whitespace, `apiVersion/kind/metadata/spec` only). +- `Encode`/`Decode` are canonical and strict (`DisallowUnknownFields`); the bundle + bytes themselves are deterministic so a store can compare them byte-for-byte. +- `Verify(b)` recomputes the hash **and recompiles the model locally**, comparing the + result structurally with the published profile. A mismatch is + `ErrProfileMismatch` ("published profile does not match local compile"). Nothing + downstream ever trusts a published profile: the receiving side always recompiles. +- Tiers are provenance for the catalogue: `built-in`, `curated`, + `ingested-unreviewed`, `local`, `local-published`. `Export` derives the default tier + from `spec.meta.provenance`. +- `IndexEntry`/`Index` are the shared wire types for the catalogue index (below) so the + hub and spoke agree without the hub importing dashboard code. + +### Hub catalogue — `pkg/hub/marketplace*.go` + +Storage is a directory under the hub data dir: + +``` +/data/marketplace/ + workflows//.bundle.json immutable; refuse overwrite of name@version + index.json derived; rebuilt from the bundles if missing + refresh/__.json last ingest run per repository +``` + +Entries are keyed `name@version` plus hash. Publishing the identical bundle again is a +no-op (`200`); different content under an existing key is `409`. The index carries +`{name, version, hash, tier, description, source, states, columns, publishedAt, +publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}`; +the last three are reserved and empty today. `ETag` is the sha256 of the index bytes. + +Routes (see [hub-server.md](../api-reference/hub-server.md)): + +| Route | Who | Notes | +|---|---|---| +| `GET /api/marketplace/workflows` | public | index; `If-None-Match` → `304` | +| `GET /api/marketplace/workflows/{name}/{version}/bundle` | public | one bundle, `Cache-Control: immutable` | +| `POST /api/marketplace/workflows` | spoke bearer or hub admin | publish; hub re-verifies | +| `POST /api/marketplace/refresh[?repo=]` | hub admin | on-demand ingest | +| `GET /api/marketplace/status` | public | last refresh, counts by tier | + +**Who may publish what.** A spoke authenticates with its heartbeat bearer bound to the +`X-Hive-ID` header (the same per-hive key the heartbeat and NPS lanes verify) and may +assert `local`/`local-published` (normalised to `local-published`) or +`ingested-unreviewed`. `curated` and `built-in` require a hub-admin session with +same-origin CSRF evidence. The hub never stores a bundle it did not itself `Verify`. + +**Refresh.** The corpus is the CNCF survey list, now embedded as +`pkg/workflowmodel/ingest/corpus` (262 projects / 299 repositories; `cmd/workflow-survey` +falls back to it when no `projects.json` is given) plus +`HIVE_HUB_MARKETPLACE_EXTRA_REPOS`. For each repository the hub fetches the default-branch +tarball through its GitHub identity, keeps only pipeline files (`ingest.ReadTree`), and +records `{sourceSHA, filesHash, files, checkedAt}`. Unchanged hash → no-op. Changed → +`ingest.Lifecycle` → `bundle.Export` → publish as `ingested-YYYYMMDD.N`, tier +`ingested-unreviewed`, with the source SHA in an annotation. Curated versions are never +touched; `marketplaceNormaliseIndex` flags a curated entry `newerIngestAvailable` when a +later ingest of the same name exists. The crawl is sequential with a short pause between +repositories and honours `X-RateLimit-Remaining`/`Reset`. It runs nightly at +`HIVE_HUB_MARKETPLACE_REFRESH_SCHEDULE` (HH:MM UTC, default `03:00`, empty disables), on +demand, and once at hub start when the catalogue is empty and a GitHub token is present +(in a goroutine; startup is not blocked). `HIVE_HUB_MARKETPLACE_DISABLED` turns the +routes off (`404`). + +### Spoke — import, publish, browse + +Config (`hive.yaml`): + +```yaml +workflow: + marketplace: + enabled: true # default true; false hides the tab and 404s the routes + url: "" # default: hub.url + refresh_interval: 15m # how long the spoke trusts its cached index before revalidating +``` + +Routes (`pkg/dashboard/api_workflows_marketplace.go`): + +- `GET /api/workflows/{name}/{version}/bundle` — export a stored model as a bundle (any role). +- `POST /api/workflows/import` (owner) — body is a bundle; `Verify` → recompile → `store.Save` + pinned to the bundle's version when free, else the next free version; `201 {entry, + verified:true}`; `422` when verification fails. Audit `workflow_import`. +- `GET /api/marketplace/workflows` — proxy to the hub with `If-None-Match`; response + `{index, stale, etag, fetchedAt, error?}`. Hub unreachable → `502` with the cached index + and `stale:true` so the UI can still render. +- `POST /api/marketplace/workflows/{name}/{version}/import` (owner) — fetch the bundle from + the hub and import as above. Audit `workflow_import`. +- `POST /api/marketplace/workflows/publish {name, version}` (owner) — export and POST to + the hub with the spoke's heartbeat credential; `201 {published, hash, tier}`. Audit + `workflow_publish`. +- `POST /api/marketplace/refresh[?repo=]` (owner) — forwards to the hub; a non-admin hive + surfaces the hub's `403`. Audit `workflow_marketplace_refresh`. + +Imported models are pinned `name@version`; a newer catalogue version is a new library entry +and **bindings never move automatically** — rebinding stays an explicit act on Settings → +Repos. Built-in names cannot be imported over. + +UI: Settings → Workflows → **Marketplace** (`config-workflows-marketplace.js`) lists the +catalogue with tier badges, search, a tier filter, the `newer ingest available` flag, +reserved Adoption/Rating columns (rendered `—`), **Import** (→ verified, compiled, library +entry opened), **Reload**, and for owners **Refresh catalogue**; library rows and the detail +view gain **Publish** (confirmed via `hiveConfirm`). A stale banner shows when the proxy +served cached data. + +### Known gaps (phase 7) + +- `adoption`, `ratings`, `benchmarks` are reserved and empty; nothing collects them yet. +- Hub-side marketplace settings are environment variables, not hub YAML. +- The hub logs publishes/refreshes through its structured logger; it has no audit store. +- Ingest versions are per-day sequence numbers, not semantic; curation (promoting an ingest + to `curated`) is a manual admin publish of a reviewed model. diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 61d2092272..3693b2908a 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -197,6 +197,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Review swarm](review-swarm.md) — the five review perspectives, the verdict collector and its report contract, and the opt-in merge-gate integration and bounded auto-fix cycle for review findings. - [Swarm mode](swarm.md) — focusing the hive on one repository for a bounded window: Discord announcements, theming (`swarm:` config, `HIVE_SWARM_EVENT_NAME` / `HIVE_SWARM_CALL_TO_ARMS`), the `/api/leaderboard/swarm` endpoint and scoring. - [Review evidence bundle](review-evidence.md) — the per-PR evidence bundle schema (v1), canonical hash, Ed25519 signing, offline verification, and retention design. +- [Benchmarks](benchmarks.md) — default-on, anonymized per-PR agent/model/runtime records (opt-out `benchmarks.publish`, public-repo attribution via `benchmarks.repo_consent`), the hourly spoke→hub→hivecommons/benchmarks cadence, and the `/api/benchmarks/export`, `/api/benchmarks/ingest` and `/api/hub/benchmarks` endpoints. - [Evidence reporting](evidence.md) — the guarded control-arm (`control_window: on|off`) evidence report: per-repository outcome comparison, the minimum-denominator rule, and why it is reporting groundwork rather than an enabled experiment. - [Portfolio search](search.md) — the opt-in (`search.enabled`) spoke-local search index over evidence, findings, sentinel hits, beads and facts: every `search.*` config key, `GET /api/search`, the report-only repo health score, and the `hivectl search` / `hivectl repo health` commands. - [SOC 2 control mapping (not a compliance claim)](soc2-control-mapping.md) — maps common auditor change-management questions to bundle fields, with sample policy language and the gaps Hive does not cover. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 31d8a5fff3..e00f395026 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -161,6 +161,7 @@ title is `[selfreport] : `. | `review_evidence_write_failed` | the github-client failed to write a review evidence bundle to persistent storage (`review evidence: could not write bundle`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `selfreport_cleared_comment_failed` | the fleet report publisher failed to post the "condition cleared" comment on a self-report issue (`fleet report: cleared comment failed`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `inference_caller_unidentified` | the inference translator refused a caller whose identity could not be established (`inference request from unidentified caller refused`, `crux/proxy/github_proxy_inference.go`). The request remains rejected with HTTP 401; this is distinct from backend credential failures. Check the per-agent UID map and socket attribution for the calling process; the log carries the remote address, method and path. Log-fed: ≥5 refusals in 10 min, clears after 20 quiet minutes | medium | +| `inference_no_route` | the inference translator identified the calling agent but had no inference route for it (`inference no route`, `crux/proxy/github_proxy_inference.go`); the request is answered with HTTP 502. The log's `reason` attribute separates the two causes: `cleared` (a late request from a lingering pane process after the route was cleared on agent stop) and `never_set` (the agent runs on a backend that does not install a route but still has `ANTHROPIC_BASE_URL` pointed at the translator). Its job is to keep this WARN out of `unclassified_error_repeated`. Log-fed: ≥5 occurrences in 10 min, clears after 20 quiet minutes | medium | | `inference_upstream_error` | a failed or non-2xx response from the configured inference backend (LiteLLM, vLLM/llm-d) on the Anthropic-to-OpenAI translator or MITM reroute path (`inference upstream error` / `inference upstream failed`), or a request the translator could not translate (`inference translate request failed`, logged at ERROR by `logTranslateFailure`), all in `crux/proxy/github_proxy_inference.go`, repeated ≥5 times in 10 min — covers transport failures and statuses other than the dedicated 401/403 auth signal. A request refused because the agent has neither an agent model nor a gateway `default_model` logs `inference model not configured; request refused` at WARN instead and is not counted here (a benign configuration condition; see below). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `proxy_plain_forward_refused` | the hive proxy refused a plain-HTTP forward to a non-globally-routable destination (`proxy: plain HTTP forward refused`, `crux/proxy/github_proxy_connect.go`) — a client ignored its proxy bypass for a private address (e.g. the Docker bridge gateway). Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: fix the client's `NO_PROXY` or target address; the refusal is counted as a proxy violation | low | | `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (`SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller keeps serving the SHA without a message. A throttled 403/429 is treated as client-wide: every SHA is parked until `Retry-After` / `X-RateLimit-Reset` elapses and a single WARN is logged per hour per status (`warnGitHubFetchOnce`, key `sha-msg:throttled:`), so a rate limit or revoked token cannot trip this class. Other statuses (e.g. 404 for a force-pushed-away commit) back off per SHA and are deduped to once per hour per (SHA, status), so one stuck commit logs one line; the class is reported only when ≥5 distinct SHA/status pairs fail within 10 min (many branches whose commits GitHub no longer serves). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | diff --git a/src/docs/github-app-setup.md b/src/docs/github-app-setup.md index 8744330566..d3c4831181 100644 --- a/src/docs/github-app-setup.md +++ b/src/docs/github-app-setup.md @@ -93,16 +93,47 @@ Repository permissions used by the dashboard setup UI are: | Actions | Read and write (Read-only with `governor.ci.auto_approve_workflows: false`) | Inspect workflow runs; the default-on approval sweep approves gated `pull_request` runs for trusted bots, the App and dashboard-admitted heads with the App token ([#16603](https://github.com/hivecommons/hive/issues/16603)). If the sweep is off or refused and GitHub holds trusted Dependabot/App/clanker runs in `action_required`, Hive posts `/ok-to-test`; the repository-local `hive-ok-to-test` workflow uses its own `GITHUB_TOKEN` with Actions write to approve those runs. With Read and write, the contributor lane also approves held runs directly for a PR author the contributor registry admitted at a `require_tiers` tier (`auto_merge.contributor_lane.approve_held_runs_for_registry_trusted`). Optional: without it the hive marks such a PR `hive:unauthorized` and waits for a maintainer to click **Approve and run** ([#14589](https://github.com/hivecommons/hive/issues/14589)). | | Administration | Read-only | Read branch-protection rules before the merge-request watcher lands a PR; Hive refuses closed if it cannot verify the base branch is protected. | | Commit statuses | Read/write | Needed when `governor.labels.automerge` is the default `lgtm` on a repository running Prow's commit-bound `lgtm` (prow-github-actions v3): the queue action sets a `prow/lgtm` success status on the approved head so Prow does not strip the label. Without the write grant the label and approval still land, but every queue action takes the failure branch and posts a PR comment saying the status could not be set. Choose a distinct label such as `hive/auto-merge` if you would rather keep this at Read. | -| Dependabot alerts | Read-only | Required for the sec-check agent's Dependabot scanning (`gh api repos///dependabot/alerts`, [#17426](https://github.com/hivecommons/hive/issues/17426)). Least privilege: only the token minted for an agent with role `sec-check` at ISSUES_AND_PRS or ISSUES_PRS_MERGE ([#17496](https://github.com/hivecommons/hive/issues/17496)) requests `vulnerability_alerts: read`; every other agent's token is unchanged, even though the App itself must declare the permission. Existing installations must re-accept the updated permission once under Settings → Integrations → the app → Review request. Optional: without it the sec-check mint retries without the grant, the Dependabot call returns 403, and the hive still mints tokens. | +| Dependabot alerts | Read-only | Required for the sec-check agent's Dependabot scanning (`gh api repos///dependabot/alerts`, [#17426](https://github.com/hivecommons/hive/issues/17426)). Least privilege: only the token minted for an agent with role `sec-check` at ISSUES_AND_PRS or ISSUES_PRS_MERGE ([#17496](https://github.com/hivecommons/hive/issues/17496)) requests `vulnerability_alerts: read`; every other agent's token is unchanged. The App itself must still declare the permission, and that is a GitHub UI change only the App owner can make — see [Adding a permission to an existing App](#adding-a-permission-to-an-existing-app). Optional: until it is declared and re-accepted, the sec-check mint logs `scoped token mint with vulnerability_alerts:read failed; retrying without it`, mints without the grant, and the Dependabot call returns 403; the hive still mints tokens and every other agent is unaffected. | | Code scanning alerts | Read-only (optional) | Only for the opt-in code scanning sweep (`code_scanning`, off by default): it reads a repository's open alerts to file grouped issues ([code-scanning.md](code-scanning.md)). This is GitHub's `security_events` permission. Without it the sweep shows a "Code Scanning Alerts Not Readable" notice for that repository and retries every six hours; nothing else uses it. | - | Workflows | Read/write | Let trusted-tier agents push branches that modify `.github/workflows/`, and let the contributor lane merge an owner-approved PR that edits `.github/workflows/**` (`auto_merge.contributor_lane.owner_approval_releases_workflow_fence`, [#14664](https://github.com/hivecommons/hive/issues/14664)); without the grant such a merge is refused and the hive marks the PR `hive:unauthorized` (`merge-workflow-path`) and parks it at 090 for a maintainer's **Merge** click. Without it GitHub also rejects any GitHub App-token push touching those files server-side ("refusing to allow a GitHub App to create or update workflow … without workflows permission") no matter what the token requests. Maintainer user credentials are separate and can still push workflow-file changes when their account has the normal repository rights; the Hive App cannot until this grant is added **and re-accepted on each installation** (an existing install must approve the new permission under Settings → Integrations → the app → Review request). See [`hive-open-pr`](hive-open-pr.md#workflow-file-pushes-rejected-by-app-tokens) for the agent delivery procedure and [#6985](https://github.com/hivecommons/hive/issues/6985) for the outstanding permission change. | + Organization permission: | Permission | Level | Why | | --- | --- | --- | | Members | Read-only | Identify contributors and owners where configured. | +### Adding a permission to an existing App + +Hive does not ship an App manifest and never edits the App's permissions for +you: GitHub exposes no API for changing a GitHub App's declared permissions, so +when a Hive release starts requesting a grant the App does not yet declare +(Workflows in [#6985](https://github.com/hivecommons/hive/issues/6985), +Dependabot alerts in [#17426](https://github.com/hivecommons/hive/issues/17426)), +the token mint is refused with `422 The permissions requested are not granted +to this installation`, Hive logs a warning, retries without the grant, and the +feature that needed it degrades (a 403 on that API call) until two people have +clicked through the GitHub UI: + +1. **The App owner** (whoever owns the App under **Developer settings**, e.g. + the organization that created it) opens **Settings → Developer settings → + GitHub Apps → _your app_ → Permissions & events → Repository + permissions**, sets the new permission (for example **Dependabot alerts: + Read-only**), and clicks **Save changes** at the bottom of the page. + Nothing changes for any installation until this is saved. +2. **Each installation owner** then sees **Permission updates requested** on + the installation page — **Settings → Integrations → Applications → + _your app_ → Configure** for a user account, or **Organization settings → + Third-party access → GitHub Apps → _your app_ → Configure** for an + organization — and must click **Review request → Accept new permissions**. + GitHub also emails the installation owner. Until an installation accepts, + that installation keeps minting without the grant; installations that have + accepted get it on their next mint without a hive restart. + +The hive confirms the grant landed when the `scoped token mint with +vulnerability_alerts:read failed` (or `workflows:write failed`) warning stops +appearing in the spoke log on the next mint for that tier. + ### Optional shared-CI recovery The shared-CI rerun sweep (`shared_ci_rerun.enabled`, default on) can inspect diff --git a/src/docs/integrations.md b/src/docs/integrations.md index e476c2feee..f4479b2e3c 100644 --- a/src/docs/integrations.md +++ b/src/docs/integrations.md @@ -93,7 +93,7 @@ per-backend confinement matrix. LogoIntegrationStatusDocs Podman logoRootless Podman agent sandboxOpt-insandbox-isolation.md Kubernetes logoKubernetes Job sandbox runtimeOpt-insandbox-isolation.md -NVIDIA logoNVIDIA OpenShell (evaluation)Planned#14092 +NVIDIA logoNVIDIA OpenShell sandbox runtimeIn progress (opt-in, phase 1)sandbox-isolation.md · #17642 OpenSandboxAAIF OpenSandbox (evaluation)Planned#14092 diff --git a/src/docs/maintainer-commands.md b/src/docs/maintainer-commands.md index fd4d951593..13e841466b 100644 --- a/src/docs/maintainer-commands.md +++ b/src/docs/maintainer-commands.md @@ -364,6 +364,19 @@ assent, Hive posts a hint asking for `/hive approve` or `/hive decision ` and leaves all labels unchanged. This avoids turning casual agreement into a state-changing instruction. +## Model-defined replies + +A repository bound to a [workflow model](design/workflow-models.md) may +declare its own human-gate verbs per state (`replies`). On a parked issue in +such a repository, `/hive ` with one of those verbs — multi-word verbs +are fine, case does not matter — is accepted like `/hive decision`: the +parking labels are cleared, `approved-direction` is added, the label the model +maps to the verb's target state is applied when it declares one, and the reply +comment says what the model did with the verb. Any text after the verb is kept +as the decision. The Activity Monitor's human-gate pill offers the same verbs +after the built-in ones. Repositories without a binding, and pull requests, +behave exactly as described above; the model's verbs are not recognised there. + ## Idempotency markers Hive marks its standing parked-issue notice with diff --git a/src/docs/network-requirements.md b/src/docs/network-requirements.md index 784534ebb5..198496a7ed 100644 --- a/src/docs/network-requirements.md +++ b/src/docs/network-requirements.md @@ -44,6 +44,59 @@ Reverse proxies, Routes, and Ingresses must preserve HTTP upgrade headers and lo | Login provider issuers | Hub with multi-provider login | GitHub OAuth endpoints, plus the OIDC issuer/JWKS hosts for each configured provider (e.g. `accounts.google.com`, `login.microsoftonline.com`, `sso.redhat.com`, or your IBMid/custom issuer). | | OCI Object Storage (`objectstorage..oraclecloud.com`) | Hub running DR backups | Only when the opt-in backup CronJob is deployed; see [backup-restore.md](backup-restore.md). | +## Egress through a corporate forward proxy + +Some environments cannot reach a destination directly — the common case is an +enterprise GitHub instance that is only reachable through a corporate HTTP +forward proxy. Hive can chain selected hosts through such a proxy with +`CONNECT`, on every path that carries TLS egress: + +- the control-plane GitHub clients (App JWT / installation-token minting, REST + and GraphQL calls); +- in-process `git` runs (knowledge sources, ad-hoc worktree provisioning), + expressed as scoped `GIT_CONFIG_*` entries; +- the in-process MITM proxy that all agent `:443` traffic is redirected into. + Agents keep `HTTPS_PROXY=http://127.0.0.1:`; the MITM proxy dials + the upstream proxy for matching hosts and speaks TLS (inspected flows) or + splices bytes (passthrough flows) over the tunnel. Agents are never handed + the corporate proxy URL. + +Configure it in `hive.yaml`: + +```yaml +network: + upstream_proxy: + url: http://proxy.corp.example:3128 # userinfo → Proxy-Authorization: Basic + hosts: ["ghe.corp.example", "*.ghe.corp.example"] + no_proxy: ["10.0.0.0/8"] +``` + +or via environment (`HIVE_UPSTREAM_PROXY_URL`, comma-separated +`HIVE_UPSTREAM_PROXY_HOSTS`, `HIVE_NO_PROXY`), which overrides the file block. +When neither is set, standard `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` apply to +every host. `hosts` patterns: exact host, `*.domain` (subdomains only), +`.domain` (domain and subdomains), IP literal, CIDR, or `*`; empty means every +host not listed in `no_proxy`. Hosts outside the rules keep the direct path. +Limitation: for `git`, only exact-host patterns can be expressed as per-URL +`http..proxy` config; wildcard/suffix/CIDR rules apply to API and MITM +traffic but not to in-process `git` unless `hosts` is empty. + +Hosted hubs set this per cluster in `/data/saas/clusters.json` so new pool +slots on that cluster are provisioned with the env vars above: + +```json +{ + "id": "corp-cluster", + "upstream_proxy": { + "url": "http://proxy.corp.example:3128", + "hosts": ["ghe.corp.example", "*.ghe.corp.example"] + } +} +``` + +Existing hive Deployments are not reconciled; patch them once with +`kubectl -n set env deploy/hive -c hive HIVE_UPSTREAM_PROXY_URL=... HIVE_UPSTREAM_PROXY_HOSTS=...`. + ## Firewall recommendations - Public edge: expose only the dashboard/hub front door (3001 for Docker/hub, or your HTTPS Ingress/Route for Kubernetes). Keep 3002 and 7681 cluster/private unless you have an explicit operator access path. diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index 5d4961fe2c..df3f2fc153 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -290,6 +290,8 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `evidence.enabled` | On by default. | The review relay writes one [review evidence bundle](../review-evidence.md) per PR head to `/data/evidence////.json` when it records a verdict or posts a review; the sentinel sweep adds its findings to the flagged head's bundle; and every merge path adds the head's CI check-run summary, the human approvals and label changes, and the merge event. Owners and mergers read bundles through `GET /api/review/evidence` and `GET /api/review/evidence/list` (see the [API reference](../api-reference.md)). Set `false` to stop writing new bundles; existing ones are left in place. | | `evidence.signing_key_file` | Unset (bundles unsigned). | Path to an Ed25519 private key (32-byte seed or 64-byte key, hex or base64). When set and readable, bundles are signed. Dashboard/API readers require a valid signature under its public half and fail closed when the configured key is unavailable; unsigned mode verifies only the bundle content hash. Writers still record `"signed": false` when the key is empty, absent or unusable and log a warning for an unusable key, so an evidence write failure never blocks review or merge work. Keep the key off shared hosts. | | `evidence.complexity.enabled` | On by default (when `evidence.enabled` is on). | Once a month, for each managed repo, the hive downloads the default-branch tarball through the GitHub App and writes one Go cyclomatic-complexity snapshot to `/data/evidence///complexity/.json` (see [Monthly complexity snapshot](../review-evidence.md#monthly-complexity-snapshot-report-only)). Report-only: it never affects merge eligibility. Cheap by design — one archive download and an AST pass per repo per month, no module downloads and no compile. An existing month is never rewritten; the hive re-checks hourly and retries a failed repo on the next check. The last 24 months are kept per repo. Set `false` to stop writing snapshots; existing ones stay. | +| `benchmarks.publish` | **On by default.** | Hourly, the spoke converts review evidence bundles into anonymized per-PR benchmark records (salted hive hash, agent/harness/model/sandbox runtime/task kind/outcome/review quality — never the hive id, PR numbers, SHAs, prompts, diffs, logins or costs) and pushes them to the hub, which commits them hourly to [hivecommons/benchmarks](https://github.com/hivecommons/benchmarks) for . `GET /api/benchmarks/export?since=` serves the same records to the owner. Set `false` to opt out: export returns nothing and the push stops. See [Benchmarks](../benchmarks.md). | +| `benchmarks.repo_consent` | On by default. | Adds `task.repo` / `source.org` to records **only for repositories GitHub reports as public** (visibility checked through the App client, cached 24h, unknown ⇒ private). Private repos are never named. Set `false` to anonymize public repos too. | | `evidence.complexity.static_analysis` | `false`. | Also run `go vet` and (when installed) `staticcheck` on the snapshot checkout and record finding counts per tool. Off by default because it compiles each repo and downloads its modules, which is not cheap on a spoke with many or large repos. | | `review.all_authors` | Deprecated no-op; accepted and logged once. | Every open PR is reviewed regardless of author — hive agent, human, clanker, dependabot or any other bot — whenever review is on (`require_approval` and `fan_out`); there is no author-class opt-out below that switch ([#12974](https://github.com/hivecommons/hive/issues/12974)). The knob used to be required for human PRs. It never lets an agent push to those PRs (see the next row). Remove it from `hive.yaml`. | | `review.fix_human_prs` | Retained for compatibility; treated as push-disabled. | No longer lets review-fix kicks push commits onto PRs whose login author is not a bot-shaped (`[bot]`) login, even when the PR is hive-attributed ([#13124](https://github.com/hivecommons/hive/issues/13124)); the one person-login exception is a contributor PR the stage reconciler adopted in place (`stage/074-adopted-by-hive`). A `changes_requested` verdict on such a PR stops at the published review, with the proposed fix as a suggestion or patch block in the comment, and the refusal is audited as `review_fix_withheld` with the setting's actual value. When the setting is on, the review-thread reconciler still answers review-bot threads on those PRs, by suggestion only, never by checkout or push ([#12897](https://github.com/hivecommons/hive/issues/12897)). Existing configs may still store `true` for UI/audit continuity, but DCO squash attribution means Hive cannot add a compliant follow-up commit without forging the PR author's sign-off. See [review-swarm.md](../review-swarm.md#who-may-be-pushed-to-all_authors-and-the-retired-fix_human_prs-push). | diff --git a/src/docs/roadmap.md b/src/docs/roadmap.md index 4fc6d22d33..ee318e3db9 100644 --- a/src/docs/roadmap.md +++ b/src/docs/roadmap.md @@ -50,6 +50,9 @@ the historical item dates below are not a fresh audit of every linked issue. | Memory and learning | Direction | Keep reusable knowledge durable, testable, and auditable. | [Knowledge design](design/knowledge-system.md) | | Kubernetes-native sandboxes | Direction | Isolate agent workloads with cluster policy where the complexity is justified. | [Security threat model](security-threat-model.md) | | v6 dashboard-optional operation | Development branch available; stable promotion gated | Let people act through GitHub threads, chat, and escalation surfaces with the same permission checks. | [#7563](https://github.com/hivecommons/hive/issues/7563) | +| v7 Hive for teams | RFC held for discussion | Shared, persistent sessions across every surface; org, project, and repo policy; multi-forge solutions; budgets and audit for more than one human at once. | [#17091](https://github.com/hivecommons/hive/issues/17091) | +| v7 containment guardrails | RFC held for discussion | Deterministic, auditable limits on what an agent may touch, spend, and reach, enforced outside the model. | [#12815](https://github.com/hivecommons/hive/issues/12815) | +| Workflow models | RFC held for discussion | Ingest a GitHub Actions or Tekton pipeline, store and edit it as a model, and drive the stage engine and Activity Monitor columns from it; built-in v5/v6 lifecycle unchanged. | [#17620](https://github.com/hivecommons/hive/issues/17620) | ## Recently completed and available foundations diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 3d40d49b25..f3eaa320c1 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -242,6 +242,54 @@ The relay touches the spool only through an `os.Root` opened at the workspace. A Out of scope: the quality and deps lanes are unchanged, and nothing inside a Job writes to GitHub directly. +## Running sandboxed kicks in an OpenShell sandbox + +`sandbox.runtime: openshell` runs an agent's sandboxed kicks in an [NVIDIA OpenShell](https://github.com/NVIDIA/OpenShell) sandbox instead of a Podman container or a plain Kubernetes Job ([ADR-0026](adr/0026-sandbox-runtime-selection.md), epic [#17642](https://github.com/hivecommons/hive/issues/17642)). OpenShell adds what the other two runtimes lack: the workload runs with no capabilities under Landlock and seccomp mediation, and **egress is default-deny** — every outbound connection goes through the gateway's proxy and is checked against a per-binary policy. Phase 1 is opt-in; nothing changes for a hive that does not select it. + +Everything above about the kick path still holds. Hive prepares the workspace on its own data PVC, writes the prompt, and after the sandbox exits reads the commits back and hands them to the push broker and PR path. Only step 2 changes: the launcher is `sandbox.OpenShellLauncher`, which talks to an OpenShell **gateway** over gRPC (`github.com/ai-openshell/sdk-go` v0.1.8); the gateway owns the sandbox pod. + +```yaml +agent_sandbox: + enabled: true + runtime: openshell + openshell: + gateway_url: https://openshell-gateway.openshell.svc:8443 # host:port or http(s):// + namespace: hive # the OpenShell workspace (their term), default "default" + token_env: OPENSHELL_TOKEN # env var holding the gateway bearer token; never the token itself + workspace_claim: hive-data # the PVC hive's /data lives on + workspace_claim_mount: /data # where hive sees it (default) + allow_hosts: [litellm.example.com, hub.hivecommons.dev] # on top of github.com / api.github.com + timeout_s: 1800 # bounds one sandbox from create to delete +agents: + deps: + sandbox: + enabled: true + image: registry.example/project-dev@sha256:… # digest-pinned + openshell: + node_selector: { kernel: "6.12" } + tolerations: + - { key: sandbox, operator: Exists, effect: NoSchedule } + binaries: [/usr/bin/git, /usr/lib/git-core/git-remote-https, /usr/local/bin/node] +``` + +The gateway, workspace, token, TLS and claim fields are cluster facts and fall back from the per-agent block to the global one; `allow_hosts`, `binaries`, `node_selector`, `tolerations` and `runtime_class_name` are per-agent choices and replace the global values when set. + +**Workspace transport is the shared claim again, not a copy.** The SDK's file transfer is single-file and not available on every gateway, so the OpenShell runtime mounts `workspace_claim` at a `sub_path` pointing at the per-kick workspace, at the same `/workspace` mount the other launchers use, through the gateway's Kubernetes driver config. That needs two things on the cluster side: the gateway setting `allow_driver_config = true`, and the claim labelled `openshell.ai/sandbox-attachable: "true"` and `openshell.ai/sandbox-attachable-workspace: ""`. The same ReadWriteMany caveat as the Job runtime applies when the sandbox lands on another node, and `agent_sandbox.workspace_dir` must live under `workspace_claim_mount`. + +**The egress policy.** Hive derives an OpenShell network policy from the hosts its own proxy brokers for a Podman sandbox: `github.com` read-write (git's fetch-pack is a `POST` even when it only reads) and `api.github.com` **read-only** — a sandbox that somehow obtained a token still could not mutate through the API — plus whatever `allow_hosts` names, as `host` (443) or `host:port`. URLs, userinfo, wildcards and anything but a literal host name are rejected, and a unit-level prover renders the policy for the shipped examples to keep it that way. Everything else is unreachable, including a raw TCP connect to an arbitrary IP. OpenShell matches rules per **real binary path** and allows nothing for an unlisted binary, so the policy is attached to a default set (`git`, `git-remote-https`, `curl`, `node`, `python3`, `crux` at their usual paths) that `binaries` overrides when an image puts them elsewhere; a denied connection shows the path OpenShell identified in the sandbox log. `network_mode: none` drops every rule. `policy_profile` is recorded on the sandbox as the annotation `hive.dev/policy-profile` for gateway admission to act on; the SDK has no create-time profile selection. + +**The credential boundary.** Same rule as the Job runtime: environment variables from the allowlist are copied in with the same credential-name filter, hive's brokered GitHub token never enters the sandbox, and hive pushes and opens the PR from the returned workspace. There is **no** Secret-reference path into the sandbox in phase 1, so model credentials have to arrive through an allowlisted non-credential variable pointing at a routable gateway named in `allow_hosts`. The Cline backend, which needs a provider file or Secret, is refused on this runtime. + +**Lifecycle.** The sandbox is named `hive-kick--` and labelled `app.kubernetes.io/managed-by=hive-openshell`. Hive waits for it to become ready, runs the kick command with the sanitized environment in `/workspace`, collects stdout, stderr and the exit code (capped at 16 MiB), and deletes the sandbox — on success, on failure, and on the kick timeout, in which case the kick ends with exit code `-1` and `context deadline exceeded`, exactly like a timed-out Job. Cleanup runs on a fresh, bounded context when the kick's own has expired, so a finished sandbox is not left behind because the deadline fired during log collection. + +**What hive checks, and what it cannot.** Hive warns at boot and on reload (`config.AgentSandboxGateWarnings`) when an openshell-runtime agent resolves no `gateway_url` or no `workspace_claim`, and refuses the kick with the same message before the agent is marked running. It also preflights each distinct gateway (`Health().Check` and `Config().GetGateway`, read-only, cached for five minutes, bounded to 30 s) and adds a warning when the gateway is unreachable or unhealthy, `allow_driver_config` is not enabled, or telemetry is on. It cannot see the node kernel, the CNI or the claim's labels from inside its pod; those remain the operator prerequisites in [deploy/openshell.md](../deploy/openshell.md). + +**The terminal pane.** An openshell-runtime agent has no tmux session either, so its ▶ terminal opens the same read-only `/terminal/job/` viewer as a job-runtime agent, behind the same handoff and role check ([#17645](https://github.com/hivecommons/hive/issues/17645)). While a kick runs it follows the newest live `hive-kick--` sandbox's gateway log (`GetLogs`, polled at up to the gateway's 2,000-line buffer per read, scrubbed of token-shaped values like the pod log); between kicks, once the launcher has deleted the sandbox, or when the gateway cannot be listed, it shows the last persisted transcript exactly as the job viewer does. The live view is weaker than the job pod log, and the pane says so: the gateway log carries the sandbox's supervisor, policy (OCSF) and tracing events — the network and filesystem decisions made around the kick — not the kick's stdout, because the kick runs as an exec session and [exec output reaches neither the gateway log nor the container log](https://docs.nvidia.com/openshell/observability/accessing-logs). Likewise the exit code the gateway reports on the sandbox is its main process's, not the kick's; the launcher takes the kick's exit code from the exec result and deletes the sandbox at once. So the pane ends by saying the kick has finished and that its output and exit code are in the transcript, which it shows on the next open. The dashboard reaches the gateway with the agent's own `gateway_url`, `namespace`, `token_env` and `insecure_tls`. + +**How to verify.** With a gateway reachable and the claim attachable, run the env-gated integration test from a pod that has the claim mounted: `OPENSHELL_GATEWAY_URL=… OPENSHELL_WORKSPACE_CLAIM=hive-data OPENSHELL_IMAGE= go test ./crux/sandbox -run TestOpenShellIntegration -v`. It writes a file through `/workspace`, reads it back on the hive side, and logs the round-trip latency. Then run a kick and confirm in the sandbox log that a `POST` to `api.github.com` is denied while `git fetch` succeeds — that denial is the whole point of the runtime. + +**Limitations (phase 1).** Kubernetes driver only (no Podman-driver path for standalone hosts yet); no Secret references into the sandbox; `policy_profile` is advisory; kernel and claim-label prerequisites are not preflighted; latency and image size are not yet measured ([#17643](https://github.com/hivecommons/hive/issues/17643)). + ## Health and metrics for sandboxed agents A sandboxed agent (Podman or Job runtime) has no tmux session, no in-pod process and no `pluk` log between kicks: the manager parks it in the `idle` state and each kick runs to completion in a container or Job. The dashboard therefore must not read "not running" as "session down" for these agents, and the token collector cannot find their CLI session files because the CLI's home directory lives in the sandbox, not in the hive pod. diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index a07370362c..bce071a20c 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:5496-5505`, descending `AgeMinutes` within each tier). + (`RankActionableIssues`, `src/pkg/github/client.go:5512-5521`, descending `AgeMinutes` within each tier). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes → refactors/docs → tests). diff --git a/src/go.mod b/src/go.mod index 506391be2a..37c073a859 100644 --- a/src/go.mod +++ b/src/go.mod @@ -34,6 +34,7 @@ require ( ) require ( + github.com/ai-openshell/sdk-go v0.1.8 // indirect github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.3 // indirect @@ -43,6 +44,8 @@ require ( github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect golang.org/x/crypto v0.58.0 // indirect + golang.org/x/oauth2 v0.37.0 // indirect + golang.org/x/sync v0.24.0 // indirect golang.org/x/time v0.16.0 // indirect ) @@ -69,12 +72,10 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opentelemetry.io/otel/log v1.47.0 // indirect - golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.24.0 // indirect ) require ( - cel.dev/expr v0.25.2 // indirect + cel.dev/expr v0.25.3 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -96,7 +97,7 @@ require ( golang.org/x/text v0.43.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect - google.golang.org/grpc v1.83.2 // indirect + google.golang.org/grpc v1.85.0 // indirect google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/src/go.sum b/src/go.sum index e87413ebca..1966517b33 100644 --- a/src/go.sum +++ b/src/go.sum @@ -1,5 +1,7 @@ -cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= -cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/ai-openshell/sdk-go v0.1.8 h1:p5i7VAZxlB/tzWy6KQWkdvX3cLIu8x0AinUx1X6wp+s= +github.com/ai-openshell/sdk-go v0.1.8/go.mod h1:nYYe/HYeEct0KG5YyF2dbLHjUWb7EEA+I1lLi6A8j14= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= @@ -172,8 +174,8 @@ golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo= golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= -golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= -golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= +golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -194,8 +196,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1: google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= -google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= -google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw= +google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/src/go.work.sum b/src/go.work.sum index 0384a48c40..49ad72cafc 100644 --- a/src/go.work.sum +++ b/src/go.work.sum @@ -1,9 +1,11 @@ cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM= cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0/go.mod h1:Yj5vHEz/aAepZGliRJsA6uvHAVAQyEwajq9ORCHPxzM= github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g= github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= @@ -21,6 +23,7 @@ github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwB github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be/go.mod h1:PYEOlng9XcrulfyWpm49jECTPV0LT4q8cO7fLW/xwgk= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= @@ -77,6 +80,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPE github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao= @@ -96,26 +101,48 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= go.etcd.io/gofail v0.2.0 h1:p19drv16FKK345a09a1iubchlw/vmRuksmRzgBIGjcA= go.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o= go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0/go.mod h1:VSme3o2fvSg5bVg0dRzyHaj4Z5EVhG+g2Fde6LKzmQA= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo= -golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= -golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= +golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/perf v0.0.0-20250813145418-2f7363a06fe1 h1:stGRioFgvBd3x8HoGVg9bb41lLTWLjBMFT/dMB7f4mQ= golang.org/x/perf v0.0.0-20250813145418-2f7363a06fe1/go.mod h1:rjfRjhHXb3XNVh/9i5Jr2tXoTd0vOlZN5rzsM8cQE6k= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:q/3oV3jAi5vwelxsVAprMBC8BcM2zmNe+IjRGd+9/ks= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/src/pkg/config/agent_config.go b/src/pkg/config/agent_config.go index 86910b6087..d918e1ee2f 100644 --- a/src/pkg/config/agent_config.go +++ b/src/pkg/config/agent_config.go @@ -4,6 +4,8 @@ import ( "fmt" "sort" "strings" + + cruxconfig "github.com/hivecommons/hive/crux/config" ) func sortedRuntimeNames() []string { @@ -122,7 +124,8 @@ func AgentSandboxGateWarnings(cfg *Config) []string { return nil } - var optedIn, noImage, noClaim, badRuntime []string + var optedIn, noImage, noClaim, noGateway, badRuntime []string + var openshellBlocks []SandboxOpenShellConfig for name, a := range cfg.Agents { if !a.SandboxEnabled(cfg.AgentSandbox) { continue @@ -136,11 +139,23 @@ func AgentSandboxGateWarnings(cfg *Config) []string { badRuntime = append(badRuntime, name+"="+rt) case rt == SandboxRuntimeJob && strings.TrimSpace(a.SandboxJob(cfg.AgentSandbox).WorkspaceClaim) == "": noClaim = append(noClaim, name) + case rt == SandboxRuntimeOpenShell: + os := a.SandboxOpenShell(cfg.AgentSandbox) + if strings.TrimSpace(os.GatewayURL) == "" { + noGateway = append(noGateway, name) + } + if strings.TrimSpace(os.WorkspaceClaim) == "" { + noClaim = append(noClaim, name) + } + if strings.TrimSpace(os.GatewayURL) != "" { + openshellBlocks = append(openshellBlocks, os) + } } } sort.Strings(optedIn) sort.Strings(noImage) sort.Strings(noClaim) + sort.Strings(noGateway) sort.Strings(badRuntime) var out []string @@ -165,15 +180,24 @@ func AgentSandboxGateWarnings(cfg *Config) []string { } if len(noClaim) > 0 { out = append(out, fmt.Sprintf( - "agent(s) %s use the job sandbox runtime but resolve no job.workspace_claim; the Job has no way to see the workspace and every kick will fail — "+ - "set agent_sandbox.job.workspace_claim (or the per-agent sandbox.job.workspace_claim) to the PVC hive's data lives on (#6311)", + "agent(s) %s use the job or openshell sandbox runtime but resolve no workspace_claim; the sandbox has no way to see the workspace and every kick will fail — "+ + "set agent_sandbox.job.workspace_claim / agent_sandbox.openshell.workspace_claim (or the per-agent one) to the PVC hive's data lives on (#6311)", strings.Join(noClaim, ", "))) } + if len(noGateway) > 0 { + out = append(out, fmt.Sprintf( + "agent(s) %s use the openshell sandbox runtime but resolve no openshell.gateway_url; every kick will fail — "+ + "set agent_sandbox.openshell.gateway_url (or the per-agent sandbox.openshell.gateway_url) to the OpenShell gateway's gRPC address (ADR-0026)", + strings.Join(noGateway, ", "))) + } if len(optedIn) < len(cfg.Agents) { out = append(out, fmt.Sprintf( "agent_sandbox.enabled is true but only %d of %d agent(s) are opted in (%s); the rest still run unconfined on the tmux path (#4918)", len(optedIn), len(cfg.Agents), strings.Join(optedIn, ", "))) } + // Last, because it may reach the network (bounded and cached): the static + // warnings above must not wait on a gateway. + out = append(out, cruxconfig.OpenShellPreflightWarnings(openshellBlocks)...) return out } diff --git a/src/pkg/config/agent_sandbox_openshell_test.go b/src/pkg/config/agent_sandbox_openshell_test.go new file mode 100644 index 0000000000..140d422564 --- /dev/null +++ b/src/pkg/config/agent_sandbox_openshell_test.go @@ -0,0 +1,114 @@ +package config + +import ( + "strings" + "testing" + + cruxconfig "github.com/hivecommons/hive/crux/config" + "gopkg.in/yaml.v3" +) + +// sandbox.runtime: openshell (ADR-0026) — parse, merge semantics and the +// boot warnings that keep a misconfigured runtime from failing silently. + +func TestSandboxOpenShellParsesAndMerges(t *testing.T) { + src := ` +agent_sandbox: + enabled: true + image: img + runtime: openshell + openshell: + gateway_url: https://openshell-gateway.openshell.svc:8443 + namespace: hive + token_env: OPENSHELL_TOKEN + workspace_claim: hive-data + allow_hosts: [litellm.example.com] + timeout_s: 600 +agents: + deps: + sandbox: + enabled: true + openshell: + allow_hosts: [hub.example.org] + node_selector: { kernel: "6.12" } +` + var cfg Config + if err := yaml.Unmarshal([]byte(src), &cfg); err != nil { + t.Fatal(err) + } + a := cfg.Agents["deps"] + if got := a.SandboxRuntime(cfg.AgentSandbox); got != SandboxRuntimeOpenShell { + t.Fatalf("runtime = %q", got) + } + eff := a.SandboxOpenShell(cfg.AgentSandbox) + if eff.GatewayURL != "https://openshell-gateway.openshell.svc:8443" || eff.Namespace != "hive" || eff.TokenEnv != "OPENSHELL_TOKEN" || eff.WorkspaceClaim != "hive-data" || eff.TimeoutS != 600 { + t.Errorf("cluster facts must fall back to the global block: %+v", eff) + } + if len(eff.AllowHosts) != 1 || eff.AllowHosts[0] != "hub.example.org" { + t.Errorf("per-agent allow_hosts must replace the global list: %v", eff.AllowHosts) + } + if eff.NodeSelector["kernel"] != "6.12" { + t.Errorf("node_selector = %v", eff.NodeSelector) + } + other := AgentConfig{} + if got := other.SandboxOpenShell(cfg.AgentSandbox); len(got.AllowHosts) != 1 || got.AllowHosts[0] != "litellm.example.com" { + t.Errorf("agent without a block gets the global one: %+v", got) + } +} + +func TestSandboxOpenShellGateWarnings(t *testing.T) { + yes := true + cfg := &Config{ + AgentRuntimeConfig: AgentRuntimeConfig{ + AgentSandbox: AgentSandboxConfig{Enabled: true, Image: "img"}, + Agents: map[string]AgentConfig{ + "deps": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimeOpenShell}}, + }, + }, + } + got := AgentSandboxGateWarnings(cfg) + joined := strings.Join(got, "\n") + if len(got) != 2 || !strings.Contains(joined, "gateway_url") || !strings.Contains(joined, "workspace_claim") { + t.Errorf("warnings = %q, want one for gateway_url and one for workspace_claim", got) + } + cfg.AgentSandbox.OpenShell = &SandboxOpenShellConfig{GatewayURL: "gw:8080", WorkspaceClaim: "hive-data"} + if got := AgentSandboxGateWarnings(cfg); len(got) != 0 { + t.Errorf("fully configured openshell runtime still warns: %q", got) + } +} + +func TestAgentSandboxGateWarningsIncludeOpenShellPreflight(t *testing.T) { + yes := true + cfg := &Config{ + AgentRuntimeConfig: AgentRuntimeConfig{ + AgentSandbox: AgentSandboxConfig{ + Enabled: true, + Image: "img", + OpenShell: &SandboxOpenShellConfig{GatewayURL: "gw:8080", WorkspaceClaim: "hive-data"}, + }, + Agents: map[string]AgentConfig{ + "a": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimeOpenShell}}, + "b": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimeOpenShell}}, + }, + }, + } + var calls int + var gotBlocks int + cruxconfig.SetOpenShellPreflight(func(b []SandboxOpenShellConfig) []string { + calls++ + gotBlocks = len(b) + return []string{"gateway gw:8080 is unreachable"} + }) + defer cruxconfig.SetOpenShellPreflight(nil) + got := AgentSandboxGateWarnings(cfg) + if len(got) != 1 || !strings.Contains(got[0], "unreachable") || calls != 1 || gotBlocks != 2 { + t.Errorf("warnings = %q calls=%d blocks=%d, want the preflight warning from one call over both agents", got, calls, gotBlocks) + } + + cfg.Agents = map[string]AgentConfig{"p": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimePodman}}} + calls = 0 + AgentSandboxGateWarnings(cfg) + if calls != 0 { + t.Errorf("preflight ran with no openshell agent") + } +} diff --git a/src/pkg/config/benchmarks.go b/src/pkg/config/benchmarks.go new file mode 100644 index 0000000000..ec82f5552e --- /dev/null +++ b/src/pkg/config/benchmarks.go @@ -0,0 +1,60 @@ +package config + +import ( + "strings" + + cruxconfig "github.com/hivecommons/hive/crux/config" +) + +// BenchmarksConfig (top-level `benchmarks:`) governs what this hive +// contributes to the fleet leaderboard at https://hivecommons.dev/benchmarks/ +// (RFC hivecommons/hive#17694, docs/benchmarks.md). +// +// Default ON, anonymized: every hive publishes per-PR records keyed by a +// salted hash of its hive id — model, harness, sandbox runtime, outcome, +// time-to-merge and ACMM level; never a login, prompt, finding or code. +// Repository and org names ride along only for PUBLIC repositories, whose +// activity is already public. Both knobs are opt-outs. +type BenchmarksConfig struct { + // Publish false stops the spoke exporting records to the hub (and empties + // GET /api/benchmarks/export). nil means on. + Publish *bool `yaml:"publish,omitempty" json:"publish,omitempty"` + // RepoConsent false anonymizes even public repositories: task.repo and + // source.org are dropped from every record. nil means public repos are + // named. + RepoConsent *bool `yaml:"repo_consent,omitempty" json:"repo_consent,omitempty"` +} + +// PublishEnabled reports whether records leave this hive. +func (b BenchmarksConfig) PublishEnabled() bool { + return b.Publish == nil || *b.Publish +} + +// RepoConsentEnabled reports whether public repositories may be named. +func (b BenchmarksConfig) RepoConsentEnabled() bool { + return b.RepoConsent == nil || *b.RepoConsent +} + +// BenchmarkSandboxRuntimeNone is the runtime published for in-process agents. +const BenchmarkSandboxRuntimeNone = "none" + +// BenchmarkSandboxRuntime is the sandbox runtime an agent's kicks run under +// as the benchmark schema names it: the crux runtime when sandboxing is on +// for that agent, else "none". +func (c *Config) BenchmarkSandboxRuntime(agent string) string { + if c == nil { + return BenchmarkSandboxRuntimeNone + } + var ac *cruxconfig.AgentConfig + if a, ok := c.Agents[agent]; ok { + ac = &a + } + enabled := c.AgentSandbox.Enabled + if ac != nil && ac.Sandbox != nil && ac.Sandbox.Enabled != nil { + enabled = *ac.Sandbox.Enabled + } + if !enabled { + return BenchmarkSandboxRuntimeNone + } + return strings.TrimSpace(ac.SandboxRuntime(c.AgentSandbox)) +} diff --git a/src/pkg/config/benchmarks_test.go b/src/pkg/config/benchmarks_test.go new file mode 100644 index 0000000000..e660c6ae6d --- /dev/null +++ b/src/pkg/config/benchmarks_test.go @@ -0,0 +1,52 @@ +package config + +import ( + "testing" + + cruxconfig "github.com/hivecommons/hive/crux/config" +) + +func TestBenchmarksConfigDefaultsOn(t *testing.T) { + var b BenchmarksConfig + if !b.PublishEnabled() || !b.RepoConsentEnabled() { + t.Fatal("unset benchmarks block must publish and name public repos") + } + off := false + b = BenchmarksConfig{Publish: &off, RepoConsent: &off} + if b.PublishEnabled() || b.RepoConsentEnabled() { + t.Fatal("explicit false must opt out") + } +} + +func TestBenchmarkSandboxRuntime(t *testing.T) { + var nilCfg *Config + if got := nilCfg.BenchmarkSandboxRuntime("a"); got != BenchmarkSandboxRuntimeNone { + t.Fatalf("nil config = %q", got) + } + on, off := true, false + cfg := &Config{} + cfg.AgentSandbox = cruxconfig.AgentSandboxConfig{Enabled: false, Runtime: "kubernetes-job"} + cfg.Agents = map[string]cruxconfig.AgentConfig{ + "opted-in": {Sandbox: &cruxconfig.AgentSandboxOverride{Enabled: &on, Runtime: " openshell "}}, + "inherits": {Sandbox: &cruxconfig.AgentSandboxOverride{Enabled: &on}}, + "opted-out": {Sandbox: &cruxconfig.AgentSandboxOverride{Enabled: &off}}, + } + for agent, want := range map[string]string{ + "opted-in": "openshell", + "inherits": "kubernetes-job", + "opted-out": BenchmarkSandboxRuntimeNone, + "unknown": BenchmarkSandboxRuntimeNone, // global sandbox off + } { + if got := cfg.BenchmarkSandboxRuntime(agent); got != want { + t.Errorf("%s: got %q want %q", agent, got, want) + } + } + cfg.AgentSandbox.Enabled = true + if got := cfg.BenchmarkSandboxRuntime("unknown"); got != "kubernetes-job" { + t.Errorf("global on, unlisted agent: got %q", got) + } + cfg.AgentSandbox.Runtime = "" + if got := cfg.BenchmarkSandboxRuntime("unknown"); got != cruxconfig.SandboxRuntimePodman { + t.Errorf("default runtime: got %q", got) + } +} diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 1d20038166..3fc03d6bac 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -107,6 +107,11 @@ type Config struct { // Debug holds process-level diagnostics knobs (pprof listener). See // docs/operator-reference.md "Runtime diagnostics". Debug DebugConfig `yaml:"debug,omitempty" json:"debug,omitempty"` + // Network holds outbound-connectivity knobs: the optional upstream HTTP + // forward proxy that selected hosts (e.g. an enterprise GitHub reachable + // only via a corporate proxy) are chained through. Env overrides and + // HTTPS_PROXY fallback: see crux/upstreamproxy.Resolve. + Network NetworkConfig `yaml:"network,omitempty" json:"network,omitempty"` // Triggers is an additive list of CEL-based declarative agent triggers. // Default empty → existing label/governor triggering is unchanged. Triggers []TriggerRule `yaml:"triggers,omitempty" json:"triggers,omitempty"` @@ -137,6 +142,9 @@ type Config struct { // Evidence controls the review evidence bundles the review relay writes // (hivecommons/hive#11060). Default on, unsigned; see EvidenceConfig. Evidence EvidenceConfig `yaml:"evidence,omitempty" json:"evidence,omitempty"` + // Benchmarks governs the anonymized per-PR records this hive contributes + // to hivecommons/benchmarks (hivecommons/hive#17694); see BenchmarksConfig. + Benchmarks BenchmarksConfig `yaml:"benchmarks,omitempty" json:"benchmarks,omitempty"` // Compliance selects framework profiles; see ComplianceConfig. Compliance ComplianceConfig `yaml:"compliance,omitempty" json:"compliance,omitempty"` Escalation EscalationConfig `yaml:"escalation,omitempty" json:"escalation,omitempty"` @@ -282,6 +290,22 @@ type Config struct { // WorkflowConfig is the `workflow:` block (hivecommons/hive#17620). type WorkflowConfig struct { Enabled bool `yaml:"enabled" json:"enabled"` + // LabelGC lets the stage engine delete a bound model's labels from a + // repository once the repository is unbound or rebound to a model that + // no longer declares them (RFC phase 4 follow-up). Only labels the hive + // itself provisioned, outside the built-in stage family and worn by no + // open issue or PR, are ever deleted; each deletion is audited as + // workflow_label_removed. nil means true; false leaves every label in + // place and logs what would have gone. + LabelGC *bool `yaml:"label_gc,omitempty" json:"label_gc,omitempty"` + // Marketplace configures the spoke's view of the hub workflow catalogue + // (hivecommons/hive#17620 phase 7). + Marketplace MarketplaceConfig `yaml:"marketplace,omitempty" json:"marketplace,omitempty"` +} + +// LabelGCEnabled is the effective workflow.label_gc (default true). +func (c WorkflowConfig) LabelGCEnabled() bool { + return c.LabelGC == nil || *c.LabelGC } type DataConfig struct { diff --git a/src/pkg/config/crux_aliases.go b/src/pkg/config/crux_aliases.go index 548f274a0f..3898aaa7b2 100644 --- a/src/pkg/config/crux_aliases.go +++ b/src/pkg/config/crux_aliases.go @@ -38,6 +38,7 @@ type ( SandboxJobResources = cruxconfig.SandboxJobResources SandboxJobToleration = cruxconfig.SandboxJobToleration SandboxJobVolume = cruxconfig.SandboxJobVolume + SandboxOpenShellConfig = cruxconfig.SandboxOpenShellConfig StatsDisplayEntry = cruxconfig.StatsDisplayEntry ToolApprovalConfig = cruxconfig.ToolApprovalConfig ToolApprovalRule = cruxconfig.ToolApprovalRule @@ -145,6 +146,7 @@ const ( RotationHeadroomSourceBuiltin = cruxconfig.RotationHeadroomSourceBuiltin RotationHeadroomSourceCCLeft = cruxconfig.RotationHeadroomSourceCCLeft SandboxRuntimeJob = cruxconfig.SandboxRuntimeJob + SandboxRuntimeOpenShell = cruxconfig.SandboxRuntimeOpenShell SandboxRuntimePodman = cruxconfig.SandboxRuntimePodman SelfMergeMinACMMLevel = cruxconfig.SelfMergeMinACMMLevel WritableBobAPIKeyFile = cruxconfig.WritableBobAPIKeyFile diff --git a/src/pkg/config/network_config.go b/src/pkg/config/network_config.go new file mode 100644 index 0000000000..ec4e46d03f --- /dev/null +++ b/src/pkg/config/network_config.go @@ -0,0 +1,32 @@ +package config + +import "github.com/hivecommons/hive/crux/upstreamproxy" + +// NetworkConfig groups outbound-connectivity settings. +type NetworkConfig struct { + UpstreamProxy UpstreamProxyConfig `yaml:"upstream_proxy,omitempty" json:"upstream_proxy,omitempty"` +} + +// UpstreamProxyConfig names a corporate HTTP forward proxy and the hosts +// whose TLS egress (control-plane GitHub clients, in-process git, and every +// agent :443 flow intercepted by the MITM proxy) must be chained through it +// with CONNECT. Env mapping: HIVE_UPSTREAM_PROXY_URL, HIVE_UPSTREAM_PROXY_HOSTS +// (comma-separated), HIVE_NO_PROXY; when none of these nor the file block is +// set, standard HTTPS_PROXY/HTTP_PROXY/NO_PROXY apply to all hosts. +type UpstreamProxyConfig struct { + // URL is the proxy, e.g. http://proxy.corp.example:3128 (userinfo is + // sent as Proxy-Authorization: Basic). + URL string `yaml:"url,omitempty" json:"url,omitempty"` + // Hosts restricts chaining to matching hosts: exact ("ghe.corp.example"), + // wildcard ("*.ghe.corp.example", subdomains only), dotted suffix + // (".corp.example"), IP, CIDR, or "*". Empty means every host not in + // NoProxy. + Hosts []string `yaml:"hosts,omitempty" json:"hosts,omitempty"` + // NoProxy lists hosts (same grammar) that are always dialled directly. + NoProxy []string `yaml:"no_proxy,omitempty" json:"no_proxy,omitempty"` +} + +// UpstreamProxy converts the file block into the selector package's input. +func (c UpstreamProxyConfig) UpstreamProxy() upstreamproxy.Config { + return upstreamproxy.Config{URL: c.URL, Hosts: c.Hosts, NoProxy: c.NoProxy} +} diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index 9758a5abaa..990383aff5 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -688,7 +688,8 @@ "dedupe": {}, "contribute": {}, "workflow": { - "enabled": false + "enabled": false, + "marketplace": {} }, "HiveID": "", "acmm_level": null, @@ -708,6 +709,9 @@ "otel": {}, "Tracing": {}, "debug": {}, + "network": { + "upstream_proxy": {} + }, "Mint": { "Enabled": false, "KeyPath": "", @@ -731,6 +735,7 @@ "evidence": { "complexity": {} }, + "benchmarks": {}, "compliance": { "posture_checks": {} }, diff --git a/src/pkg/config/workflow_marketplace.go b/src/pkg/config/workflow_marketplace.go new file mode 100644 index 0000000000..935bf6665b --- /dev/null +++ b/src/pkg/config/workflow_marketplace.go @@ -0,0 +1,50 @@ +package config + +import ( + "strings" + "time" +) + +// MarketplaceConfig is the `workflow.marketplace:` block. The spoke proxies +// the hub catalogue index, imports bundles from it and publishes library +// entries to it (hivecommons/hive#17620 phase 7). +type MarketplaceConfig struct { + // Enabled defaults to true; set false to hide the Marketplace view and + // return 404 from the proxy routes. + Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // URL is the hub base URL serving /api/marketplace/*. Empty means the + // configured hub.url (which itself defaults to the public hub). + URL string `yaml:"url,omitempty" json:"url,omitempty"` + // RefreshInterval bounds how long the spoke serves a cached index + // before revalidating with the hub (If-None-Match). Zero means the + // default of 15 minutes. + RefreshInterval time.Duration `yaml:"refresh_interval,omitempty" json:"refresh_interval,omitempty"` +} + +// DefaultMarketplaceRefreshInterval is the spoke-side index cache TTL. +const DefaultMarketplaceRefreshInterval = 15 * time.Minute + +// MarketplaceEnabled reports the effective workflow.marketplace.enabled. +func (m MarketplaceConfig) MarketplaceEnabled() bool { + return m.Enabled == nil || *m.Enabled +} + +// EffectiveRefreshInterval returns refresh_interval or its default. +func (m MarketplaceConfig) EffectiveRefreshInterval() time.Duration { + if m.RefreshInterval <= 0 { + return DefaultMarketplaceRefreshInterval + } + return m.RefreshInterval +} + +// MarketplaceURL resolves the hub base URL for marketplace calls: the +// explicit workflow.marketplace.url, else hub.url. +func (c *Config) MarketplaceURL() string { + if c == nil { + return "" + } + if u := strings.TrimSpace(c.Workflow.Marketplace.URL); u != "" { + return strings.TrimRight(u, "/") + } + return strings.TrimRight(strings.TrimSpace(c.Hub.URL), "/") +} diff --git a/src/pkg/config/workflow_marketplace_test.go b/src/pkg/config/workflow_marketplace_test.go new file mode 100644 index 0000000000..c810fb1cff --- /dev/null +++ b/src/pkg/config/workflow_marketplace_test.go @@ -0,0 +1,35 @@ +package config + +import ( + "testing" + "time" +) + +func TestMarketplaceConfigDefaults(t *testing.T) { + var m MarketplaceConfig + if !m.MarketplaceEnabled() { + t.Fatal("marketplace should default to enabled") + } + if m.EffectiveRefreshInterval() != DefaultMarketplaceRefreshInterval { + t.Fatalf("interval = %v", m.EffectiveRefreshInterval()) + } + off := false + m = MarketplaceConfig{Enabled: &off, RefreshInterval: time.Minute} + if m.MarketplaceEnabled() || m.EffectiveRefreshInterval() != time.Minute { + t.Fatalf("explicit values ignored: %+v", m) + } + + var nilCfg *Config + if nilCfg.MarketplaceURL() != "" { + t.Fatal("nil config URL") + } + cfg := &Config{} + cfg.Hub.URL = "https://hub.example/" + if got := cfg.MarketplaceURL(); got != "https://hub.example" { + t.Fatalf("falls back to hub.url, got %q", got) + } + cfg.Workflow.Marketplace.URL = " https://mkt.example// " + if got := cfg.MarketplaceURL(); got != "https://mkt.example" { + t.Fatalf("explicit url, got %q", got) + } +} diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index c58a544502..1ef02621b6 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -563,6 +563,7 @@ func (s *Server) RegisterAPI(deps *Dependencies) { // consulted by any merge decision. s.mux.HandleFunc("GET /api/evidence/summary", s.handleEvidenceSummary) s.mux.HandleFunc("GET /api/repos/{owner}/{repo}/health", s.handleRepoHealth) + s.mux.HandleFunc("GET /api/benchmarks/export", s.handleBenchmarksExport) // write_surface is a top-level Config field; the lane write allowlist // editor lives on the governor Security tab (#9587, api_config_write_surface.go). s.mux.HandleFunc("GET /api/config/write-surface", s.handleWriteSurfaceGet) diff --git a/src/pkg/dashboard/api_benchmarks.go b/src/pkg/dashboard/api_benchmarks.go new file mode 100644 index 0000000000..ae05bddc3c --- /dev/null +++ b/src/pkg/dashboard/api_benchmarks.go @@ -0,0 +1,377 @@ +package dashboard + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" + ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +// Benchmark publication (hivecommons/hive#17694). The spoke converts its +// review evidence bundles into anonymized leaderboard records +// (pkg/benchmarks) and, hourly, pushes the ones newer than the hub's last +// acknowledgement to POST /api/benchmarks/ingest on the hub — the same +// spoke→hub bearer pattern as NPS and task-status, chosen over a hub pull +// because the hub holds no bearer for spokes and most spoke URLs are +// cluster-private. The hub batches and commits to hivecommons/benchmarks. + +const ( + // benchmarksPushInterval is the spoke→hub cadence; the hub publishes + // hourly too, so the public site lags the fleet by ≤ ~2 h. + benchmarksPushInterval = time.Hour + // benchmarksPushInitialDelay lets the evidence walk and the heartbeat + // registration settle before the first push. + benchmarksPushInitialDelay = 3 * time.Minute + // benchmarksPushTimeout bounds one hub POST. + benchmarksPushTimeout = 60 * time.Second + // benchmarksPushMaxRecords caps one batch; the rest go next hour. + benchmarksPushMaxRecords = 2000 + // benchmarksHubIngestPath is the hub route. + benchmarksHubIngestPath = "/api/benchmarks/ingest" + // benchmarksRepoVisibilityTTL bounds how long a repo's public/private + // answer is reused. + benchmarksRepoVisibilityTTL = 24 * time.Hour + // benchmarksExportMaxSinceDays bounds ?since= so one request cannot + // re-emit years of history on a hot path. + benchmarksExportDefaultWindow = 90 * 24 * time.Hour +) + +// benchmarksPublishStatePath persists the last hub acknowledgement so a +// restart does not resend history. A var for tests. +var benchmarksPublishStatePath = filepath.Join(ghpkg.ReviewLinksDir, "benchmarks-publish.json") + +// BenchmarksStatus is the /api/status block and the Settings → Benchmarks +// panel source. +type BenchmarksStatus struct { + HiveHash string `json:"hive_hash"` + Publish bool `json:"publish"` + RepoConsent bool `json:"repo_consent"` + LastPushAt string `json:"last_push_at,omitempty"` + LastPushCount int `json:"last_push_count"` + TotalPushed int `json:"total_pushed"` + LastError string `json:"last_error,omitempty"` + WatermarkAt string `json:"watermark_at,omitempty"` + SiteURL string `json:"site_url"` +} + +// benchmarksSiteURL is where the records end up. +const benchmarksSiteURL = "https://hivecommons.dev/benchmarks/" + +type benchmarksPublishState struct { + // Watermark is the newest bundle UpdatedAt the hub has acknowledged. + Watermark time.Time `json:"watermark"` + LastPushAt time.Time `json:"last_push_at"` + LastCount int `json:"last_count"` + TotalPushed int `json:"total_pushed"` + LastError string `json:"last_error,omitempty"` +} + +type benchmarksPublisher struct { + mu sync.Mutex + state benchmarksPublishState + vis map[string]benchmarksVisibility +} + +type benchmarksVisibility struct { + public bool + at time.Time +} + +func (s *Server) benchmarks() *benchmarksPublisher { + s.benchmarksOnce.Do(func() { + p := &benchmarksPublisher{vis: map[string]benchmarksVisibility{}} + if data, err := os.ReadFile(benchmarksPublishStatePath); err == nil { + _ = json.Unmarshal(data, &p.state) + } + s.benchmarksPub = p + }) + return s.benchmarksPub +} + +func (p *benchmarksPublisher) save() { + data, err := json.Marshal(p.state) + if err != nil { + return + } + _ = os.MkdirAll(filepath.Dir(benchmarksPublishStatePath), 0o755) + _ = os.WriteFile(benchmarksPublishStatePath, data, 0o644) +} + +// repoPublic answers through the hive's GitHub client with a daily cache. +// Unknown (API error, no client) is private: the record stays anonymous. +func (s *Server) benchmarksRepoPublic(ctx context.Context, repo string) bool { + p := s.benchmarks() + now := time.Now() + p.mu.Lock() + if v, ok := p.vis[repo]; ok && now.Sub(v.at) < benchmarksRepoVisibilityTTL { + p.mu.Unlock() + return v.public + } + p.mu.Unlock() + public := false + if s.deps != nil && s.deps.GHClient != nil { + if owner, name, ok := strings.Cut(repo, "/"); ok { + rctx, cancel := context.WithTimeout(ctx, 10*time.Second) + if r, _, err := s.deps.GHClient.GetRepo(rctx, owner, name); err == nil && r != nil { + public = !r.GetPrivate() + } + cancel() + } + } + p.mu.Lock() + p.vis[repo] = benchmarksVisibility{public: public, at: now} + p.mu.Unlock() + return public +} + +// benchmarksOptions builds the converter options from live config. +func (s *Server) benchmarksOptions(ctx context.Context, since time.Time) benchmarks.Options { + cfg := s.deps.Config + release, _ := fleetReportBuildInfo() + return benchmarks.Options{ + HiveID: cfg.HiveID, + HiveVersion: release, + Since: since, + RepoConsent: cfg.Benchmarks.RepoConsentEnabled(), + RepoPublic: func(repo string) bool { return s.benchmarksRepoPublic(ctx, repo) }, + SandboxRuntime: cfg.BenchmarkSandboxRuntime, + } +} + +// benchmarksRecords converts every bundle updated at/after since. +func (s *Server) benchmarksRecords(ctx context.Context, since time.Time) ([]benchmarks.Record, time.Time, error) { + verifier, err := s.reviewEvidenceVerifier() + if err != nil { + return nil, time.Time{}, err + } + bundles, _, err := loadEvidenceBundles(ghpkg.ReviewEvidenceRoot, verifier) + if err != nil { + return nil, time.Time{}, err + } + var newest time.Time + for _, b := range bundles { + if b != nil && b.UpdatedAt.After(newest) { + newest = b.UpdatedAt + } + } + return benchmarks.Convert(bundles, s.benchmarksOptions(ctx, since)), newest, nil +} + +// handleBenchmarksExport serves GET /api/benchmarks/export?since=. +// Owner-authenticated like every /api route. Returns no records when +// benchmarks.publish is false so the endpoint and the hub agree. +func (s *Server) handleBenchmarksExport(w http.ResponseWriter, r *http.Request) { + if s.deps == nil || s.deps.Config == nil { + jsonError(w, "config not loaded", http.StatusServiceUnavailable) + return + } + cfg := s.deps.Config + since := time.Now().Add(-benchmarksExportDefaultWindow) + if raw := strings.TrimSpace(r.URL.Query().Get("since")); raw != "" { + t, err := time.Parse(time.RFC3339, raw) + if err != nil { + t, err = time.Parse("2006-01-02", raw) + } + if err != nil { + jsonError(w, "since must be RFC3339 or YYYY-MM-DD", http.StatusBadRequest) + return + } + since = t + } + resp := map[string]any{ + "hive_hash": benchmarks.HiveHash(cfg.HiveID), + "generated_at": time.Now().UTC().Format(time.RFC3339), + "publish": cfg.Benchmarks.PublishEnabled(), + "since": since.UTC().Format(time.RFC3339), + "records": []benchmarks.Record{}, + "count": 0, + } + if cfg.Benchmarks.PublishEnabled() { + recs, _, err := s.benchmarksRecords(r.Context(), since) + if err != nil && !errors.Is(err, os.ErrNotExist) { + jsonError(w, "load evidence: "+err.Error(), http.StatusInternalServerError) + return + } + if recs == nil { + recs = []benchmarks.Record{} + } + resp["records"] = recs + resp["count"] = len(recs) + } + jsonResponse(w, resp) +} + +// BenchmarksStatusBlock is the /api/status view. +func (s *Server) BenchmarksStatusBlock() *BenchmarksStatus { + if s.deps == nil || s.deps.Config == nil { + return nil + } + cfg := s.deps.Config + p := s.benchmarks() + p.mu.Lock() + st := p.state + p.mu.Unlock() + out := &BenchmarksStatus{ + HiveHash: benchmarks.HiveHash(cfg.HiveID), + Publish: cfg.Benchmarks.PublishEnabled(), + RepoConsent: cfg.Benchmarks.RepoConsentEnabled(), + LastPushCount: st.LastCount, + TotalPushed: st.TotalPushed, + LastError: st.LastError, + SiteURL: benchmarksSiteURL, + } + if !st.LastPushAt.IsZero() { + out.LastPushAt = st.LastPushAt.UTC().Format(time.RFC3339) + } + if !st.Watermark.IsZero() { + out.WatermarkAt = st.Watermark.UTC().Format(time.RFC3339) + } + return out +} + +// StartBenchmarksPublisher runs the hourly spoke→hub push until ctx ends. +// It returns at once when there is no hub, no hive id or no hub credential. +func (s *Server) StartBenchmarksPublisher(ctx context.Context) { + if s.deps == nil || s.deps.Config == nil { + return + } + cfg := s.deps.Config + hubURL := strings.TrimRight(strings.TrimSpace(cfg.Hub.URL), "/") + if hubURL == "" || strings.TrimSpace(cfg.HiveID) == "" { + return + } + logger := s.logger + if logger == nil { + logger = slog.Default() + } + go func() { + select { + case <-ctx.Done(): + return + case <-time.After(benchmarksPushInitialDelay): + } + ticker := time.NewTicker(benchmarksPushInterval) + defer ticker.Stop() + for { + s.pushBenchmarks(ctx, hubURL, logger) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() +} + +// benchmarksIngestPayload is the body of POST /api/benchmarks/ingest. +type benchmarksIngestPayload struct { + HiveID string `json:"hive_id"` + HiveHash string `json:"hive_hash"` + Watermark string `json:"watermark"` + Records []benchmarks.Record `json:"records"` +} + +// pushBenchmarks sends one batch and advances the watermark on 2xx. +func (s *Server) pushBenchmarks(ctx context.Context, hubURL string, logger *slog.Logger) { + cfg := s.deps.Config + p := s.benchmarks() + if !cfg.Benchmarks.PublishEnabled() { + return + } + bearer := spoke.SpokeHeartbeatKey() + if bearer == "" { + return + } + p.mu.Lock() + since := p.state.Watermark + p.mu.Unlock() + recs, newest, err := s.benchmarksRecords(ctx, since) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + s.benchmarksFail(p, "load evidence: "+err.Error()) + } + return + } + if len(recs) == 0 { + return + } + if len(recs) > benchmarksPushMaxRecords { + recs = recs[:benchmarksPushMaxRecords] + newest = since // partial batch: keep the watermark so the rest follows + } + body, err := json.Marshal(benchmarksIngestPayload{ + HiveID: cfg.HiveID, + HiveHash: benchmarks.HiveHash(cfg.HiveID), + Watermark: newest.UTC().Format(time.RFC3339), + Records: recs, + }) + if err != nil { + return + } + reqCtx, cancel := context.WithTimeout(ctx, benchmarksPushTimeout) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, hubURL+benchmarksHubIngestPath, bytes.NewReader(body)) + if err != nil { + return + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+bearer) + resp, err := http.DefaultClient.Do(req) + if err != nil { + s.benchmarksFail(p, "post: "+err.Error()) + return + } + defer closeHTTPBody(resp.Body) + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + s.benchmarksFail(p, "hub answered "+resp.Status) + return + } + p.mu.Lock() + if newest.After(p.state.Watermark) { + p.state.Watermark = newest + } + p.state.LastPushAt = time.Now() + p.state.LastCount = len(recs) + p.state.TotalPushed += len(recs) + p.state.LastError = "" + p.save() + p.mu.Unlock() + logger.Info("benchmarks: pushed records to hub", "count", len(recs), "watermark", newest.UTC().Format(time.RFC3339)) +} + +func (s *Server) benchmarksFail(p *benchmarksPublisher, msg string) { + p.mu.Lock() + p.state.LastError = msg + p.save() + p.mu.Unlock() + if s.logger != nil { + s.logger.Warn("benchmarks: push failed", "error", msg) + } +} + +// featuresBenchmarksResponse is the Settings → Features → Benchmarks panel +// source (config_features.js). +func (s *Server) featuresBenchmarksResponse(cfg *config.Config) map[string]interface{} { + out := map[string]interface{}{ + "benchmarksPublish": cfg.Benchmarks.PublishEnabled(), + "benchmarksRepoConsent": cfg.Benchmarks.RepoConsentEnabled(), + "benchmarksHiveHash": benchmarks.HiveHash(cfg.HiveID), + } + if st := s.BenchmarksStatusBlock(); st != nil { + out["benchmarksLastPushAt"] = st.LastPushAt + out["benchmarksLastPushCount"] = st.LastPushCount + } + return out +} diff --git a/src/pkg/dashboard/api_benchmarks_test.go b/src/pkg/dashboard/api_benchmarks_test.go new file mode 100644 index 0000000000..c5f8589a8d --- /dev/null +++ b/src/pkg/dashboard/api_benchmarks_test.go @@ -0,0 +1,167 @@ +package dashboard + +import ( + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/evidence" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" + ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +func writeBenchmarkFixtureBundle(t *testing.T, root string, b *evidence.Bundle) { + t.Helper() + if err := evidence.Seal(b, nil); err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, filepath.FromSlash(b.Repo), "1") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + data, _ := json.Marshal(b) + if err := os.WriteFile(filepath.Join(dir, b.HeadSHA+".json"), data, 0o644); err != nil { + t.Fatal(err) + } +} + +func benchmarksTestServer(t *testing.T, publish bool) *Server { + t.Helper() + root := t.TempDir() + prev := ghpkg.ReviewEvidenceRoot + ghpkg.ReviewEvidenceRoot = root + t.Cleanup(func() { ghpkg.ReviewEvidenceRoot = prev }) + now := time.Now().UTC().Add(-time.Hour) + writeBenchmarkFixtureBundle(t, root, &evidence.Bundle{ + SchemaVersion: evidence.SchemaVersion, Repo: "o/r", Number: 1, HeadSHA: "abc1234", + Author: evidence.Author{Login: "bot", Kind: evidence.AuthorAgent, Model: "claude-fable-5.1", Backend: "copilot", Runtime: "podman", TaskKind: "docs"}, + CreatedAt: now, UpdatedAt: now, + Merge: &evidence.MergeEvent{Actor: "hive", Method: "squash", At: now.Add(30 * time.Minute), OpenedAt: now}, + }) + s := newTestServer() + s.deps = &Dependencies{Config: &config.Config{HiveID: "h1", Benchmarks: config.BenchmarksConfig{Publish: &publish}}} + return s +} + +func TestBenchmarksExportEmitsSchemaRecordsAndHonoursSince(t *testing.T) { + s := benchmarksTestServer(t, true) + rec := httptest.NewRecorder() + s.handleBenchmarksExport(rec, httptest.NewRequest(http.MethodGet, "/api/benchmarks/export", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("code=%d body=%s", rec.Code, rec.Body.String()) + } + var resp struct { + HiveHash string `json:"hive_hash"` + Count int `json:"count"` + Records []benchmarks.Record `json:"records"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.HiveHash != benchmarks.HiveHash("h1") || resp.Count != 1 { + t.Fatalf("resp=%+v", resp) + } + r := resp.Records[0] + if r.Source.HiveID != resp.HiveHash || r.Model.Name != "claude-fable-5.1" || r.SandboxRuntime != "podman" || r.Task.Kind != "docs" || !r.Outcome.PRMerged { + t.Errorf("record=%+v", r) + } + if r.Task.Repo != "" { + t.Errorf("repo of unknown visibility must be anonymized, got %q", r.Task.Repo) + } + + rec = httptest.NewRecorder() + s.handleBenchmarksExport(rec, httptest.NewRequest(http.MethodGet, "/api/benchmarks/export?since="+time.Now().UTC().Format(time.RFC3339), nil)) + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil || resp.Count != 0 { + t.Errorf("since in the future should yield 0 records: count=%d err=%v", resp.Count, err) + } + rec = httptest.NewRecorder() + s.handleBenchmarksExport(rec, httptest.NewRequest(http.MethodGet, "/api/benchmarks/export?since=yesterday", nil)) + if rec.Code != http.StatusBadRequest { + t.Errorf("bad since: code=%d", rec.Code) + } +} + +func TestBenchmarksExportRespectsPublishOptOut(t *testing.T) { + s := benchmarksTestServer(t, false) + rec := httptest.NewRecorder() + s.handleBenchmarksExport(rec, httptest.NewRequest(http.MethodGet, "/api/benchmarks/export", nil)) + var resp struct { + Publish bool `json:"publish"` + Count int `json:"count"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil || rec.Code != http.StatusOK { + t.Fatalf("code=%d err=%v", rec.Code, err) + } + if resp.Publish || resp.Count != 0 { + t.Errorf("opt-out must export nothing: %+v", resp) + } + if st := s.BenchmarksStatusBlock(); st == nil || st.Publish || st.HiveHash != benchmarks.HiveHash("h1") { + t.Errorf("status block = %+v", st) + } +} + +func TestPushBenchmarksAdvancesWatermarkAndRecordsFailures(t *testing.T) { + s := benchmarksTestServer(t, true) + prevState := benchmarksPublishStatePath + benchmarksPublishStatePath = filepath.Join(t.TempDir(), "benchmarks-publish.json") + t.Cleanup(func() { benchmarksPublishStatePath = prevState }) + t.Setenv(spoke.EnvHeartbeatKey, "spoke-key") + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + var got struct { + HiveID string `json:"hive_id"` + Records []benchmarks.Record `json:"records"` + } + var bearer string + status := http.StatusAccepted + hub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + bearer = r.Header.Get("Authorization") + _ = json.NewDecoder(r.Body).Decode(&got) + w.WriteHeader(status) + })) + defer hub.Close() + + s.pushBenchmarks(t.Context(), hub.URL, logger) + if bearer != "Bearer spoke-key" || got.HiveID != "h1" || len(got.Records) != 1 { + t.Fatalf("push: bearer=%q hive=%q n=%d", bearer, got.HiveID, len(got.Records)) + } + st := s.BenchmarksStatusBlock() + if st.TotalPushed != 1 || st.LastPushAt == "" || st.WatermarkAt == "" || st.LastError != "" { + t.Fatalf("status after push = %+v", st) + } + if _, err := os.Stat(benchmarksPublishStatePath); err != nil { + t.Fatalf("watermark not persisted: %v", err) + } + + // Nothing new since the watermark: no second request. + got.Records = nil + s.pushBenchmarks(t.Context(), hub.URL, logger) + if got.Records != nil { + t.Fatal("pushed again with nothing newer than the watermark") + } + + // A hub error is surfaced and keeps the watermark where it was. + s.benchmarks().state.Watermark = time.Time{} + status = http.StatusInternalServerError + s.pushBenchmarks(t.Context(), hub.URL, logger) + if st := s.BenchmarksStatusBlock(); st.LastError == "" || st.TotalPushed != 1 { + t.Fatalf("failure not recorded: %+v", st) + } + + // Opt-out short-circuits before any request. + off := false + s.deps.Config.Benchmarks.Publish = &off + bearer = "" + s.pushBenchmarks(t.Context(), hub.URL, logger) + if bearer != "" { + t.Fatal("publish:false must not push") + } +} diff --git a/src/pkg/dashboard/api_governor_features.go b/src/pkg/dashboard/api_governor_features.go index 12fc8cc167..f58e79db10 100644 --- a/src/pkg/dashboard/api_governor_features.go +++ b/src/pkg/dashboard/api_governor_features.go @@ -725,6 +725,9 @@ func (s *Server) featuresLinkedView() map[string]interface{} { // which is what the governor config GET serves. func (s *Server) featuresSectionWithLinked(cfg *config.Config) map[string]interface{} { out := featuresSectionResponse(cfg) + for k, v := range s.featuresBenchmarksResponse(cfg) { + out[k] = v + } for k, v := range s.featuresLinkedView() { out[k] = v } diff --git a/src/pkg/dashboard/api_governor_features_platform.go b/src/pkg/dashboard/api_governor_features_platform.go index 5fe2a5b1c8..1d8c865d73 100644 --- a/src/pkg/dashboard/api_governor_features_platform.go +++ b/src/pkg/dashboard/api_governor_features_platform.go @@ -36,6 +36,9 @@ type featuresPlatformBody struct { EvidenceEnabled *bool `json:"evidenceEnabled"` + BenchmarksPublish *bool `json:"benchmarksPublish"` + BenchmarksRepoConsent *bool `json:"benchmarksRepoConsent"` + GiteaURL *string `json:"giteaUrl"` GiteaTokenEnv *string `json:"giteaTokenEnv"` GitLabURL *string `json:"gitlabUrl"` @@ -174,6 +177,14 @@ func (b featuresPlatformBody) prepare() (func(*config.Config), error) { v := *b.EvidenceEnabled steps = append(steps, func(c *config.Config) { c.Evidence.Enabled = &v }) } + if b.BenchmarksPublish != nil { + v := *b.BenchmarksPublish + steps = append(steps, func(c *config.Config) { c.Benchmarks.Publish = &v }) + } + if b.BenchmarksRepoConsent != nil { + v := *b.BenchmarksRepoConsent + steps = append(steps, func(c *config.Config) { c.Benchmarks.RepoConsent = &v }) + } for _, f := range []struct { raw *string diff --git a/src/pkg/dashboard/api_knowledge.go b/src/pkg/dashboard/api_knowledge.go index 1ee806acb2..9d1b1143f1 100644 --- a/src/pkg/dashboard/api_knowledge.go +++ b/src/pkg/dashboard/api_knowledge.go @@ -1217,7 +1217,7 @@ func (s *Server) handleGitSourcesConnect(w http.ResponseWriter, r *http.Request) } s.auditFromRequest(r, "git_source_connect", auditDetail("name", req.Name, "url", req.URL), "") - jsonResponse(w, map[string]interface{}{"ok": true, "name": req.Name, "url": req.URL}) + jsonResponse(w, map[string]interface{}{"ok": true, "name": req.Name, "url": req.URL, "branch": req.Branch}) } func (s *Server) handleGitSourcesDisconnect(w http.ResponseWriter, r *http.Request) { diff --git a/src/pkg/dashboard/api_knowledge_gitsources_test.go b/src/pkg/dashboard/api_knowledge_gitsources_test.go index a2e26578bf..a6a1bc4ab7 100644 --- a/src/pkg/dashboard/api_knowledge_gitsources_test.go +++ b/src/pkg/dashboard/api_knowledge_gitsources_test.go @@ -136,7 +136,7 @@ case "$*" in exit "${HIVE_FAKE_GIT_PULL_EXIT:-0}" ;; *ls-remote*) - printf 'ref: refs/heads/main\tHEAD\n' + printf 'ref: refs/heads/%s\tHEAD\n' "${HIVE_FAKE_GIT_DEFAULT_BRANCH:-main}" ;; esac exit 0 @@ -266,6 +266,47 @@ func TestGitSourcesConnectSuccessPersistsAndDedupes(t *testing.T) { } } +// TestGitSourcesConnectBlankBranchResolvesRemoteDefault pins that a blank +// branch is resolved from the remote's HEAD symref instead of being guessed as +// "main": the clone uses the resolved branch, and it is persisted to hive.yaml +// and returned so the row shows what was indexed. +func TestGitSourcesConnectBlankBranchResolvesRemoteDefault(t *testing.T) { + logPath := installHermeticGit(t) + t.Setenv("HIVE_FAKE_GIT_DEFAULT_BRANCH", "trunk") + s, deps, url := gitSourceTestServer(t) + + rec := doPost(s, "/api/knowledge/git-sources", map[string]any{"name": "docs", "url": url}) + if rec.Code != http.StatusOK { + t.Fatalf("connect: expected 200, got %d (body %q)", rec.Code, rec.Body.String()) + } + var resp struct { + Branch string `json:"branch"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.Branch != "trunk" { + t.Errorf("response branch = %q, want trunk", resp.Branch) + } + want := config.GitSourceConfigYAML{Name: "docs", URL: url, Branch: "trunk", Layer: "project"} + if got := deps.Config.Knowledge.GitSources; len(got) != 1 || got[0] != want { + t.Fatalf("persisted git sources = %+v, want [%+v]", got, want) + } + if infos := deps.Knowledge.GitSources(); len(infos) != 1 || infos[0].Branch != "trunk" { + t.Errorf("live git sources = %+v, want branch trunk", infos) + } + logged, err := os.ReadFile(logPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(logged), "ls-remote --symref -- "+url+" HEAD") { + t.Errorf("default branch was not resolved via ls-remote; log:\n%s", logged) + } + if !strings.Contains(string(logged), "--branch trunk") { + t.Errorf("clone did not use the resolved branch; log:\n%s", logged) + } +} + func TestGitSourcesConnectPartialFailureDoesNotPersist(t *testing.T) { installHermeticGit(t) t.Setenv("HIVE_FAKE_GIT_SPARSE_SET_EXIT", "7") diff --git a/src/pkg/dashboard/api_repo_hold.go b/src/pkg/dashboard/api_repo_hold.go index 6915ffa081..48b02a8b24 100644 --- a/src/pkg/dashboard/api_repo_hold.go +++ b/src/pkg/dashboard/api_repo_hold.go @@ -47,6 +47,30 @@ func repoHoldResult(repoFull string, number int, itemType string, held bool, lab Next: "The hive picks it up on its next pass."} } +// repoHoldCoveringPR is the open covering PR that carries the hold an +// issue row shows (the row draws it, so its gate pill reads the PR's hold), +// or 0. A release on the issue removes nothing there +// (hivecommons/hive#17564). +func repoHoldCoveringPR(it StageTrackerItem) int { + for _, lp := range it.LinkedPRs { + if lp.State == "closed" || (lp.DrawnUnder != 0 && lp.DrawnUnder != it.Number) { + continue + } + if lp.Hold != nil || slices.Contains(lp.Modifiers, modHold) { + return lp.Number + } + } + return 0 +} + +// repoHoldElsewhereResult answers a release that removed nothing from an +// issue whose covering PR carries the hold: no effect, naming the PR. +func repoHoldElsewhereResult(repoFull string, number, pr int) *RowActionResult { + return &RowActionResult{Action: rowActionNoEffect, Target: rowActionTarget(repoFull, rowActionTargetIssue, number), + Outcome: "Nothing to release on #" + strconv.Itoa(number) + " — the hold is on #" + strconv.Itoa(pr) + ".", + Next: "Release the hold on PR #" + strconv.Itoa(pr) + "."} +} + type repoHoldPermissionCacheEntry struct { allowed bool at time.Time @@ -706,6 +730,14 @@ func (s *Server) handleRepoItemHold(w http.ResponseWriter, r *http.Request) { itemType = "issue" } minStatusSeq := s.applyRepoHoldToStatus(repoFull, number, itemType, body.Held, canonical, removed) + result := repoHoldResult(repoFull, number, itemType, body.Held, canonical, removed) + if !body.Held && len(removed) == 0 && itemType == rowActionTargetIssue { + if it, ok := s.stageTrackerItemFor(repoFull, number); ok { + if pr := repoHoldCoveringPR(it); pr > 0 { + result = repoHoldElsewhereResult(repoFull, number, pr) + } + } + } jsonResponse(w, repoHoldResponse{ OK: true, Repo: repoFull, @@ -714,6 +746,6 @@ func (s *Server) handleRepoItemHold(w http.ResponseWriter, r *http.Request) { HoldLabel: canonical, Removed: removed, MinStatusSeq: minStatusSeq, - Result: repoHoldResult(repoFull, number, itemType, body.Held, canonical, removed), + Result: result, }) } diff --git a/src/pkg/dashboard/api_repo_hold_test.go b/src/pkg/dashboard/api_repo_hold_test.go index f76240c2df..550a45a5bd 100644 --- a/src/pkg/dashboard/api_repo_hold_test.go +++ b/src/pkg/dashboard/api_repo_hold_test.go @@ -290,3 +290,31 @@ func TestRepoHoldPermissionEndpointUsesCachedGitHubPermission(t *testing.T) { t.Fatalf("permission response: code=%d body=%s", w.Code, w.Body.String()) } } + +// TestRepoHoldReleaseOnIssueNamesCoveringPRHold: a release on an issue that +// removed nothing, while the covering PR the row draws carries the hold, is +// a no-effect answer naming the PR — not a silent "no change" +// (hivecommons/hive#17564). +func TestRepoHoldReleaseOnIssueNamesCoveringPRHold(t *testing.T) { + issue := StageTrackerItem{Repo: "o/r", Kind: "issue", Number: 17558, LinkedPRs: []StageTrackerLinkedPR{ + {Number: 17500, State: "closed", Modifiers: []string{modHold}}, + {Number: 17559, DrawnUnder: 17000, Modifiers: []string{modHold}}, + {Number: 17561, Modifiers: []string{modHold}}, + }} + if got := repoHoldCoveringPR(issue); got != 17561 { + t.Fatalf("repoHoldCoveringPR = %d, want 17561 (open, drawn here, held)", got) + } + if got := repoHoldCoveringPR(StageTrackerItem{Number: 1, LinkedPRs: []StageTrackerLinkedPR{{Number: 2}}}); got != 0 { + t.Fatalf("repoHoldCoveringPR on an unheld PR = %d, want 0", got) + } + if got := repoHoldCoveringPR(StageTrackerItem{Number: 1, LinkedPRs: []StageTrackerLinkedPR{{Number: 3, Hold: &StageTrackerHold{}}}}); got != 3 { + t.Fatalf("repoHoldCoveringPR with a hold record = %d, want 3", got) + } + res := repoHoldElsewhereResult("o/r", 17558, 17561) + if res.Action != rowActionNoEffect || res.Target.Number != 17558 || res.Target.Kind != rowActionTargetIssue { + t.Fatalf("result = %+v, want no-effect on the issue", res) + } + if want := "Nothing to release on #17558 — the hold is on #17561."; res.Outcome != want { + t.Fatalf("outcome = %q, want %q", res.Outcome, want) + } +} diff --git a/src/pkg/dashboard/api_repo_item_decision.go b/src/pkg/dashboard/api_repo_item_decision.go index c6a7026252..815b23a052 100644 --- a/src/pkg/dashboard/api_repo_item_decision.go +++ b/src/pkg/dashboard/api_repo_item_decision.go @@ -85,16 +85,23 @@ const ( // superseding PR (hivecommons/hive#15136). It queues the same adoption // the Adopt decision does; it is never implied by an Approve. repoItemDecisionReauthor = "re-author" + // model-reply records one of the human-gate replies the repository's + // bound workflow model declares for the issue's current state + // (hivecommons/hive#17620, RFC phase 4): the verb rides in Reply and is + // recorded as `/hive `; the model's mapped label, if any, is + // applied. Refused on unbound repositories. + repoItemDecisionModelReply = "model-reply" repoItemDecisionMaxBody = 16 << 10 repoItemDecisionMaxComment = 4000 - auditRepoItemDecisionApprove = "repo_item_decision_approve" - auditRepoItemDecisionDirection = "repo_item_decision_direction" - auditRepoItemDecisionClose = "repo_item_decision_close" - auditRepoItemDecisionAdopt = "repo_item_decision_adopt" - auditRepoItemDecisionMerge = "repo_item_decision_merge" - auditRepoItemDecisionQueue = "repo_item_decision_queue_merge" + auditRepoItemDecisionApprove = "repo_item_decision_approve" + auditRepoItemDecisionDirection = "repo_item_decision_direction" + auditRepoItemDecisionModelReply = "repo_item_decision_model_reply" + auditRepoItemDecisionClose = "repo_item_decision_close" + auditRepoItemDecisionAdopt = "repo_item_decision_adopt" + auditRepoItemDecisionMerge = "repo_item_decision_merge" + auditRepoItemDecisionQueue = "repo_item_decision_queue_merge" // repoItemDecisionAdoptMarker opens the attributed note an Adopt // decision leaves on the PR. @@ -125,6 +132,8 @@ type repoItemDecisionRequest struct { Force bool `json:"force,omitempty"` // To with decision "reroute" is the target clanker's login, or "any". To string `json:"to,omitempty"` + // Reply with decision "model-reply" is the workflow model's verb. + Reply string `json:"reply,omitempty"` } // parseRepoItemSupersededBy resolves a "superseded by" reference the way an @@ -266,6 +275,10 @@ func repoItemDecisionResultWith(repoFull string, number int, itemType, decision, out.Outcome = "`/hive approve` recorded; parking labels lifted, `" + label + "` applied." out.Next = "The hive un-parks the issue and carries on; nothing is merged." } + case repoItemDecisionModelReply: + out.Action = rowActionDirected + out.Outcome = "Workflow reply recorded as `/hive " + strings.TrimSuffix(strings.TrimSpace(comment), ".") + "`; parking labels lifted, `" + label + "` applied." + out.Next = "The hive applies the label the model maps to this reply, if any, and carries on; nothing is merged." case repoItemDecisionDirection: out.Action = rowActionDirected out.Outcome = "Direction recorded as `/hive decision`: " + strings.TrimSuffix(strings.TrimSpace(comment), ".") + "." @@ -518,6 +531,39 @@ func (s *Server) handleRepoItemDecision(w http.ResponseWriter, r *http.Request) s.audit.LogRecord(user, auditRepoItemDecisionClose, auditDetail("repo", repoFull, "number", numStr, "type", itemType, "reason", verb, "superseded_by", supersededBy), "", repoFull, number) jsonResponse(w, repoItemDecisionResponse{OK: true, Repo: repoFull, Number: number, Decision: decision, Status: "closed", Label: label, Result: repoItemDecisionResult(repoFull, number, itemType, decision, comment, supersededBy, label, nil)}) + case repoItemDecisionModelReply: + verb := strings.TrimSpace(body.Reply) + if verb == "" { + jsonError(w, "a model reply needs the verb", http.StatusBadRequest) + return + } + if itemType == "pr" { + jsonError(w, "model replies are recorded on issues only", http.StatusBadRequest) + return + } + profile := s.workflowProfileForRepo(repoFull) + if profile == nil { + jsonError(w, "repository is not bound to a workflow model", http.StatusConflict) + return + } + event, err := gh.RecordOperatorIssueModelReply(ctx, repoFull, number, user, verb, comment, profile) + switch { + case errors.Is(err, ghpkg.ErrUnknownModelReply): + jsonError(w, err.Error(), http.StatusConflict) + return + case errors.Is(err, ghpkg.ErrIssueNotParked): + jsonError(w, "issue is not waiting on a decision", http.StatusConflict) + return + case errors.Is(err, ghpkg.ErrDecisionTargetIsPR): + jsonError(w, "this is a pull request; model replies are recorded on issues only", http.StatusBadRequest) + return + case err != nil: + jsonError(w, err.Error(), http.StatusBadGateway) + return + } + s.audit.LogRecord(user, auditRepoItemDecisionModelReply, auditDetail("repo", repoFull, "number", numStr, "workflow", profile.Name+"@"+profile.Version, "reply", verb, "decision", event.Decision), "", repoFull, number) + jsonResponse(w, repoItemDecisionResponse{OK: true, Repo: repoFull, Number: number, Decision: decision, Status: "unparked", Label: ghpkg.HumanAckLabel, + Result: repoItemDecisionResult(repoFull, number, itemType, decision, event.Decision, "", ghpkg.HumanAckLabel, nil)}) case repoItemDecisionReroute: if itemType != "pr" { jsonError(w, "reroute applies to pull requests", http.StatusBadRequest) @@ -567,7 +613,7 @@ func (s *Server) handleRepoItemDecision(w http.ResponseWriter, r *http.Request) } s.handlePullRequestMergeDecision(ctx, w, repoFull, number, user, decision, comment) default: - jsonError(w, "unknown decision; want approve, direction, close-superseded, close-wontfix, close-read, close-issue, adopt, reroute, re-author, merge-now or queue-merge", http.StatusBadRequest) + jsonError(w, "unknown decision; want approve, direction, model-reply, close-superseded, close-wontfix, close-read, close-issue, adopt, reroute, re-author, merge-now or queue-merge", http.StatusBadRequest) } } diff --git a/src/pkg/dashboard/api_workflows.go b/src/pkg/dashboard/api_workflows.go index c1065c0952..d937eca0b9 100644 --- a/src/pkg/dashboard/api_workflows.go +++ b/src/pkg/dashboard/api_workflows.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net/http" + "strings" "time" "github.com/hivecommons/hive/pkg/config" @@ -26,6 +27,11 @@ func (s *Server) registerWorkflowRoutes() { s.mux.HandleFunc("GET /api/workflows/{name}/{version}", s.handleWorkflowGet) s.mux.HandleFunc("PUT /api/workflows/{name}/{version}", s.handleWorkflowSave) s.mux.HandleFunc("DELETE /api/workflows/{name}/{version}", s.handleWorkflowDelete) + s.registerWorkflowVersionRoutes() + s.mux.HandleFunc("POST /api/workflows/{name}/{version}/simulate", s.handleWorkflowSimulate) + s.registerWorkflowEditorRoutes() + s.registerWorkflowIngestSourceRoutes() + s.registerWorkflowMarketplaceRoutes() } func (s *Server) workflowStore(w http.ResponseWriter) *wfstore.Store { @@ -127,6 +133,15 @@ func (s *Server) handleWorkflowSave(w http.ResponseWriter, r *http.Request) { jsonError(w, fmt.Sprintf("body is %s@%s, path is %s@%s", m.Metadata.Name, m.Spec.Meta.Version, name, r.PathValue("version")), http.StatusBadRequest) return } + // The editor posts meta.version "" to mean "next free integer version". + if r.PathValue("name") == "" && strings.TrimSpace(m.Spec.Meta.Version) == "" && m.Metadata.Name != "" { + v, err := st.NextVersion(m.Metadata.Name) + if err != nil { + workflowStoreError(w, err) + return + } + m.Spec.Meta.Version = v + } e, err := st.Save(&m, requestUser(r), time.Now().UTC()) if err != nil { workflowStoreError(w, err) diff --git a/src/pkg/dashboard/api_workflows_editor.go b/src/pkg/dashboard/api_workflows_editor.go new file mode 100644 index 0000000000..e9596da9c0 --- /dev/null +++ b/src/pkg/dashboard/api_workflows_editor.go @@ -0,0 +1,222 @@ +package dashboard + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/hivecommons/hive/pkg/stageengine" + "github.com/hivecommons/hive/pkg/stagelabels" + "github.com/hivecommons/hive/pkg/workflowmodel" + "gopkg.in/yaml.v3" +) + +// Workflow-model editor support (hivecommons/hive#17620, RFC phase 4 UI). +// +// The editor works on an unsaved model in the browser, so the read-only +// computations it needs — YAML round-trip and a what-if derivation — take +// the model in the request body rather than a stored name@version. Like +// POST /api/workflows/compile they need no store and no owner role. +func (s *Server) registerWorkflowEditorRoutes() { + s.mux.HandleFunc("POST /api/workflows/yaml", s.handleWorkflowYAML) + s.mux.HandleFunc("POST /api/workflows/simulate", s.handleWorkflowSimulateModel) +} + +// workflowYAMLRequest carries either a model (→ yaml) or yaml (→ model). +type workflowYAMLRequest struct { + Model json.RawMessage `json:"model,omitempty"` + YAML string `json:"yaml,omitempty"` +} + +// handleWorkflowYAML converts between the JSON model and its YAML form. +// Field names are the json tags on both sides: the JSON document is parsed +// as YAML (JSON is a YAML subset) and re-encoded, so a round-trip changes +// nothing but whitespace. A YAML parse or schema error answers 200 with +// `error` set so the editor can show it inline next to the text. +func (s *Server) handleWorkflowYAML(w http.ResponseWriter, r *http.Request) { + var req workflowYAMLRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + switch { + case len(req.Model) > 0 && strings.TrimSpace(string(req.Model)) != "null": + var m workflowmodel.Model + if err := json.Unmarshal(req.Model, &m); err != nil { + jsonError(w, "model: "+err.Error(), http.StatusBadRequest) + return + } + out, err := workflowModelToYAML(&m) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + jsonResponse(w, map[string]interface{}{"yaml": out}) + case strings.TrimSpace(req.YAML) != "": + m, err := workflowModelFromYAML(req.YAML) + if err != nil { + jsonResponse(w, map[string]interface{}{"error": err.Error()}) + return + } + jsonResponse(w, map[string]interface{}{"model": m}) + default: + jsonError(w, "set model (to get yaml) or yaml (to get a model)", http.StatusBadRequest) + } +} + +func workflowModelToYAML(m *workflowmodel.Model) (string, error) { + b, err := json.Marshal(m) + if err != nil { + return "", err + } + var node yaml.Node + if err := yaml.Unmarshal(b, &node); err != nil { + return "", err + } + // The JSON decoder leaves flow style on every mapping; clear it so the + // encoder writes block YAML. + workflowYAMLBlockStyle(&node) + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + if err := enc.Encode(&node); err != nil { + return "", err + } + if err := enc.Close(); err != nil { + return "", err + } + return buf.String(), nil +} + +func workflowYAMLBlockStyle(n *yaml.Node) { + if n == nil { + return + } + if n.Kind == yaml.MappingNode || n.Kind == yaml.SequenceNode { + n.Style = 0 + } + if n.Kind == yaml.ScalarNode && n.Style == yaml.DoubleQuotedStyle { + n.Style = 0 + } + for _, c := range n.Content { + workflowYAMLBlockStyle(c) + } +} + +func workflowModelFromYAML(text string) (*workflowmodel.Model, error) { + var generic interface{} + if err := yaml.Unmarshal([]byte(text), &generic); err != nil { + return nil, err + } + generic = workflowYAMLToJSONValue(generic) + b, err := json.Marshal(generic) + if err != nil { + return nil, err + } + var m workflowmodel.Model + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + if err := dec.Decode(&m); err != nil { + return nil, err + } + return &m, nil +} + +// workflowYAMLToJSONValue turns yaml.v3's map[string]interface{} and +// []interface{} trees into JSON-marshalable values (yaml.v3 already uses +// string keys for string-keyed maps; non-string keys are stringified). +func workflowYAMLToJSONValue(v interface{}) interface{} { + switch t := v.(type) { + case map[string]interface{}: + out := make(map[string]interface{}, len(t)) + for k, val := range t { + out[k] = workflowYAMLToJSONValue(val) + } + return out + case map[interface{}]interface{}: + out := make(map[string]interface{}, len(t)) + for k, val := range t { + out[fmt.Sprint(k)] = workflowYAMLToJSONValue(val) + } + return out + case []interface{}: + for i := range t { + t[i] = workflowYAMLToJSONValue(t[i]) + } + return t + } + return v +} + +// workflowSimulateModelRequest is the editor's variant of the stored-model +// simulate request: the model travels in the body and facts are required +// (no forge fetch — the canvas may hold something that was never saved). +type workflowSimulateModelRequest struct { + Model *workflowmodel.Model `json:"model"` + Kind string `json:"kind"` + Facts *SimulateFacts `json:"facts"` +} + +// handleWorkflowSimulateModel answers POST /api/workflows/simulate with the +// same SimulateResponse as the stored-model variant. 422 with diagnostics +// when the posted model does not compile, so the editor can point at the +// offending element instead of a bare "does not compile". +func (s *Server) handleWorkflowSimulateModel(w http.ResponseWriter, r *http.Request) { + var req workflowSimulateModelRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + if req.Model == nil { + jsonError(w, "model is required", http.StatusBadRequest) + return + } + req.Kind = strings.ToLower(strings.TrimSpace(req.Kind)) + if req.Kind == "" { + jsonError(w, "kind must be issue or pr", http.StatusBadRequest) + return + } + if req.Facts == nil { + jsonError(w, "facts are required when simulating an unsaved model", http.StatusBadRequest) + return + } + profile, diags, cerr := workflowmodel.Compile(req.Model) + if cerr != nil { + if diags == nil { + diags = []workflowmodel.Diagnostic{} + } + jsonStatusResponse(w, http.StatusUnprocessableEntity, map[string]interface{}{ + "error": fmt.Sprintf("model does not compile: %d diagnostics", len(diags)), + "diagnostics": diags, + }) + return + } + ref, err := stageengine.SimulateRef(req.Kind, "sim", "sim", 0) + if err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + var notes []string + norm := normalizeSimulateFacts(req.Kind, *req.Facts) + ef := norm.toEngine(ref) + if ref.Kind == stagelabels.PRItem && len(ef.PR.Lanes) == 0 { + lane := stageengine.SimulatedLane(ef) + ef.PR.Lanes = []stageengine.LaneFact{lane} + norm.PR.Lane = &SimulateLane{Lane: lane.Lane, Reason: lane.Reason} + notes = append(notes, fmt.Sprintf("no lane observation supplied; simulated lane %s with reason %q from the head facts", lane.Lane, lane.Reason)) + } + dec, trace := stageengine.Explain(profile, ref.Kind, ef) + if trace == nil { + trace = []stageengine.RuleTrace{} + } + jsonResponse(w, SimulateResponse{ + Decision: simulateDecision(profile, dec), + Trace: trace, + Facts: norm, + Filled: []string{}, + Unknown: []string{}, + Notes: notes, + }) +} diff --git a/src/pkg/dashboard/api_workflows_editor_test.go b/src/pkg/dashboard/api_workflows_editor_test.go new file mode 100644 index 0000000000..2e16349e68 --- /dev/null +++ b/src/pkg/dashboard/api_workflows_editor_test.go @@ -0,0 +1,124 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/hivedefault" +) + +func TestWorkflowYAMLRoundTrip(t *testing.T) { + s := workflowTestServer(t, false) + m := techdocsModel(t) + rec := wfReq(s, http.MethodPost, "/api/workflows/yaml", map[string]any{"model": m}, false) + if rec.Code != http.StatusOK { + t.Fatalf("to yaml: %d %s", rec.Code, rec.Body) + } + var out struct { + YAML string `json:"yaml"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(out.YAML, "apiVersion: ") || !strings.Contains(out.YAML, "\nspec:\n") || strings.Contains(out.YAML, "{") { + t.Fatalf("expected block yaml, got:\n%s", out.YAML) + } + rec = wfReq(s, http.MethodPost, "/api/workflows/yaml", map[string]any{"yaml": out.YAML}, false) + if rec.Code != http.StatusOK { + t.Fatalf("from yaml: %d %s", rec.Code, rec.Body) + } + var back struct { + Model *workflowmodel.Model `json:"model"` + Error string `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &back); err != nil { + t.Fatal(err) + } + if back.Error != "" || back.Model == nil { + t.Fatalf("round trip: %s", rec.Body) + } + a, _ := json.Marshal(m) + b, _ := json.Marshal(back.Model) + if string(a) != string(b) { + t.Fatalf("model changed across yaml round trip:\n%s\n%s", a, b) + } +} + +func TestWorkflowYAMLErrorsInline(t *testing.T) { + s := workflowTestServer(t, false) + for _, text := range []string{"spec: [unclosed", "metadata:\n name: x\nbogus: 1\n"} { + rec := wfReq(s, http.MethodPost, "/api/workflows/yaml", map[string]any{"yaml": text}, false) + if rec.Code != http.StatusOK { + t.Fatalf("%q: %d %s", text, rec.Code, rec.Body) + } + var out map[string]any + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if e, _ := out["error"].(string); e == "" { + t.Fatalf("%q: expected inline error, got %s", text, rec.Body) + } + } + rec := wfReq(s, http.MethodPost, "/api/workflows/yaml", map[string]any{}, false) + if rec.Code != http.StatusBadRequest { + t.Fatalf("empty body: %d", rec.Code) + } +} + +func TestWorkflowSimulateUnsavedModel(t *testing.T) { + s := workflowTestServer(t, false) + rec := wfReq(s, http.MethodPost, "/api/workflows/simulate", map[string]any{ + "model": hivedefault.Generate(), "kind": "pr", + "facts": map[string]any{ + "labels": []string{"role/trusted"}, "open": true, + "author": map[string]any{"login": "andy", "kind": "human", "role": "trusted"}, + "pr": map[string]any{"checks": "green", "review": "approved", "dco": "yes"}, + }, + }, false) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + var out SimulateResponse + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + if out.Decision.Stage != 80 || out.Decision.State != "awaiting-merge-admission" || len(out.Trace) != 1 { + t.Fatalf("decision %+v trace %+v", out.Decision, out.Trace) + } + + // Missing facts and missing kind are 400s; a non-compiling model is a + // 422 that carries the diagnostics. + rec = wfReq(s, http.MethodPost, "/api/workflows/simulate", map[string]any{"model": hivedefault.Generate(), "kind": "pr"}, false) + if rec.Code != http.StatusBadRequest { + t.Fatalf("no facts: %d", rec.Code) + } + rec = wfReq(s, http.MethodPost, "/api/workflows/simulate", map[string]any{"model": hivedefault.Generate(), "facts": map[string]any{}}, false) + if rec.Code != http.StatusBadRequest { + t.Fatalf("no kind: %d", rec.Code) + } + broken := techdocsModel(t) + broken.Spec.Transitions = nil + rec = wfReq(s, http.MethodPost, "/api/workflows/simulate", map[string]any{"model": broken, "kind": "issue", "facts": map[string]any{"open": true}}, false) + if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), `"diagnostics"`) { + t.Fatalf("broken model: %d %s", rec.Code, rec.Body) + } +} + +func TestWorkflowSaveAssignsNextVersionWhenEmpty(t *testing.T) { + s := workflowTestServer(t, true) + m := techdocsModel(t) + m.Metadata.Name = "editor-draft" + m.Spec.Meta.Version = "" + rec := wfReq(s, http.MethodPost, "/api/workflows", m, true) + if rec.Code != http.StatusCreated { + t.Fatalf("first save: %d %s", rec.Code, rec.Body) + } + if !strings.Contains(rec.Body.String(), `"version":"1"`) { + t.Fatalf("expected version 1: %s", rec.Body) + } + rec = wfReq(s, http.MethodPost, "/api/workflows", m, true) + if rec.Code != http.StatusCreated || !strings.Contains(rec.Body.String(), `"version":"2"`) { + t.Fatalf("second save: %d %s", rec.Code, rec.Body) + } +} diff --git a/src/pkg/dashboard/api_workflows_ingest_sources.go b/src/pkg/dashboard/api_workflows_ingest_sources.go new file mode 100644 index 0000000000..77fd312a99 --- /dev/null +++ b/src/pkg/dashboard/api_workflows_ingest_sources.go @@ -0,0 +1,366 @@ +package dashboard + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/cluster" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/spec" +) + +// RFC hivecommons/hive#17620 phase 6: ingest Tekton from a live cluster +// context and written process specs (LLM-assisted, every element +// inferred). This file owns the three routes; api_workflows.go only calls +// registerWorkflowIngestSourceRoutes. + +const ( + // workflowSpecURLTimeout bounds fetching a spec by URL. + workflowSpecURLTimeout = 20 * time.Second + // workflowSpecLLMTimeout bounds the whole spec ingest including up to + // spec.MaxAttempts gateway turns. + workflowSpecLLMTimeout = 3 * time.Minute + // workflowSpecLLMProfile is the delegatedecision profile label for the + // spec-ingest turn (audit/logging only). + workflowSpecLLMProfile = "workflow-spec-ingest" + // workflowSpecMaxTextChars caps a pasted or fetched spec, mirroring + // spec.MaxBytes so the handler refuses before decoding a huge body. + workflowSpecMaxTextChars = spec.MaxBytes +) + +func (s *Server) registerWorkflowIngestSourceRoutes() { + s.mux.HandleFunc("GET /api/workflows/ingest/capabilities", s.handleWorkflowIngestCapabilities) + s.mux.HandleFunc("POST /api/workflows/ingest/cluster", s.handleWorkflowIngestCluster) + s.mux.HandleFunc("POST /api/workflows/ingest/spec", s.handleWorkflowIngestSpec) +} + +// workflowSpecLLMConfigured says whether the optional LLM stage can run: +// a runner is wired and the reviewer gateway resolves an endpoint and a +// model. The model id is reported so the UI and the model annotation can +// name it. +func (s *Server) workflowSpecLLMConfigured() (bool, string) { + if s.deps == nil || s.deps.WorkflowSpecLLM == nil || s.deps.Config == nil { + return false, "" + } + if !s.deps.Config.Governor.ReviewerReady() { + return false, "" + } + _, _, model := s.deps.Config.Governor.ResolveReviewer() + return true, model +} + +func (s *Server) workflowClusterRunner() cluster.Runner { + if s.deps == nil { + return nil + } + return s.deps.WorkflowClusterRunner +} + +// handleWorkflowIngestCapabilities answers GET /api/workflows/ingest/capabilities: +// {cluster, clusterReason, contexts[], spec: {llm, model}}. Owner-gated +// like the ingest routes it describes. +func (s *Server) handleWorkflowIngestCapabilities(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + resp := map[string]interface{}{"cluster": false, "contexts": []string{}} + if runner := s.workflowClusterRunner(); runner != nil { + resp["cluster"] = true + ctx, cancel := context.WithTimeout(r.Context(), cluster.KubectlTimeout) + defer cancel() + contexts, err := cluster.Contexts(ctx, runner) + if err != nil { + resp["clusterReason"] = "kube contexts unavailable: " + err.Error() + } else if contexts != nil { + resp["contexts"] = contexts + } + } else { + _, reason := cluster.Available() + resp["clusterReason"] = reason + } + llm, model := s.workflowSpecLLMConfigured() + resp["spec"] = map[string]interface{}{"llm": llm, "model": model} + jsonResponse(w, resp) +} + +// workflowIngestClusterRequest is the body of POST /api/workflows/ingest/cluster. +type workflowIngestClusterRequest struct { + Context string `json:"context"` + Namespace string `json:"namespace"` + AllNamespaces bool `json:"allNamespaces,omitempty"` + Selector string `json:"selector,omitempty"` + // RunsPerPipeline is how many recent PipelineRuns per pipeline to keep + // (default cluster.DefaultRunsPerPipeline, capped). + RunsPerPipeline int `json:"runsPerPipeline,omitempty"` + // Repository scopes the model (owner/name); defaults to the context + // and namespace so the draft is still addressable. + Repository string `json:"repository,omitempty"` + Save bool `json:"save,omitempty"` + Name string `json:"name,omitempty"` +} + +// handleWorkflowIngestCluster lists Tekton objects from a kube context and +// ingests them through the file-based Tekton parser. Same response shape as +// POST /api/workflows/ingest plus source, runs and namespaces. +func (s *Server) handleWorkflowIngestCluster(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + var req workflowIngestClusterRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + runner := s.workflowClusterRunner() + if runner == nil { + _, reason := cluster.Available() + jsonError(w, "cluster ingest not available on this spoke: "+reason, http.StatusUnprocessableEntity) + return + } + src := cluster.Source{ + Context: strings.TrimSpace(req.Context), + Namespace: strings.TrimSpace(req.Namespace), + AllNamespaces: req.AllNamespaces, + Selector: strings.TrimSpace(req.Selector), + RunsPerPipeline: req.RunsPerPipeline, + } + repo := strings.TrimSpace(req.Repository) + if repo == "" { + repo = "cluster/" + workflowmodel.Slug(src.String()) + } + ctx, cancel := context.WithTimeout(r.Context(), workflowIngestTimeout) + defer cancel() + res, err := cluster.Ingest(ctx, runner, src, workflowmodel.Scope{Forge: "github.com", Repository: repo}) + if err != nil { + jsonError(w, err.Error(), clusterErrorStatus(err)) + return + } + m := res.Model + if req.Name != "" { + m.Metadata.Name = req.Name + } + p, diags, cerr := workflowmodel.Compile(m) + if diags == nil { + diags = []workflowmodel.Diagnostic{} + } + resp := map[string]interface{}{ + "model": m, "compiled": cerr == nil, "diagnostics": diags, + "ref": res.Source, "source": res.Source, "files": res.Files, "pipelines": len(m.Spec.Pipelines), + "runs": res.Runs, "namespaces": res.Namespaces, + } + if p != nil { + resp["profile"] = p + } + s.finishWorkflowIngest(w, r, req.Save, m, resp, fmt.Sprintf("from %s files=%d", res.Source, res.Files)) +} + +// clusterErrorStatus maps cluster ingest errors to HTTP statuses with the +// package's clear messages. +func clusterErrorStatus(err error) int { + switch { + case errors.Is(err, cluster.ErrNamespaceRequired): + return http.StatusBadRequest + case errors.Is(err, cluster.ErrForbidden): + return http.StatusForbidden + case errors.Is(err, cluster.ErrNoContext): + return http.StatusNotFound + case errors.Is(err, cluster.ErrNoCRD), errors.Is(err, cluster.ErrNoRunner): + return http.StatusUnprocessableEntity + case errors.Is(err, cluster.ErrTooLarge): + return http.StatusRequestEntityTooLarge + } + return http.StatusBadGateway +} + +// workflowIngestSpecRequest is the body of POST /api/workflows/ingest/spec. +// Exactly one of Text, Repo+Path or URL is set. +type workflowIngestSpecRequest struct { + Text string `json:"text,omitempty"` + Repo string `json:"repo,omitempty"` + Path string `json:"path,omitempty"` + URL string `json:"url,omitempty"` + // UseLLM asks for the gateway refinement; nil means "if configured". + UseLLM *bool `json:"useLLM,omitempty"` + // Repository scopes the model (owner/name); defaults to Repo or + // spec/. + Repository string `json:"repository,omitempty"` + Save bool `json:"save,omitempty"` + Name string `json:"name,omitempty"` +} + +// handleWorkflowIngestSpec turns a written process spec into a draft model. +// Response: {model, compiled, diagnostics, profile?, source, llm:{used, +// model, attempts, accepted, error?}, sections[], entry?}. +func (s *Server) handleWorkflowIngestSpec(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + var req workflowIngestSpecRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + req.Repo, req.Path, req.URL = strings.TrimSpace(req.Repo), strings.TrimSpace(req.Path), strings.TrimSpace(req.URL) + inputs := 0 + for _, set := range []bool{strings.TrimSpace(req.Text) != "", req.Repo != "" || req.Path != "", req.URL != ""} { + if set { + inputs++ + } + } + if inputs != 1 { + jsonError(w, "set exactly one of text, repo+path or url", http.StatusBadRequest) + return + } + llmReady, llmModel := s.workflowSpecLLMConfigured() + useLLM := llmReady + if req.UseLLM != nil { + useLLM = *req.UseLLM + } + if useLLM && !llmReady { + jsonError(w, "AI assistance requested but no model gateway is configured (Settings → Model Gateways); retry with useLLM=false for the deterministic draft", http.StatusUnprocessableEntity) + return + } + + ctx, cancel := context.WithTimeout(r.Context(), workflowSpecLLMTimeout) + defer cancel() + text, origin, repo, status, err := s.loadWorkflowSpecText(ctx, req) + if err != nil { + jsonError(w, err.Error(), status) + return + } + if strings.TrimSpace(req.Repository) != "" { + repo = strings.TrimSpace(req.Repository) + } + opts := spec.Options{Origin: origin, Name: strings.TrimSpace(req.Name), Scope: workflowmodel.Scope{Forge: "github.com", Repository: repo}} + if useLLM { + runner := s.deps.WorkflowSpecLLM + opts.ModelID = llmModel + opts.Completer = spec.CompleterFunc(func(ctx context.Context, system, user string) (string, error) { + return runner.RunDeciderTurn(ctx, workflowSpecLLMProfile, system, user) + }) + } + res, err := spec.Ingest(ctx, text, opts) + if err != nil { + status := http.StatusBadRequest + if errors.Is(err, spec.ErrTooLarge) { + status = http.StatusRequestEntityTooLarge + } + jsonError(w, err.Error(), status) + return + } + diags := res.Diagnostics + if diags == nil { + diags = []workflowmodel.Diagnostic{} + } + resp := map[string]interface{}{ + "model": res.Model, "compiled": res.Compiled, "diagnostics": diags, + "source": res.Source, "llm": res.LLM, "sections": res.Sections, + } + if res.Profile != nil { + resp["profile"] = res.Profile + } + s.finishWorkflowIngest(w, r, req.Save, res.Model, resp, fmt.Sprintf("from %s llm=%v attempts=%d", res.Source, res.LLM.Accepted, res.LLM.Attempts)) +} + +// loadWorkflowSpecText resolves the spec body from the request: pasted +// text, a default-branch file through the forge, or an https URL (public +// hosts only, size-capped). It returns the text, its origin label, the +// repository to scope the model to and, on error, the HTTP status. +func (s *Server) loadWorkflowSpecText(ctx context.Context, req workflowIngestSpecRequest) (text, origin, repo string, status int, err error) { + switch { + case strings.TrimSpace(req.Text) != "": + if len(req.Text) > workflowSpecMaxTextChars { + return "", "", "", http.StatusRequestEntityTooLarge, spec.ErrTooLarge + } + return req.Text, "pasted", "spec/pasted", 0, nil + case req.URL != "": + u, perr := url.Parse(req.URL) + if perr != nil || u.Scheme != "https" || u.Host == "" { + return "", "", "", http.StatusBadRequest, errors.New("url must be https://host/path") + } + if isPrivateURL(ctx, req.URL) { + return "", "", "", http.StatusBadRequest, errors.New("url resolves to a private or internal host") + } + body, ferr := fetchWorkflowSpecURL(ctx, req.URL) + if ferr != nil { + return "", "", "", http.StatusBadGateway, fmt.Errorf("fetch %s: %w", req.URL, ferr) + } + return body, req.URL, "spec/" + workflowmodel.Slug(u.Host+u.Path), 0, nil + default: + if req.Repo == "" || req.Path == "" || !strings.Contains(req.Repo, "/") { + return "", "", "", http.StatusBadRequest, errors.New("repo (owner/name) and path are both required") + } + if s.deps == nil || s.deps.WorkflowRepoFile == nil { + return "", "", "", http.StatusServiceUnavailable, errors.New("repository spec ingest not available: no forge client") + } + body, ferr := s.deps.WorkflowRepoFile(ctx, req.Repo, req.Path) + if ferr != nil { + return "", "", "", http.StatusBadGateway, fmt.Errorf("fetch %s:%s: %w", req.Repo, req.Path, ferr) + } + if len(body) > workflowSpecMaxTextChars { + return "", "", "", http.StatusRequestEntityTooLarge, spec.ErrTooLarge + } + return body, req.Repo + ":" + strings.TrimPrefix(req.Path, "/"), req.Repo, 0, nil + } +} + +// workflowSpecHTTP is the client for URL specs; a seam for tests. +var workflowSpecHTTP = &http.Client{Timeout: workflowSpecURLTimeout, CheckRedirect: noRedirectToPrivate} + +func fetchWorkflowSpecURL(ctx context.Context, rawURL string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return "", err + } + req.Header.Set("Accept", "text/markdown, text/plain;q=0.9, */*;q=0.1") + resp, err := workflowSpecHTTP.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("status %d", resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, int64(workflowSpecMaxTextChars)+1)) + if err != nil { + return "", err + } + if len(body) > workflowSpecMaxTextChars { + return "", spec.ErrTooLarge + } + return string(body), nil +} + +// finishWorkflowIngest either answers the preview or saves the model as +// the next draft version, audits it and answers 201 with the entry; the +// same tail POST /api/workflows/ingest has. +func (s *Server) finishWorkflowIngest(w http.ResponseWriter, r *http.Request, save bool, m *workflowmodel.Model, resp map[string]interface{}, detail string) { + if !save { + jsonResponse(w, resp) + return + } + st := s.workflowStore(w) + if st == nil { + return + } + version, err := nextWorkflowVersion(st, m.Metadata.Name) + if err != nil { + workflowStoreError(w, err) + return + } + m.Spec.Meta.Version = version + e, err := st.Save(m, requestUser(r), time.Now().UTC()) + if err != nil { + workflowStoreError(w, err) + return + } + s.auditFromRequest(r, "workflow_ingest", fmt.Sprintf("%s %s compiled=%v", e.Key(), detail, e.Compiled), "") + resp["entry"] = e + jsonStatusResponse(w, http.StatusCreated, resp) +} diff --git a/src/pkg/dashboard/api_workflows_ingest_sources_test.go b/src/pkg/dashboard/api_workflows_ingest_sources_test.go new file mode 100644 index 0000000000..11a241d36a --- /dev/null +++ b/src/pkg/dashboard/api_workflows_ingest_sources_test.go @@ -0,0 +1,456 @@ +package dashboard + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/cluster" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/spec" + wfstore "github.com/hivecommons/hive/pkg/workflowmodel/store" +) + +// fakeClusterRunner answers kubectl get with a one-pipeline Tekton +// namespace and kubectl config get-contexts with two contexts. +type fakeClusterRunner struct { + err error + calls int +} + +func (f *fakeClusterRunner) Run(_ context.Context, args ...string) ([]byte, error) { + f.calls++ + if f.err != nil { + return nil, f.err + } + argv := strings.Join(args, " ") + if strings.HasPrefix(argv, "config get-contexts") { + return []byte("kind-ci\nprod\n"), nil + } + items := []any{} + meta := func(kind, name string) map[string]any { + return map[string]any{"apiVersion": "tekton.dev/v1", "kind": kind, "metadata": map[string]any{"name": name, "namespace": "ci"}} + } + switch { + case strings.Contains(argv, "get pipelines."): + p := meta("Pipeline", "build") + p["spec"] = map[string]any{"tasks": []any{map[string]any{"name": "test", "taskRef": map[string]any{"name": "go-test"}}}} + items = append(items, p) + case strings.Contains(argv, "get pipelineruns."): + r := meta("PipelineRun", "build-1") + r["metadata"].(map[string]any)["annotations"] = map[string]any{"pipelinesascode.tekton.dev/on-event": "[pull_request]"} + r["spec"] = map[string]any{"pipelineRef": map[string]any{"name": "build"}} + r["status"] = map[string]any{"completionTime": "2026-01-01T00:00:00Z", "conditions": []any{map[string]any{"type": "Succeeded", "status": "False", "reason": "Failed"}}} + items = append(items, r) + case strings.Contains(argv, "triggers.tekton.dev"): + return nil, errors.New(`error: the server doesn't have a resource type "eventlisteners"`) + } + return json.Marshal(map[string]any{"items": items}) +} + +type fakeTurnRunner struct { + answers []string + err error + calls int + system string + user string +} + +func (f *fakeTurnRunner) RunDeciderTurn(_ context.Context, _, system, user string) (string, error) { + f.calls++ + f.system, f.user = system, user + if f.err != nil { + return "", f.err + } + return f.answers[(f.calls-1)%len(f.answers)], nil +} + +func configureSpecLLM(s *Server, ready bool) { + g := &s.deps.Config.Governor + if ready { + g.Trajectory.Endpoint = "https://reviewer.local" + g.Trajectory.Model = "reviewer/model-1" + return + } + g.Trajectory.Endpoint, g.Trajectory.Model = "", "" + g.LiteLLM.Endpoint, g.LiteLLM.DefaultModel = "", "" +} + +const specSample = "# Release process\n\n1. Request: the author opens an issue.\n2. Review by a maintainer. If rejected it goes back to request.\n3. Approve\n4. Merge\n" + +func wfDecode(t *testing.T, rec *httptest.ResponseRecorder, v any) { + t.Helper() + if err := json.Unmarshal(rec.Body.Bytes(), v); err != nil { + t.Fatalf("%v: %s", err, rec.Body) + } +} + +func TestWorkflowIngestCapabilities(t *testing.T) { + s := workflowTestServer(t, false) + configureSpecLLM(s, false) + s.deps.WorkflowClusterRunner = nil + if rec := wfReq(s, http.MethodGet, "/api/workflows/ingest/capabilities", nil, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner %d", rec.Code) + } + rec := wfReq(s, http.MethodGet, "/api/workflows/ingest/capabilities", nil, true) + var caps struct { + Cluster bool `json:"cluster"` + ClusterReason string `json:"clusterReason"` + Contexts []string `json:"contexts"` + Spec struct { + LLM bool `json:"llm"` + Model string `json:"model"` + } `json:"spec"` + } + wfDecode(t, rec, &caps) + if caps.Cluster || caps.Contexts == nil || len(caps.Contexts) != 0 || caps.Spec.LLM || caps.Spec.Model != "" { + t.Fatalf("%+v", caps) + } + + s.deps.WorkflowClusterRunner = &fakeClusterRunner{} + s.deps.WorkflowSpecLLM = &fakeTurnRunner{} + configureSpecLLM(s, true) + rec = wfReq(s, http.MethodGet, "/api/workflows/ingest/capabilities", nil, true) + wfDecode(t, rec, &caps) + if !caps.Cluster || strings.Join(caps.Contexts, ",") != "kind-ci,prod" || !caps.Spec.LLM || caps.Spec.Model != "reviewer/model-1" { + t.Fatalf("%+v", caps) + } + // A runner that cannot list contexts still reports cluster=true with a reason. + s.deps.WorkflowClusterRunner = &fakeClusterRunner{err: errors.New("kubeconfig unreadable")} + rec = wfReq(s, http.MethodGet, "/api/workflows/ingest/capabilities", nil, true) + wfDecode(t, rec, &caps) + if !caps.Cluster || !strings.Contains(caps.ClusterReason, "kubeconfig unreadable") || len(caps.Contexts) != 0 { + t.Fatalf("%+v", caps) + } + // A gateway runner wired but unconfigured is not advertised. + configureSpecLLM(s, false) + rec = wfReq(s, http.MethodGet, "/api/workflows/ingest/capabilities", nil, true) + wfDecode(t, rec, &caps) + if caps.Spec.LLM { + t.Fatalf("%+v", caps) + } +} + +func TestWorkflowIngestClusterPreviewAndSave(t *testing.T) { + s := workflowTestServer(t, true) + runner := &fakeClusterRunner{} + s.deps.WorkflowClusterRunner = runner + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/cluster", map[string]any{"namespace": "ci"}, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner %d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/cluster", "{bad", true); rec.Code != http.StatusBadRequest { + t.Fatalf("bad body %d", rec.Code) + } + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/cluster", map[string]any{"context": "kind-ci", "namespace": "ci", "selector": "team=a"}, true) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + var resp struct { + Compiled bool `json:"compiled"` + Pipelines int `json:"pipelines"` + Files int `json:"files"` + Source string `json:"source"` + Ref string `json:"ref"` + Runs []cluster.RunStatus `json:"runs"` + Namespaces []string `json:"namespaces"` + Model workflowmodel.Model `json:"model"` + Diags []workflowmodel.Diagnostic `json:"diagnostics"` + Profile *workflowmodel.Profile `json:"profile"` + Entry *wfstore.Entry `json:"entry"` + } + wfDecode(t, rec, &resp) + if !resp.Compiled || resp.Pipelines != 1 || resp.Files != 2 || resp.Source != "cluster:kind-ci/ci" || resp.Ref != resp.Source || resp.Profile == nil || resp.Entry != nil { + t.Fatalf("%+v", resp) + } + if len(resp.Runs) != 1 || resp.Runs[0].Status != "Failed" || strings.Join(resp.Namespaces, ",") != "ci" { + t.Fatalf("runs %+v ns %v", resp.Runs, resp.Namespaces) + } + if resp.Model.Spec.Scope.Repository != "cluster/cluster-kind-ci-ci" || resp.Model.Spec.Meta.Provenance != workflowmodel.ProvenanceIngestedUnreviewed { + t.Fatalf("scope %+v meta %+v", resp.Model.Spec.Scope, resp.Model.Spec.Meta) + } + if resp.Model.Metadata.Annotations[cluster.AnnotationObservedRun+resp.Model.Spec.Pipelines[0].ID] != "Failed" { + t.Fatalf("observed run annotation %v", resp.Model.Metadata.Annotations) + } + + before := time.Now().Add(-time.Minute) + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/cluster", map[string]any{"namespace": "ci", "repository": "acme/widgets", "name": "ci-draft", "save": true}, true) + if rec.Code != http.StatusCreated { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + wfDecode(t, rec, &resp) + if resp.Entry == nil || resp.Entry.Key() != "ci-draft@1" || resp.Model.Spec.Scope.Repository != "acme/widgets" || resp.Source != "cluster:current-context/ci" { + t.Fatalf("%+v", resp) + } + audits := s.audit.RecentWithPrefixSince(before, "workflow_ingest") + if len(audits) != 1 || !strings.Contains(audits[0].Detail, "ci-draft@1 from cluster:current-context/ci files=2 compiled=true") { + t.Fatalf("audit %+v", audits) + } + // Saving needs the store. + s2 := workflowTestServer(t, false) + s2.deps.WorkflowClusterRunner = runner + if rec := wfReq(s2, http.MethodPost, "/api/workflows/ingest/cluster", map[string]any{"namespace": "ci", "save": true}, true); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no store %d %s", rec.Code, rec.Body) + } +} + +func TestWorkflowIngestClusterErrors(t *testing.T) { + s := workflowTestServer(t, false) + s.deps.WorkflowClusterRunner = nil + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/cluster", map[string]any{"namespace": "ci"}, true) + if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "not available on this spoke") { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + cases := []struct { + name string + body map[string]any + err error + code int + }{ + {"namespace required", map[string]any{}, nil, http.StatusBadRequest}, + {"forbidden", map[string]any{"namespace": "ci"}, errors.New("Error from server (Forbidden): pipelines.tekton.dev is forbidden"), http.StatusForbidden}, + {"no context", map[string]any{"namespace": "ci", "context": "nope"}, errors.New(`error: context "nope" does not exist`), http.StatusNotFound}, + {"no crd", map[string]any{"namespace": "ci"}, errors.New(`error: the server doesn't have a resource type "pipelines"`), http.StatusUnprocessableEntity}, + {"other", map[string]any{"namespace": "ci"}, errors.New("dial tcp: connection refused"), http.StatusBadGateway}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s.deps.WorkflowClusterRunner = &fakeClusterRunner{err: tc.err} + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/cluster", tc.body, true) + if rec.Code != tc.code { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + }) + } + if clusterErrorStatus(cluster.ErrTooLarge) != http.StatusRequestEntityTooLarge || clusterErrorStatus(cluster.ErrNoRunner) != http.StatusUnprocessableEntity { + t.Fatal("status mapping") + } +} + +type specResponse struct { + Compiled bool `json:"compiled"` + Source string `json:"source"` + LLM spec.LLMReport `json:"llm"` + Sections []spec.Section `json:"sections"` + Model workflowmodel.Model `json:"model"` + Diags []workflowmodel.Diagnostic `json:"diagnostics"` + Profile *workflowmodel.Profile `json:"profile"` + Entry *wfstore.Entry `json:"entry"` +} + +func TestWorkflowIngestSpecTextDeterministic(t *testing.T) { + s := workflowTestServer(t, true) + configureSpecLLM(s, false) + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample}, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner %d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", "{bad", true); rec.Code != http.StatusBadRequest { + t.Fatalf("bad body %d", rec.Code) + } + for _, body := range []map[string]any{{}, {"text": "x", "url": "https://a/b"}, {"text": "x", "repo": "a/b", "path": "p"}} { + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", body, true); rec.Code != http.StatusBadRequest { + t.Fatalf("%v -> %d", body, rec.Code) + } + } + // No gateway: default is deterministic; asking for the LLM is 422. + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample, "useLLM": true}, true) + if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "no model gateway") { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample}, true) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + var resp specResponse + wfDecode(t, rec, &resp) + if !resp.Compiled || resp.LLM.Used || resp.Source != "spec:pasted" || len(resp.Sections) != 1 || resp.Profile == nil || resp.Entry != nil { + t.Fatalf("%+v", resp) + } + if resp.Model.Spec.Scope.Repository != "spec/pasted" || resp.Model.Metadata.Name != "release-process" || resp.Model.Spec.Meta.Source != workflowmodel.SourceSpec { + t.Fatalf("%+v %+v", resp.Model.Metadata, resp.Model.Spec.Meta) + } + for _, st := range resp.Model.Spec.States { + if st.Lifecycle.Confidence != workflowmodel.ConfidenceInferred || !strings.HasPrefix(st.Lifecycle.Citations[0].Path, "spec:Release process#L") { + t.Fatalf("state %+v", st) + } + } + if _, ok := resp.Model.Metadata.Annotations[spec.AnnotationLLMAssisted]; ok { + t.Fatal("deterministic result must not claim LLM assistance") + } + // Save + audit. + before := time.Now().Add(-time.Minute) + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample, "name": "proc", "repository": "acme/widgets", "save": true}, true) + if rec.Code != http.StatusCreated { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + wfDecode(t, rec, &resp) + if resp.Entry == nil || resp.Entry.Key() != "proc@1" || resp.Model.Spec.Scope.Repository != "acme/widgets" { + t.Fatalf("%+v", resp.Entry) + } + audits := s.audit.RecentWithPrefixSince(before, "workflow_ingest") + if len(audits) != 1 || !strings.Contains(audits[0].Detail, "proc@1 from spec:pasted llm=false attempts=0") { + t.Fatalf("audit %+v", audits) + } + // Size caps. + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": strings.Repeat("x", workflowSpecMaxTextChars+1)}, true) + if rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("%d", rec.Code) + } +} + +func TestWorkflowIngestSpecWithLLM(t *testing.T) { + s := workflowTestServer(t, false) + configureSpecLLM(s, true) + // The gateway answers with the deterministic skeleton relabelled, which + // compiles; the handler must mark it inferred and name the model. + secs := spec.Split(specSample) + good := spec.Extract("pasted", "", workflowmodel.Scope{Forge: "github.com", Repository: "spec/pasted"}, secs) + good.Spec.States[0].Label = "Request a release" + raw, _ := json.Marshal(good) + runner := &fakeTurnRunner{answers: []string{"```json\n" + string(raw) + "\n```"}} + s.deps.WorkflowSpecLLM = runner + + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample}, true) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + var resp specResponse + wfDecode(t, rec, &resp) + if !resp.LLM.Used || !resp.LLM.Accepted || resp.LLM.Attempts != 1 || resp.LLM.Model != "reviewer/model-1" || !resp.Compiled { + t.Fatalf("%+v", resp.LLM) + } + if resp.Model.Spec.States[0].Label != "Request a release" || resp.Model.Metadata.Annotations[spec.AnnotationLLMAssisted] != "LLM-assisted (model reviewer/model-1)" { + t.Fatalf("%+v %v", resp.Model.Spec.States[0], resp.Model.Metadata.Annotations) + } + if runner.system != spec.SystemPrompt || !strings.Contains(runner.user, "# Specification (pasted)") { + t.Fatal("gateway must receive the package prompt") + } + // useLLM=false opts out even when configured. + runner.calls = 0 + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample, "useLLM": false}, true) + wfDecode(t, rec, &resp) + if rec.Code != http.StatusOK || resp.LLM.Used || runner.calls != 0 { + t.Fatalf("%d %+v calls=%d", rec.Code, resp.LLM, runner.calls) + } + // A gateway failure falls back to the skeleton and is reported, not a 5xx. + s.deps.WorkflowSpecLLM = &fakeTurnRunner{err: errors.New("502 from gateway")} + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample}, true) + wfDecode(t, rec, &resp) + if rec.Code != http.StatusOK || !resp.LLM.Used || resp.LLM.Accepted || !strings.Contains(resp.LLM.Error, "502 from gateway") || !resp.Compiled { + t.Fatalf("%d %+v", rec.Code, resp.LLM) + } + found := false + for _, f := range resp.Model.Spec.Coverage.Findings { + found = found || f.Code == spec.CodeLLMFellBack + } + if !found { + t.Fatalf("findings %+v", resp.Model.Spec.Coverage.Findings) + } +} + +func TestWorkflowIngestSpecFromRepoAndURL(t *testing.T) { + s := workflowTestServer(t, false) + configureSpecLLM(s, false) + // repo+path validation and missing forge client. + s.deps.WorkflowRepoFile = nil + for body, code := range map[string]int{ + `{"repo":"acme/widgets"}`: http.StatusBadRequest, + `{"path":"docs/p.md"}`: http.StatusBadRequest, + `{"repo":"nope","path":"docs/p.md"}`: http.StatusBadRequest, + `{"repo":"acme/widgets","path":"docs/p.md"}`: http.StatusServiceUnavailable, + } { + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", body, true); rec.Code != code { + t.Fatalf("%s -> %d %s", body, rec.Code, rec.Body) + } + } + var asked string + s.deps.WorkflowRepoFile = func(_ context.Context, repo, path string) (string, error) { + asked = repo + ":" + path + switch path { + case "missing.md": + return "", errors.New("404") + case "huge.md": + return strings.Repeat("x", workflowSpecMaxTextChars+1), nil + } + return specSample, nil + } + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"repo": "acme/widgets", "path": "/docs/process.md"}, true) + if rec.Code != http.StatusOK || asked != "acme/widgets:/docs/process.md" { + t.Fatalf("%d %s asked=%q", rec.Code, rec.Body, asked) + } + var resp specResponse + wfDecode(t, rec, &resp) + if resp.Source != "spec:acme/widgets:docs/process.md" || resp.Model.Spec.Scope.Repository != "acme/widgets" { + t.Fatalf("%+v", resp) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"repo": "acme/widgets", "path": "missing.md"}, true); rec.Code != http.StatusBadGateway { + t.Fatalf("%d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"repo": "acme/widgets", "path": "huge.md"}, true); rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("%d", rec.Code) + } + + // URL: https only, no private hosts, size cap, upstream status. + for _, u := range []string{"http://example.com/spec.md", "ftp://x/y", "https://", "not a url", "https://127.0.0.1/spec.md", "https://localhost/spec.md"} { + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"url": u}, true); rec.Code != http.StatusBadRequest { + t.Fatalf("%s -> %d %s", u, rec.Code, rec.Body) + } + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/spec.md": + fmt.Fprint(w, specSample) + case "/huge.md": + fmt.Fprint(w, strings.Repeat("x", workflowSpecMaxTextChars+1)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + orig := workflowSpecHTTP + origResolver := privateURLResolver + t.Cleanup(func() { workflowSpecHTTP, privateURLResolver = orig, origResolver }) + privateURLResolver = func(_ context.Context, host string) ([]string, error) { + if host == "specs.example.com" { + return []string{"93.184.216.34"}, nil + } + return nil, errors.New("no such host") + } + workflowSpecHTTP = srv.Client() + // The handler insists on https and a public host; route the public name + // to the test server by rewriting in the transport. + workflowSpecHTTP.Transport = rewriteTransport{base: srv.Client().Transport, to: srv.URL} + + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"url": "https://specs.example.com/spec.md"}, true) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + wfDecode(t, rec, &resp) + if resp.Source != "spec:https://specs.example.com/spec.md" || resp.Model.Spec.Scope.Repository != "spec/specs-example-com-spec-md" { + t.Fatalf("%+v", resp) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"url": "https://specs.example.com/huge.md"}, true); rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "exceeds") { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"url": "https://specs.example.com/nope.md"}, true); rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "status 404") { + t.Fatalf("%d %s", rec.Code, rec.Body) + } +} + +// rewriteTransport sends every request to a fixed test server URL. +type rewriteTransport struct { + base http.RoundTripper + to string +} + +func (rt rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + clone := req.Clone(req.Context()) + clone.URL.Scheme = "http" + clone.URL.Host = strings.TrimPrefix(rt.to, "http://") + return rt.base.RoundTrip(clone) +} diff --git a/src/pkg/dashboard/api_workflows_marketplace.go b/src/pkg/dashboard/api_workflows_marketplace.go new file mode 100644 index 0000000000..98d2390836 --- /dev/null +++ b/src/pkg/dashboard/api_workflows_marketplace.go @@ -0,0 +1,518 @@ +package dashboard + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + spoke "github.com/hivecommons/hive/pkg/hub/spoke" + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" + wfstore "github.com/hivecommons/hive/pkg/workflowmodel/store" +) + +// RFC #17620 phase 7: bundle export/import on the spoke plus a proxy to the +// hub marketplace catalogue. Routes live in their own file so concurrent +// work on api_workflows.go only needs the one-line registration hook. + +const ( + // marketplaceHubTimeout bounds one spoke→hub call. + marketplaceHubTimeout = 20 * time.Second + // marketplaceMaxBody caps bundle and index payloads read from clients + // or the hub. + marketplaceMaxBody = 4 << 20 + // marketplaceHiveIDHeader carries the spoke's hive_id alongside the + // heartbeat bearer so the hub can bind the credential to a hive. + marketplaceHiveIDHeader = "X-Hive-ID" + // marketplaceStatusStale is the status returned when the hub is + // unreachable; the body may still carry a cached index. + marketplaceStatusStale = http.StatusBadGateway +) + +// marketplaceHTTP is the client used for hub calls; tests swap it. +var marketplaceHTTP = &http.Client{Timeout: marketplaceHubTimeout} + +// marketplaceIndexCache is the spoke's ETag-validated copy of the hub index. +type marketplaceIndexCache struct { + mu sync.Mutex + etag string + body []byte + fetchedAt time.Time +} + +var marketplaceCache marketplaceIndexCache + +func (s *Server) registerWorkflowMarketplaceRoutes() { + s.mux.HandleFunc("GET /api/workflows/{name}/{version}/bundle", s.handleWorkflowBundleGet) + s.mux.HandleFunc("POST /api/workflows/import", s.handleWorkflowImport) + s.mux.HandleFunc("GET /api/marketplace/workflows", s.handleMarketplaceIndex) + s.mux.HandleFunc("POST /api/marketplace/workflows/publish", s.handleMarketplacePublish) + s.mux.HandleFunc("POST /api/marketplace/workflows/{name}/{version}/import", s.handleMarketplaceImport) + s.mux.HandleFunc("POST /api/marketplace/refresh", s.handleMarketplaceRefresh) +} + +// marketplaceEnabled reports workflow.marketplace.enabled (default true). +func (s *Server) marketplaceEnabled() bool { + if s.deps == nil || s.deps.Config == nil { + return true + } + return s.deps.Config.Workflow.Marketplace.MarketplaceEnabled() +} + +// marketplaceBase is the hub base URL for /api/marketplace/*, or "" when the +// marketplace is disabled or no hub is configured. +func (s *Server) marketplaceBase() string { + if !s.marketplaceEnabled() || s.deps == nil || s.deps.Config == nil { + return "" + } + base := s.deps.Config.MarketplaceURL() + if u, err := url.Parse(base); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return "" + } + return base +} + +func (s *Server) requireMarketplace(w http.ResponseWriter) (string, bool) { + if !s.marketplaceEnabled() { + jsonError(w, "workflow marketplace disabled (workflow.marketplace.enabled)", http.StatusNotFound) + return "", false + } + base := s.marketplaceBase() + if base == "" { + jsonError(w, "workflow marketplace has no hub URL (workflow.marketplace.url or hub.url)", http.StatusServiceUnavailable) + return "", false + } + return base, true +} + +// handleWorkflowBundleGet exports one stored model as a verified bundle. +// Any role: bundles contain nothing the library view does not already show. +func (s *Server) handleWorkflowBundleGet(w http.ResponseWriter, r *http.Request) { + st := s.workflowStore(w) + if st == nil { + return + } + b, err := s.exportBundle(st, r.PathValue("name"), r.PathValue("version"), requestUser(r)) + if err != nil { + if errors.Is(err, workflowmodel.ErrCompile) || errors.Is(err, workflowmodel.ErrInvalid) { + jsonError(w, err.Error(), http.StatusUnprocessableEntity) + return + } + workflowStoreError(w, err) + return + } + data, err := bundle.Encode(b) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(data) +} + +func (s *Server) exportBundle(st *wfstore.Store, name, version, by string) (*bundle.Bundle, error) { + m, _, err := st.Get(name, version) + if err != nil { + return nil, err + } + return bundle.Export(m, by, time.Now().UTC()) +} + +// handleWorkflowImport verifies a bundle posted by the client and stores its +// model, recompiling locally. 422 when the bundle does not verify. Owner. +func (s *Server) handleWorkflowImport(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + st := s.workflowStore(w) + if st == nil { + return + } + raw, err := io.ReadAll(io.LimitReader(r.Body, marketplaceMaxBody+1)) + if err != nil || len(raw) > marketplaceMaxBody { + jsonError(w, "bundle too large or unreadable", http.StatusRequestEntityTooLarge) + return + } + b, err := bundle.Decode(raw) + if err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + e, err := s.importBundle(st, b, requestUser(r)) + if err != nil { + var ve *importVerifyError + if errors.As(err, &ve) { + jsonError(w, ve.Error(), http.StatusUnprocessableEntity) + return + } + workflowStoreError(w, err) + return + } + s.auditFromRequest(r, "workflow_import", fmt.Sprintf("%s hash=%s tier=%s", e.Key(), shortHash(b.Hash), b.Tier), "") + jsonStatusResponse(w, http.StatusCreated, map[string]interface{}{"entry": e, "verified": true}) +} + +type importVerifyError struct{ err error } + +func (e *importVerifyError) Error() string { return "bundle verification failed: " + e.err.Error() } +func (e *importVerifyError) Unwrap() error { return e.err } + +// importBundle verifies b, pins the model to the bundle's version when that +// slot is free (otherwise the next integer version) and saves it. The +// store recompiles on Save, so the published profile is never persisted. +func (s *Server) importBundle(st *wfstore.Store, b *bundle.Bundle, by string) (wfstore.Entry, error) { + if err := bundle.Verify(b); err != nil { + return wfstore.Entry{}, &importVerifyError{err} + } + m := b.Model + name := m.Metadata.Name + version := m.Spec.Meta.Version + if wfstore.IsBuiltin(name) { + return wfstore.Entry{}, fmt.Errorf("%w: %s is a built-in model", wfstore.ErrReadOnly, name) + } + if version != "" { + if _, _, err := st.Get(name, version); err == nil { + version = "" + } else if !errors.Is(err, wfstore.ErrNotFound) { + return wfstore.Entry{}, err + } + } + if version == "" { + next, err := nextWorkflowVersion(st, name) + if err != nil { + return wfstore.Entry{}, err + } + version = next + } + m.Spec.Meta.Version = version + if m.Spec.Meta.Provenance == "" { + m.Spec.Meta.Provenance = workflowmodel.Provenance(b.Tier) + } + if m.Spec.Meta.Provenance == workflowmodel.ProvenanceBuiltIn || string(m.Spec.Meta.Provenance) == bundle.TierLocalPublished { + m.Spec.Meta.Provenance = workflowmodel.ProvenanceLocal + } + return st.Save(m, by, time.Now().UTC()) +} + +// hubRequest builds an authenticated spoke→hub request. +func (s *Server) hubRequest(ctx context.Context, method, rawURL string, body []byte) (*http.Request, error) { + var rd io.Reader + if body != nil { + rd = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, rawURL, rd) + if err != nil { + return nil, err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("Accept", "application/json") + if key := spoke.SpokeHeartbeatKey(); key != "" { + req.Header.Set("Authorization", "Bearer "+key) + } + if s.deps != nil && s.deps.Config != nil { + if id := strings.TrimSpace(s.deps.Config.HiveID); id != "" { + req.Header.Set(marketplaceHiveIDHeader, id) + } + } + return req, nil +} + +// handleMarketplaceIndex proxies the hub index with an ETag cache. When the +// hub is unreachable it answers 502 carrying the last cached index (if any) +// and stale:true so the UI can still render. +func (s *Server) handleMarketplaceIndex(w http.ResponseWriter, r *http.Request) { + base, ok := s.requireMarketplace(w) + if !ok { + return + } + ttl := s.deps.Config.Workflow.Marketplace.EffectiveRefreshInterval() + marketplaceCache.mu.Lock() + defer marketplaceCache.mu.Unlock() + c := &marketplaceCache + fresh := c.body != nil && time.Since(c.fetchedAt) < ttl && r.URL.Query().Get("refresh") == "" + if !fresh { + ctx, cancel := context.WithTimeout(r.Context(), marketplaceHubTimeout) + defer cancel() + req, err := s.hubRequest(ctx, http.MethodGet, base+"/api/marketplace/workflows", nil) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + if c.etag != "" { + req.Header.Set("If-None-Match", c.etag) + } + resp, err := marketplaceHTTP.Do(req) + if err != nil { + writeMarketplaceIndex(w, c, true, "hub unreachable: "+err.Error()) + return + } + defer closeHTTPBody(resp.Body) + switch resp.StatusCode { + case http.StatusNotModified: + c.fetchedAt = time.Now() + case http.StatusOK: + body, err := io.ReadAll(io.LimitReader(resp.Body, marketplaceMaxBody+1)) + if err != nil || len(body) > marketplaceMaxBody || !json.Valid(body) { + writeMarketplaceIndex(w, c, true, "hub index unreadable") + return + } + c.body = body + c.etag = resp.Header.Get("ETag") + c.fetchedAt = time.Now() + default: + writeMarketplaceIndex(w, c, true, fmt.Sprintf("hub returned HTTP %d", resp.StatusCode)) + return + } + } + writeMarketplaceIndex(w, c, false, "") +} + +// writeMarketplaceIndex wraps the cached hub index as +// {index, stale, etag, fetchedAt, error?}. +func writeMarketplaceIndex(w http.ResponseWriter, c *marketplaceIndexCache, stale bool, errMsg string) { + out := map[string]interface{}{"stale": stale} + if c.body != nil { + out["index"] = json.RawMessage(c.body) + out["etag"] = c.etag + out["fetchedAt"] = c.fetchedAt.UTC() + } else { + out["index"] = bundle.Index{Entries: []bundle.IndexEntry{}} + } + if errMsg != "" { + out["error"] = errMsg + } + if stale { + jsonStatusResponse(w, marketplaceStatusStale, out) + return + } + jsonResponse(w, out) +} + +// fetchHubBundle downloads name@version from the hub and decodes it. +func (s *Server) fetchHubBundle(ctx context.Context, base, name, version string) (*bundle.Bundle, int, error) { + u := base + "/api/marketplace/workflows/" + url.PathEscape(name) + "/" + url.PathEscape(version) + "/bundle" + req, err := s.hubRequest(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, http.StatusInternalServerError, err + } + resp, err := marketplaceHTTP.Do(req) + if err != nil { + return nil, http.StatusBadGateway, fmt.Errorf("hub unreachable: %w", err) + } + defer closeHTTPBody(resp.Body) + if resp.StatusCode == http.StatusNotFound { + return nil, http.StatusNotFound, fmt.Errorf("%s@%s not in marketplace", name, version) + } + if resp.StatusCode != http.StatusOK { + return nil, http.StatusBadGateway, fmt.Errorf("hub returned HTTP %d", resp.StatusCode) + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, marketplaceMaxBody+1)) + if err != nil || len(raw) > marketplaceMaxBody { + return nil, http.StatusBadGateway, errors.New("hub bundle unreadable or too large") + } + b, err := bundle.Decode(raw) + if err != nil { + return nil, http.StatusBadGateway, fmt.Errorf("hub bundle invalid: %w", err) + } + return b, http.StatusOK, nil +} + +// handleMarketplaceImport fetches a catalogue bundle and imports it. Owner. +func (s *Server) handleMarketplaceImport(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + base, ok := s.requireMarketplace(w) + if !ok { + return + } + st := s.workflowStore(w) + if st == nil { + return + } + name, version := r.PathValue("name"), r.PathValue("version") + ctx, cancel := context.WithTimeout(r.Context(), marketplaceHubTimeout) + defer cancel() + b, code, err := s.fetchHubBundle(ctx, base, name, version) + if err != nil { + jsonError(w, err.Error(), code) + return + } + if b.Model.Metadata.Name != name || b.Model.Spec.Meta.Version != version { + jsonError(w, fmt.Sprintf("hub returned %s for %s@%s", b.Key(), name, version), http.StatusBadGateway) + return + } + e, err := s.importBundle(st, b, requestUser(r)) + if err != nil { + var ve *importVerifyError + if errors.As(err, &ve) { + jsonError(w, ve.Error(), http.StatusUnprocessableEntity) + return + } + workflowStoreError(w, err) + return + } + s.auditFromRequest(r, "workflow_import", fmt.Sprintf("%s from marketplace %s@%s hash=%s tier=%s", e.Key(), name, version, shortHash(b.Hash), b.Tier), "") + jsonStatusResponse(w, http.StatusCreated, map[string]interface{}{"entry": e, "verified": true, "source": name + "@" + version}) +} + +// marketplacePublishRequest is the body of POST /api/marketplace/workflows/publish. +type marketplacePublishRequest struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// handleMarketplacePublish exports a library entry and posts it to the hub +// with the spoke's heartbeat credential. Owner; audited. +func (s *Server) handleMarketplacePublish(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + base, ok := s.requireMarketplace(w) + if !ok { + return + } + st := s.workflowStore(w) + if st == nil { + return + } + var req marketplacePublishRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + req.Name, req.Version = strings.TrimSpace(req.Name), strings.TrimSpace(req.Version) + if req.Name == "" || req.Version == "" { + jsonError(w, "name and version are required", http.StatusBadRequest) + return + } + b, err := s.exportBundle(st, req.Name, req.Version, requestUser(r)) + if err != nil { + if errors.Is(err, workflowmodel.ErrCompile) || errors.Is(err, workflowmodel.ErrInvalid) { + jsonError(w, err.Error(), http.StatusUnprocessableEntity) + return + } + workflowStoreError(w, err) + return + } + // A spoke publishes from its own library: the hub records such bundles + // as local-published whatever the model's provenance says. + if b.Tier != bundle.TierIngestedUnreviewed { + b.Tier = bundle.TierLocalPublished + } + payload, err := bundle.Encode(b) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + ctx, cancel := context.WithTimeout(r.Context(), marketplaceHubTimeout) + defer cancel() + hreq, err := s.hubRequest(ctx, http.MethodPost, base+"/api/marketplace/workflows", payload) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + resp, err := marketplaceHTTP.Do(hreq) + if err != nil { + jsonError(w, "hub unreachable: "+err.Error(), http.StatusBadGateway) + return + } + defer closeHTTPBody(resp.Body) + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + code := http.StatusBadGateway + switch resp.StatusCode { + case http.StatusConflict: + code = http.StatusConflict + case http.StatusUnauthorized, http.StatusForbidden: + code = http.StatusForbidden + case http.StatusUnprocessableEntity, http.StatusBadRequest: + code = http.StatusUnprocessableEntity + } + jsonError(w, fmt.Sprintf("hub refused publish (HTTP %d): %s", resp.StatusCode, strings.TrimSpace(hubErrorMessage(msg))), code) + return + } + marketplaceCache.mu.Lock() + marketplaceCache.fetchedAt = time.Time{} + marketplaceCache.mu.Unlock() + s.auditFromRequest(r, "workflow_publish", fmt.Sprintf("%s hash=%s tier=%s to %s", b.Key(), shortHash(b.Hash), b.Tier, base), "") + jsonStatusResponse(w, http.StatusCreated, map[string]interface{}{"published": b.Key(), "hash": b.Hash, "tier": b.Tier}) +} + +// handleMarketplaceRefresh asks the hub to re-ingest its corpus (or one +// ?repo=). Owner on the spoke; the hub applies its own admin policy and may +// refuse, which is surfaced as 403. +func (s *Server) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + base, ok := s.requireMarketplace(w) + if !ok { + return + } + u := base + "/api/marketplace/refresh" + if repo := strings.TrimSpace(r.URL.Query().Get("repo")); repo != "" { + u += "?repo=" + url.QueryEscape(repo) + } + ctx, cancel := context.WithTimeout(r.Context(), marketplaceHubTimeout) + defer cancel() + hreq, err := s.hubRequest(ctx, http.MethodPost, u, []byte("{}")) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + resp, err := marketplaceHTTP.Do(hreq) + if err != nil { + jsonError(w, "hub unreachable: "+err.Error(), http.StatusBadGateway) + return + } + defer closeHTTPBody(resp.Body) + body, _ := io.ReadAll(io.LimitReader(resp.Body, marketplaceMaxBody)) + switch { + case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden: + jsonError(w, "hub refused refresh: "+strings.TrimSpace(hubErrorMessage(body)), http.StatusForbidden) + return + case resp.StatusCode >= 300: + jsonError(w, fmt.Sprintf("hub refresh failed (HTTP %d): %s", resp.StatusCode, strings.TrimSpace(hubErrorMessage(body))), http.StatusBadGateway) + return + } + marketplaceCache.mu.Lock() + marketplaceCache.fetchedAt = time.Time{} + marketplaceCache.mu.Unlock() + s.auditFromRequest(r, "workflow_marketplace_refresh", "requested hub refresh "+r.URL.Query().Get("repo"), "") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(resp.StatusCode) + if json.Valid(body) { + _, _ = w.Write(body) + } else { + _, _ = w.Write([]byte(`{"accepted":true}`)) + } +} + +// hubErrorMessage extracts {"error": "..."} from a hub reply, else the raw text. +func hubErrorMessage(body []byte) string { + var e struct { + Error string `json:"error"` + } + if json.Unmarshal(body, &e) == nil && e.Error != "" { + return e.Error + } + return string(body) +} + +func shortHash(h string) string { + if len(h) > 12 { + return h[:12] + } + return h +} diff --git a/src/pkg/dashboard/api_workflows_marketplace_test.go b/src/pkg/dashboard/api_workflows_marketplace_test.go new file mode 100644 index 0000000000..875fe0b7d0 --- /dev/null +++ b/src/pkg/dashboard/api_workflows_marketplace_test.go @@ -0,0 +1,440 @@ +package dashboard + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" + wfstore "github.com/hivecommons/hive/pkg/workflowmodel/store" +) + +func resetMarketplaceCache(t *testing.T) { + t.Helper() + clear := func() { + marketplaceCache.mu.Lock() + marketplaceCache.body, marketplaceCache.etag, marketplaceCache.fetchedAt = nil, "", time.Time{} + marketplaceCache.mu.Unlock() + } + clear() + t.Cleanup(clear) +} + +func saveTechdocs(t *testing.T, s *Server) { + t.Helper() + m := techdocsModel(t) + if _, err := s.deps.WorkflowStore.Save(m, "andy", time.Now()); err != nil { + t.Fatal(err) + } +} + +// uncompilableTechdocs validates (so the store accepts it) but does not +// compile: its first gate names no releasing role. +func uncompilableTechdocs(t *testing.T) *workflowmodel.Model { + t.Helper() + m := techdocsModel(t) + m.Spec.Meta.Version = "9" + m.Spec.Gates[0].ReleasedBy = "" + return m +} + +func marketplaceServer(t *testing.T, hubURL string) *Server { + t.Helper() + resetMarketplaceCache(t) + s := workflowTestServer(t, true) + s.deps.Config.Workflow.Marketplace.URL = hubURL + s.deps.Config.HiveID = "spoke-1" + t.Setenv("HIVE_HUB_SECRET", "test-hub-secret-marketplace") + t.Setenv("HIVE_ID", "spoke-1") + return s +} + +// fakeHub is a minimal marketplace hub: one bundle, an ETag'd index, and a +// recorder of what the spoke sent. +type fakeHub struct { + t *testing.T + bundle []byte + index []byte + hits atomic.Int32 + notMod atomic.Int32 + published [][]byte + lastAuth string + lastHive string + refreshes atomic.Int32 + publishRC int +} + +func newFakeHub(t *testing.T) (*fakeHub, *httptest.Server) { + t.Helper() + b, err := bundle.Export(techdocsModel(t), "hub", time.Now()) + if err != nil { + t.Fatal(err) + } + b.Tier = bundle.TierCurated + raw, _ := bundle.Encode(b) + e := bundle.EntryFor(b, time.Now(), "hub", "") + idx, _ := json.Marshal(bundle.Index{GeneratedAt: time.Now(), Entries: []bundle.IndexEntry{e}}) + h := &fakeHub{t: t, bundle: raw, index: idx, publishRC: http.StatusCreated} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h.lastAuth = r.Header.Get("Authorization") + h.lastHive = r.Header.Get(marketplaceHiveIDHeader) + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/marketplace/workflows": + h.hits.Add(1) + if r.Header.Get("If-None-Match") == `"idx-1"` { + h.notMod.Add(1) + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", `"idx-1"`) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(h.index) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/bundle"): + if !strings.Contains(r.URL.Path, "/cncf-techdocs/1/") { + http.Error(w, `{"error":"not found"}`, http.StatusNotFound) + return + } + _, _ = w.Write(h.bundle) + case r.Method == http.MethodPost && r.URL.Path == "/api/marketplace/workflows": + body, _ := io.ReadAll(r.Body) + h.published = append(h.published, body) + w.WriteHeader(h.publishRC) + _, _ = w.Write([]byte(`{"error":"nope"}`)) + case r.Method == http.MethodPost && r.URL.Path == "/api/marketplace/refresh": + h.refreshes.Add(1) + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"accepted":true,"repo":"` + r.URL.Query().Get("repo") + `"}`)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return h, srv +} + +func TestWorkflowBundleGetAndImportRoundTrip(t *testing.T) { + s := marketplaceServer(t, "") + saveTechdocs(t, s) + + rec := wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/1/bundle", nil, false) + if rec.Code != http.StatusOK { + t.Fatalf("bundle get: %d %s", rec.Code, rec.Body.String()) + } + b, err := bundle.Decode(rec.Body.Bytes()) + if err != nil || bundle.Verify(b) != nil { + t.Fatalf("exported bundle does not verify: %v", err) + } + if rec := wfReq(s, http.MethodGet, "/api/workflows/missing/1/bundle", nil, false); rec.Code != http.StatusNotFound { + t.Fatalf("missing: %d", rec.Code) + } + + // Import into a fresh store: version 1 is free so it stays pinned. + s2 := marketplaceServer(t, "") + rec = wfReq(s2, http.MethodPost, "/api/workflows/import", rec.Body.String(), true) + if rec.Code != http.StatusCreated { + t.Fatalf("import: %d %s", rec.Code, rec.Body.String()) + } + var out struct { + Entry wfstore.Entry `json:"entry"` + Verified bool `json:"verified"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if !out.Verified || out.Entry.Key() != "cncf-techdocs@1" || !out.Entry.Compiled { + t.Fatalf("import result = %+v", out) + } + // Same bundle again: version 1 is taken, so it lands at 2. + raw, _ := bundle.Encode(b) + rec = wfReq(s2, http.MethodPost, "/api/workflows/import", string(raw), true) + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if rec.Code != http.StatusCreated || out.Entry.Version != "2" { + t.Fatalf("second import: %d %+v", rec.Code, out.Entry) + } + // Non-owner, bad JSON, tampered. + if rec := wfReq(s2, http.MethodPost, "/api/workflows/import", string(raw), false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner import: %d", rec.Code) + } + if rec := wfReq(s2, http.MethodPost, "/api/workflows/import", "{", true); rec.Code != http.StatusBadRequest { + t.Fatalf("bad json: %d", rec.Code) + } + b.Model.Spec.Meta.Description = "tampered" + raw, _ = bundle.Encode(b) + rec = wfReq(s2, http.MethodPost, "/api/workflows/import", string(raw), true) + if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "verification failed") { + t.Fatalf("tampered import: %d %s", rec.Code, rec.Body.String()) + } + // Built-in names are refused. + m := techdocsModel(t) + m.Metadata.Name = "hive-default" + bb, _ := bundle.Export(m, "x", time.Now()) + raw, _ = bundle.Encode(bb) + if rec := wfReq(s2, http.MethodPost, "/api/workflows/import", string(raw), true); rec.Code != http.StatusForbidden { + t.Fatalf("built-in import: %d %s", rec.Code, rec.Body.String()) + } + // Without a store. + noStore := workflowTestServer(t, false) + if rec := wfReq(noStore, http.MethodPost, "/api/workflows/import", string(raw), true); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no store: %d", rec.Code) + } + if rec := wfReq(noStore, http.MethodGet, "/api/workflows/x/1/bundle", nil, false); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no store get: %d", rec.Code) + } +} + +func TestWorkflowBundleGetRefusesUncompilable(t *testing.T) { + s := marketplaceServer(t, "") + m := uncompilableTechdocs(t) + if _, err := s.deps.WorkflowStore.Save(m, "andy", time.Now()); err != nil { + t.Fatal(err) + } + if rec := wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/9/bundle", nil, false); rec.Code != http.StatusUnprocessableEntity { + t.Fatalf("uncompilable export: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestMarketplaceIndexProxyCachesWithETag(t *testing.T) { + hub, srv := newFakeHub(t) + s := marketplaceServer(t, srv.URL) + + rec := wfReq(s, http.MethodGet, "/api/marketplace/workflows", nil, false) + if rec.Code != http.StatusOK { + t.Fatalf("index: %d %s", rec.Code, rec.Body.String()) + } + var out struct { + Stale bool `json:"stale"` + ETag string `json:"etag"` + Index bundle.Index `json:"index"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if out.Stale || out.ETag != `"idx-1"` || len(out.Index.Entries) != 1 || out.Index.Entries[0].Tier != bundle.TierCurated { + t.Fatalf("index out = %+v", out) + } + if hub.lastAuth == "" || !strings.HasPrefix(hub.lastAuth, "Bearer ") || hub.lastHive != "spoke-1" { + t.Fatalf("spoke credentials not sent: auth=%q hive=%q", hub.lastAuth, hub.lastHive) + } + // Within TTL: served from cache, no hub hit. + wfReq(s, http.MethodGet, "/api/marketplace/workflows", nil, false) + if hub.hits.Load() != 1 { + t.Fatalf("expected cached, hub hits=%d", hub.hits.Load()) + } + // ?refresh=1 revalidates with If-None-Match → 304. + rec = wfReq(s, http.MethodGet, "/api/marketplace/workflows?refresh=1", nil, false) + if rec.Code != http.StatusOK || hub.notMod.Load() != 1 { + t.Fatalf("revalidate: %d notMod=%d", rec.Code, hub.notMod.Load()) + } + // Hub down → 502 with cached index and stale:true. + srv.Close() + rec = wfReq(s, http.MethodGet, "/api/marketplace/workflows?refresh=1", nil, false) + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if rec.Code != http.StatusBadGateway || !out.Stale || len(out.Index.Entries) != 1 { + t.Fatalf("stale: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestMarketplaceIndexProxyErrors(t *testing.T) { + // Hub returns 500 with an empty cache → 502, empty index, stale. + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer bad.Close() + s := marketplaceServer(t, bad.URL) + rec := wfReq(s, http.MethodGet, "/api/marketplace/workflows", nil, false) + if rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), `"stale":true`) || !strings.Contains(rec.Body.String(), `"entries":[]`) { + t.Fatalf("hub 500: %d %s", rec.Code, rec.Body.String()) + } + // Disabled → 404. + off := false + s.deps.Config.Workflow.Marketplace.Enabled = &off + if rec := wfReq(s, http.MethodGet, "/api/marketplace/workflows", nil, false); rec.Code != http.StatusNotFound { + t.Fatalf("disabled: %d", rec.Code) + } + s.deps.Config.Workflow.Marketplace.Enabled = nil + // No usable URL → 503. + s.deps.Config.Workflow.Marketplace.URL = "not a url" + s.deps.Config.Hub.URL = "" + if rec := wfReq(s, http.MethodGet, "/api/marketplace/workflows", nil, false); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no url: %d %s", rec.Code, rec.Body.String()) + } + // Garbage index body with empty cache → stale 502. + resetMarketplaceCache(t) + bad2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })) + defer bad2.Close() + s.deps.Config.Workflow.Marketplace.URL = bad2.URL + if rec := wfReq(s, http.MethodGet, "/api/marketplace/workflows", nil, false); rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "unreadable") { + t.Fatalf("garbage: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestMarketplaceImportFromHub(t *testing.T) { + hub, srv := newFakeHub(t) + s := marketplaceServer(t, srv.URL) + + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/cncf-techdocs/1/import", nil, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner: %d", rec.Code) + } + rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/cncf-techdocs/1/import", nil, true) + if rec.Code != http.StatusCreated { + t.Fatalf("import: %d %s", rec.Code, rec.Body.String()) + } + var out struct { + Entry wfstore.Entry `json:"entry"` + Source string `json:"source"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if out.Entry.Key() != "cncf-techdocs@1" || out.Source != "cncf-techdocs@1" { + t.Fatalf("out = %+v", out) + } + m, _, err := s.deps.WorkflowStore.Get("cncf-techdocs", "1") + if err != nil || m.Spec.Meta.Provenance != workflowmodel.ProvenanceCurated { + t.Fatalf("imported provenance = %v %v", m.Spec.Meta.Provenance, err) + } + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/nothere/1/import", nil, true); rec.Code != http.StatusNotFound { + t.Fatalf("missing on hub: %d %s", rec.Code, rec.Body.String()) + } + // Re-import lands on the next free version. + rec = wfReq(s, http.MethodPost, "/api/marketplace/workflows/cncf-techdocs/1/import", nil, true) + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if rec.Code != http.StatusCreated || out.Entry.Version != "2" { + t.Fatalf("re-import: %d %+v", rec.Code, out.Entry) + } + // Tampered bundle on hub → 422. + b, _ := bundle.Decode(hub.bundle) + b.Profile.Columns = nil + hub.bundle, _ = bundle.Encode(b) + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/cncf-techdocs/1/import", nil, true); rec.Code != http.StatusUnprocessableEntity { + t.Fatalf("tampered hub bundle: %d %s", rec.Code, rec.Body.String()) + } + // Invalid bundle JSON on hub → 502. + hub.bundle = []byte("{") + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/cncf-techdocs/1/import", nil, true); rec.Code != http.StatusBadGateway { + t.Fatalf("invalid hub bundle: %d", rec.Code) + } + // Hub down → 502. + srv.Close() + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/cncf-techdocs/1/import", nil, true); rec.Code != http.StatusBadGateway { + t.Fatalf("hub down: %d", rec.Code) + } +} + +func TestMarketplaceImportKeyMismatch(t *testing.T) { + // A hub that answers every bundle path with the same bundle. + b, _ := bundle.Export(techdocsModel(t), "hub", time.Now()) + raw, _ := bundle.Encode(b) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(raw) })) + defer srv.Close() + s := marketplaceServer(t, srv.URL) + rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/other/7/import", nil, true) + if rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "hub returned cncf-techdocs@1") { + t.Fatalf("mismatch: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestMarketplacePublish(t *testing.T) { + hub, srv := newFakeHub(t) + s := marketplaceServer(t, srv.URL) + saveTechdocs(t, s) + + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": "cncf-techdocs", "version": "1"}, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner: %d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": ""}, true); rec.Code != http.StatusBadRequest { + t.Fatalf("missing fields: %d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", "{", true); rec.Code != http.StatusBadRequest { + t.Fatalf("bad json: %d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": "nope", "version": "1"}, true); rec.Code != http.StatusNotFound { + t.Fatalf("unknown entry: %d", rec.Code) + } + rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": "cncf-techdocs", "version": "1"}, true) + if rec.Code != http.StatusCreated || !strings.Contains(rec.Body.String(), `"published":"cncf-techdocs@1"`) { + t.Fatalf("publish: %d %s", rec.Code, rec.Body.String()) + } + if len(hub.published) != 1 { + t.Fatalf("hub received %d bundles", len(hub.published)) + } + sent, err := bundle.Decode(hub.published[0]) + if err != nil || bundle.Verify(sent) != nil || sent.Tier != bundle.TierLocalPublished { + t.Fatalf("sent bundle = %v tier=%q", err, sent.Tier) + } + if !strings.HasPrefix(hub.lastAuth, "Bearer ") || hub.lastHive != "spoke-1" { + t.Fatalf("credentials: %q %q", hub.lastAuth, hub.lastHive) + } + // Hub refusals map onto spoke statuses. + for hubRC, want := range map[int]int{ + http.StatusConflict: http.StatusConflict, + http.StatusUnauthorized: http.StatusForbidden, + http.StatusForbidden: http.StatusForbidden, + http.StatusUnprocessableEntity: http.StatusUnprocessableEntity, + http.StatusInternalServerError: http.StatusBadGateway, + } { + hub.publishRC = hubRC + rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": "cncf-techdocs", "version": "1"}, true) + if rec.Code != want || !strings.Contains(rec.Body.String(), "nope") { + t.Fatalf("hub %d → %d (want %d): %s", hubRC, rec.Code, want, rec.Body.String()) + } + } + // Uncompilable entry → 422 before any hub call. + if _, err := s.deps.WorkflowStore.Save(uncompilableTechdocs(t), "andy", time.Now()); err != nil { + t.Fatal(err) + } + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": "cncf-techdocs", "version": "9"}, true); rec.Code != http.StatusUnprocessableEntity { + t.Fatalf("uncompilable publish: %d", rec.Code) + } + srv.Close() + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": "cncf-techdocs", "version": "1"}, true); rec.Code != http.StatusBadGateway { + t.Fatalf("hub down: %d", rec.Code) + } +} + +func TestMarketplaceRefreshProxy(t *testing.T) { + hub, srv := newFakeHub(t) + s := marketplaceServer(t, srv.URL) + if rec := wfReq(s, http.MethodPost, "/api/marketplace/refresh", nil, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner: %d", rec.Code) + } + rec := wfReq(s, http.MethodPost, "/api/marketplace/refresh?repo=org/repo", nil, true) + if rec.Code != http.StatusAccepted || !strings.Contains(rec.Body.String(), `"repo":"org/repo"`) || hub.refreshes.Load() != 1 { + t.Fatalf("refresh: %d %s", rec.Code, rec.Body.String()) + } + deny := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.RawQuery, "boom") { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("plain text failure")) + return + } + http.Error(w, `{"error":"admin only"}`, http.StatusForbidden) + })) + defer deny.Close() + s.deps.Config.Workflow.Marketplace.URL = deny.URL + rec = wfReq(s, http.MethodPost, "/api/marketplace/refresh", nil, true) + if rec.Code != http.StatusForbidden || !strings.Contains(rec.Body.String(), "admin only") { + t.Fatalf("denied: %d %s", rec.Code, rec.Body.String()) + } + rec = wfReq(s, http.MethodPost, "/api/marketplace/refresh?repo=boom", nil, true) + if rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "plain text failure") { + t.Fatalf("hub 500: %d %s", rec.Code, rec.Body.String()) + } + srv.Close() + s.deps.Config.Workflow.Marketplace.URL = srv.URL + if rec := wfReq(s, http.MethodPost, "/api/marketplace/refresh", nil, true); rec.Code != http.StatusBadGateway { + t.Fatalf("hub down: %d", rec.Code) + } +} + +func TestMarketplaceBaseFallsBackToHubURL(t *testing.T) { + s := marketplaceServer(t, "") + s.deps.Config.Hub.URL = "https://hub.example/" + if got := s.marketplaceBase(); got != "https://hub.example" { + t.Fatalf("base = %q", got) + } + bare := &Server{} + if !bare.marketplaceEnabled() || bare.marketplaceBase() != "" { + t.Fatal("server without deps should be enabled but have no base") + } +} diff --git a/src/pkg/dashboard/api_workflows_simulate.go b/src/pkg/dashboard/api_workflows_simulate.go new file mode 100644 index 0000000000..7cd01141e2 --- /dev/null +++ b/src/pkg/dashboard/api_workflows_simulate.go @@ -0,0 +1,469 @@ +package dashboard + +import ( + "context" + "fmt" + "net/http" + "sort" + "strings" + "time" + + ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/stageengine" + "github.com/hivecommons/hive/pkg/stagelabels" + "github.com/hivecommons/hive/pkg/workflowmodel" +) + +// Workflow-model simulate (hivecommons/hive#17620, RFC phase 3c): "paste or +// pick a real issue/PR and see which state the model puts it in and why, +// using the same Facts the engine would use". Read-only, open to any +// dashboard user like GET on the model. + +// workflowSimulateTimeout bounds the forge reads behind a repo+number fill. +const workflowSimulateTimeout = 30 * time.Second + +// workflowSimulateRequest is the body of POST .../simulate. Facts is the +// JSON mirror of stageengine.Facts; when it is absent and Repo+Number are +// set the handler fills what one forge read can tell and marks the rest +// unknown. When both are present Facts wins and the forge is not read. +type workflowSimulateRequest struct { + Kind string `json:"kind"` + Facts *SimulateFacts `json:"facts,omitempty"` + Repo string `json:"repo,omitempty"` + Number int `json:"number,omitempty"` +} + +// SimulateFacts is the wire form of stageengine.Facts. Every field is +// optional; the zero value is a closed, unlabeled item by an unknown +// author with no kind-specific evidence. Scalar enumerations use the +// rules-vocabulary spellings (author.kind, pr.checks, pr.review, pr.dco in +// pkg/workflowmodel.Facts) so what the operator types is what a rule tests. +type SimulateFacts struct { + Labels []string `json:"labels"` + Open bool `json:"open"` + Merged bool `json:"merged"` + Draft bool `json:"draft"` + Author SimulateAuthor `json:"author"` + Park SimulatePark `json:"park"` + Origin string `json:"origin,omitempty"` + Issue *SimulateIssueFacts `json:"issue,omitempty"` + PR *SimulatePRFacts `json:"pr,omitempty"` +} + +// SimulateAuthor mirrors stageengine.AuthorFacts. +type SimulateAuthor struct { + Login string `json:"login,omitempty"` + Kind string `json:"kind,omitempty"` // hive | clanker | human | unknown + SelfAuthored bool `json:"selfAuthored,omitempty"` + Role string `json:"role,omitempty"` // trusted | contributor | newcomer | untrusted | "" +} + +// SimulatePark mirrors stageengine.ParkFacts. +type SimulatePark struct { + Holds []string `json:"holds,omitempty"` + NeedsHuman bool `json:"needsHuman,omitempty"` + DecisionLabel string `json:"decisionLabel,omitempty"` + PossiblyExists bool `json:"possiblyExists,omitempty"` + Sentinel bool `json:"sentinel,omitempty"` + HumanOnlyReason string `json:"humanOnlyReason,omitempty"` + ApprovedDirection bool `json:"approvedDirection,omitempty"` + DecisionQuestion string `json:"decisionQuestion,omitempty"` + DecisionSuperseded bool `json:"decisionSuperseded,omitempty"` +} + +// SimulateIssueFacts mirrors the issue-side stageengine.IssueFacts the +// derivers and the rules vocabulary read. +type SimulateIssueFacts struct { + Bucket string `json:"bucket,omitempty"` // actionable | claimed | in_progress | claimed_by_pr | filtered | reporter_triage + FilteredReason string `json:"filteredReason,omitempty"` + Claimed bool `json:"claimed,omitempty"` + ClaimHolder string `json:"claimHolder,omitempty"` + ClaimPRStarted bool `json:"claimPRStarted,omitempty"` + LikelyDone bool `json:"likelyDone,omitempty"` + Stalled bool `json:"stalled,omitempty"` + CoveringPR int `json:"coveringPR,omitempty"` + CoveringPRStage int `json:"coveringPRStage,omitempty"` + OpenChildren int `json:"openChildren,omitempty"` + HoldLabel string `json:"holdLabel,omitempty"` + BlockedReason string `json:"blockedReason,omitempty"` + NeedsHumanLabel string `json:"needsHumanLabel,omitempty"` + Blocker string `json:"blocker,omitempty"` + ApprovedDirectionLabel string `json:"approvedDirectionLabel,omitempty"` + UmbrellaLabel string `json:"umbrellaLabel,omitempty"` + DiscussionLabel string `json:"discussionLabel,omitempty"` + TriageAwaiting string `json:"triageAwaiting,omitempty"` + TriageAwaitingLabel string `json:"triageAwaitingLabel,omitempty"` +} + +// SimulatePRFacts mirrors the PR-side stageengine.PRFacts the derivers and +// the rules vocabulary read. Lane is optional; without one the built-in +// deriver is fed stageengine.SimulatedLane and the response says so. +type SimulatePRFacts struct { + HeadSHA string `json:"headSHA,omitempty"` + HeadFresh bool `json:"headFresh,omitempty"` + BaseRef string `json:"baseRef,omitempty"` + Conflicting bool `json:"conflicting,omitempty"` + ChangesRequested bool `json:"changesRequested,omitempty"` + Checks string `json:"checks,omitempty"` // green | red | pending | missing | unknown + Review string `json:"review,omitempty"` // pending | approved | changes_requested | requires_human | unavailable + DCO string `json:"dco,omitempty"` // yes | no | unknown + DCOGate bool `json:"dcoGate,omitempty"` + HumanApprover string `json:"humanApprover,omitempty"` + Lane *SimulateLane `json:"lane,omitempty"` +} + +// SimulateLane is one automerge lane opinion (stageengine.LaneFact). +type SimulateLane struct { + Lane string `json:"lane"` + Reason string `json:"reason,omitempty"` +} + +// SimulateDecision is the wire form of the stageengine.Decision. +type SimulateDecision struct { + State string `json:"state"` + Column string `json:"column"` + Stage int `json:"stage"` + Label string `json:"label"` + Reason string `json:"reason"` + Abstain bool `json:"abstain"` + Covered bool `json:"covered"` + Terminal bool `json:"terminal"` +} + +// SimulateResponse is the body of a 200 from .../simulate. +type SimulateResponse struct { + Decision SimulateDecision `json:"decision"` + Trace []stageengine.RuleTrace `json:"trace"` + Facts SimulateFacts `json:"facts"` + Filled []string `json:"filled"` + Unknown []string `json:"unknown"` + Notes []string `json:"notes,omitempty"` +} + +func (s *Server) handleWorkflowSimulate(w http.ResponseWriter, r *http.Request) { + st := s.workflowStore(w) + if st == nil { + return + } + var req workflowSimulateRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + req.Kind = strings.ToLower(strings.TrimSpace(req.Kind)) + req.Repo = strings.TrimSpace(req.Repo) + m, _, err := st.Get(r.PathValue("name"), r.PathValue("version")) + if err != nil { + workflowStoreError(w, err) + return + } + profile, diags, cerr := workflowmodel.Compile(m) + if cerr != nil { + jsonError(w, fmt.Sprintf("%s@%s does not compile: %d diagnostics", m.Metadata.Name, m.Spec.Meta.Version, len(diags)), http.StatusUnprocessableEntity) + return + } + + var filled, unknown, notes []string + facts := req.Facts + if facts == nil { + if req.Repo == "" || req.Number <= 0 { + jsonError(w, "set facts, or repo (owner/repo) and number to fetch them", http.StatusBadRequest) + return + } + if !strings.Contains(req.Repo, "/") { + jsonError(w, "repo must be owner/repo", http.StatusBadRequest) + return + } + if s.deps == nil || s.deps.WorkflowItemSnapshot == nil { + jsonError(w, "fetching an item needs a forge client; set facts by hand instead", http.StatusUnprocessableEntity) + return + } + ctx, cancel := context.WithTimeout(r.Context(), workflowSimulateTimeout) + defer cancel() + snap, err := s.deps.WorkflowItemSnapshot(ctx, req.Repo, req.Number) + if err != nil { + jsonError(w, fmt.Sprintf("fetch %s#%d: %v", req.Repo, req.Number, err), http.StatusBadGateway) + return + } + if req.Kind == "" { + req.Kind = snap.Kind + } else if req.Kind != snap.Kind { + jsonError(w, fmt.Sprintf("%s#%d is a %s, not a %s", req.Repo, req.Number, snap.Kind, req.Kind), http.StatusBadRequest) + return + } + facts, filled, unknown = simulateFactsFromSnapshot(snap) + } else if req.Kind == "" { + jsonError(w, "kind must be issue or pr", http.StatusBadRequest) + return + } + + owner, repo := "sim", "sim" + if i := strings.IndexByte(req.Repo, '/'); i > 0 { + owner, repo = req.Repo[:i], req.Repo[i+1:] + } + ref, err := stageengine.SimulateRef(req.Kind, owner, repo, req.Number) + if err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + norm := normalizeSimulateFacts(req.Kind, *facts) + ef := norm.toEngine(ref) + if ref.Kind == stagelabels.PRItem && len(ef.PR.Lanes) == 0 { + lane := stageengine.SimulatedLane(ef) + ef.PR.Lanes = []stageengine.LaneFact{lane} + norm.PR.Lane = &SimulateLane{Lane: lane.Lane, Reason: lane.Reason} + notes = append(notes, fmt.Sprintf("no lane observation supplied; simulated lane %s with reason %q from the head facts", lane.Lane, lane.Reason)) + } + dec, trace := stageengine.Explain(profile, ref.Kind, ef) + if trace == nil { + trace = []stageengine.RuleTrace{} + } + if filled == nil { + filled = []string{} + } + if unknown == nil { + unknown = []string{} + } + jsonResponse(w, SimulateResponse{ + Decision: simulateDecision(profile, dec), + Trace: trace, + Facts: norm, + Filled: filled, + Unknown: unknown, + Notes: notes, + }) +} + +// simulateDecision maps the engine's Decision onto the profile: the state +// that carries the stage number and its column. +func simulateDecision(p *workflowmodel.Profile, d stageengine.Decision) SimulateDecision { + out := SimulateDecision{Reason: d.Reason, Abstain: d.Abstain, Terminal: d.Terminal} + if d.Abstain { + out.Covered = !strings.HasPrefix(d.Reason, stageengine.ReasonNotCovered) + return out + } + out.Covered = true + if d.Stage != stagelabels.None { + out.Stage = d.Stage.Number() + out.Label = d.Stage.Label() + if st, ok := p.StateForNumber(out.Stage); ok { + out.State = st + out.Column = p.ColumnOf[st] + } + } + return out +} + +// simulateFactsFromSnapshot fills a SimulateFacts from one forge read and +// names what it could and could not fill. +func simulateFactsFromSnapshot(snap ghpkg.WorkflowItemSnapshot) (*SimulateFacts, []string, []string) { + f := &SimulateFacts{Labels: snap.Labels, Open: snap.Open, Merged: snap.Merged, Draft: snap.Draft} + if f.Labels == nil { + f.Labels = []string{} + } + f.Author.Login = snap.Author + f.Author.Kind = "unknown" + f.Author.Role = stagelabels.RoleOf(snap.Labels).String() + f.Park.Holds = stagelabels.Holds(snap.Labels) + f.Park.NeedsHuman = hasLabelFold(snap.Labels, stageengine.NeedsHumanLabel) + f.Origin = originLabel(snap.Labels) + filled := []string{"labels", "open", "author.login", "author.role", "park.holds", "park.needsHuman", "origin"} + unknown := []string{"author.kind"} + if snap.Kind == "pr" { + f.PR = &SimulatePRFacts{HeadSHA: snap.HeadSHA, BaseRef: snap.BaseRef, Review: "pending", DCO: "unknown", Checks: "unknown"} + filled = append(filled, "merged", "draft", "pr.headSHA", "pr.baseRef") + if snap.ConflictKnown { + f.PR.Conflicting = snap.Conflicting + filled = append(filled, "pr.conflicting") + } else { + unknown = append(unknown, "pr.conflicting") + } + if snap.ChecksKnown { + f.PR.Checks = stageengine.ChecksName(stagelabels.CheckSummary{Known: true, Total: snap.Checks.Total, Green: snap.Checks.Green, Pending: snap.Checks.Pending, Red: snap.Checks.Red, Missing: snap.Checks.MissingRequired}) + switch snap.Checks.DCO { + case 1: + f.PR.DCO = "yes" + case -1: + f.PR.DCO = "no" + } + filled = append(filled, "pr.checks", "pr.dco") + } else { + unknown = append(unknown, "pr.checks", "pr.dco") + } + unknown = append(unknown, "pr.review", "pr.changesRequested", "pr.lane") + } else { + f.Issue = &SimulateIssueFacts{Bucket: "actionable"} + if !snap.Open { + f.Issue.Bucket = "" + } + unknown = append(unknown, "issue.bucket", "issue.claimed", "issue.coveringPR", "issue.stalled") + } + sort.Strings(unknown) + return f, filled, unknown +} + +func hasLabelFold(labels []string, want string) bool { + for _, l := range labels { + if strings.EqualFold(strings.TrimSpace(l), want) { + return true + } + } + return false +} + +func originLabel(labels []string) string { + for _, l := range labels { + if strings.HasPrefix(strings.ToLower(l), "origin/") { + return l + } + } + return "" +} + +// normalizeSimulateFacts fills the kind-specific pointer the deriver +// needs, lower-cases the enumerations and drops the other kind's evidence. +func normalizeSimulateFacts(kind string, f SimulateFacts) SimulateFacts { + if f.Labels == nil { + f.Labels = []string{} + } + f.Author.Kind = stageengine.AuthorKindName(stageengine.ParseAuthorKind(f.Author.Kind)) + f.Author.Role = strings.ToLower(strings.TrimSpace(f.Author.Role)) + if f.Park.Holds == nil { + f.Park.Holds = stagelabels.Holds(f.Labels) + } + if kind == "pr" { + f.Issue = nil + if f.PR == nil { + f.PR = &SimulatePRFacts{} + } + f.PR.Checks = stageengine.ChecksName(stageengine.ParseChecks(f.PR.Checks)) + f.PR.Review = simulateReviewName(f.PR.Review) + f.PR.DCO = stageengine.DCOName(stageengine.ParseDCO(f.PR.DCO)) + } else { + f.PR = nil + if f.Issue == nil { + f.Issue = &SimulateIssueFacts{} + } + f.Issue.Bucket = strings.ToLower(strings.TrimSpace(f.Issue.Bucket)) + } + return f +} + +// simulateReviewName keeps requires_human distinct (stageengine.ReviewName +// folds it into pending for the rules vocabulary, but the built-in deriver +// parks on it). +func simulateReviewName(s string) string { + st := stageengine.ParseReview(s) + if st == stagelabels.VerdictRequiresHuman { + return "requires_human" + } + return stageengine.ReviewName(st) +} + +func simulateRole(s string) stagelabels.Role { + for _, r := range []stagelabels.Role{stagelabels.RoleTrusted, stagelabels.RoleContributor, stagelabels.RoleNewcomer, stagelabels.RoleUntrusted} { + if r.String() == s { + return r + } + } + return stagelabels.RoleNone +} + +// toEngine converts the wire facts into the Facts the deriver reads. +func (f SimulateFacts) toEngine(ref stageengine.Ref) stageengine.Facts { + out := stageengine.Facts{ + Ref: ref, + ObservedAt: time.Now().UTC(), + Labels: append([]string(nil), f.Labels...), + Open: f.Open, + Merged: f.Merged, + Draft: f.Draft, + Origin: f.Origin, + Author: stageengine.AuthorFacts{ + Login: f.Author.Login, + Kind: stageengine.ParseAuthorKind(f.Author.Kind), + SelfAuthored: f.Author.SelfAuthored, + Role: simulateRole(f.Author.Role), + }, + Park: stageengine.ParkFacts{ + Holds: append([]string(nil), f.Park.Holds...), + NeedsHuman: f.Park.NeedsHuman, + DecisionLabel: f.Park.DecisionLabel, + PossiblyExists: f.Park.PossiblyExists, + Sentinel: f.Park.Sentinel, + HumanOnlyReason: f.Park.HumanOnlyReason, + ApprovedDirection: f.Park.ApprovedDirection, + DecisionQuestion: f.Park.DecisionQuestion, + DecisionSuperseded: f.Park.DecisionSuperseded, + }, + } + if f.Issue != nil { + is := &stageengine.IssueFacts{ + Bucket: stageengine.IssueBucket(f.Issue.Bucket), + FilteredReason: f.Issue.FilteredReason, + LikelyDone: f.Issue.LikelyDone, + Stalled: f.Issue.Stalled, + OpenChildren: f.Issue.OpenChildren, + HoldLabel: f.Issue.HoldLabel, + BlockedReason: f.Issue.BlockedReason, + NeedsHumanLabel: f.Issue.NeedsHumanLabel, + Blocker: f.Issue.Blocker, + ApprovedDirectionLabel: f.Issue.ApprovedDirectionLabel, + UmbrellaLabel: f.Issue.UmbrellaLabel, + DiscussionLabel: f.Issue.DiscussionLabel, + TriageAwaiting: f.Issue.TriageAwaiting, + TriageAwaitingLabel: f.Issue.TriageAwaitingLabel, + } + if f.Issue.Claimed || f.Issue.ClaimHolder != "" { + is.Claim = &stageengine.ClaimFact{Holder: f.Issue.ClaimHolder, PRStarted: f.Issue.ClaimPRStarted} + } + if f.Issue.CoveringPR > 0 { + is.CoveringPR = &stageengine.Ref{Owner: ref.Owner, Repo: ref.Repo, Number: f.Issue.CoveringPR, Kind: stagelabels.PRItem} + if st, ok := stageByNumberForSimulate(f.Issue.CoveringPRStage); ok { + is.CoveringPRStage = st + } + } + out.Issue = is + } + if f.PR != nil { + review := stageengine.ParseReview(f.PR.Review) + pr := &stageengine.PRFacts{ + HeadSHA: f.PR.HeadSHA, + HeadFresh: f.PR.HeadFresh, + BaseRef: f.PR.BaseRef, + Conflicting: f.PR.Conflicting, + ChangesRequested: f.PR.ChangesRequested || review == stagelabels.VerdictChangesRequested, + Checks: stageengine.ParseChecks(f.PR.Checks), + DCO: stageengine.ParseDCO(f.PR.DCO), + DCOGate: f.PR.DCOGate, + Review: stageengine.ReviewFacts{State: review, HumanApprover: f.PR.HumanApprover}, + } + // The built-in deriver reads the raw verdict string behind the + // folded state (VerdictBlocks / VerdictNeedsHuman), so spell it. + switch review { + case stagelabels.VerdictApproved: + pr.Review.VerdictKnown, pr.Review.Verdict = true, "approve" + case stagelabels.VerdictChangesRequested: + pr.Review.VerdictKnown, pr.Review.Verdict = true, "changes_requested" + case stagelabels.VerdictRequiresHuman: + pr.Review.VerdictKnown, pr.Review.Verdict = true, "requires_human" + } + if f.PR.Lane != nil && f.PR.Lane.Lane != "" { + pr.Lanes = []stageengine.LaneFact{{Lane: f.PR.Lane.Lane, Reason: f.PR.Lane.Reason, Role: out.Author.Role, Merged: f.Merged}} + } + out.PR = pr + } + return out +} + +func stageByNumberForSimulate(n int) (stagelabels.Stage, bool) { + if n <= 0 { + return stagelabels.None, false + } + for _, def := range stagelabels.Table { + if def.Number == n { + return def.Stage, true + } + } + return stagelabels.None, false +} diff --git a/src/pkg/dashboard/api_workflows_simulate_test.go b/src/pkg/dashboard/api_workflows_simulate_test.go new file mode 100644 index 0000000000..20ae4760ce --- /dev/null +++ b/src/pkg/dashboard/api_workflows_simulate_test.go @@ -0,0 +1,225 @@ +package dashboard + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "strings" + "testing" + + ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/stageengine" +) + +func simulateCall(t *testing.T, s *Server, path string, body any) (int, SimulateResponse, string) { + t.Helper() + rec := wfReq(s, http.MethodPost, path, body, false) + var out SimulateResponse + if rec.Code == http.StatusOK { + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("decode: %v\n%s", err, rec.Body) + } + } + return rec.Code, out, rec.Body.String() +} + +// hive-default (derivation builtin): a green, approved PR by a trusted +// author derives the built-in 080 and the trace is the single builtin +// entry naming the state and column the profile puts it in. +func TestWorkflowSimulateHiveDefaultPR(t *testing.T) { + s := workflowTestServer(t, true) + code, out, raw := simulateCall(t, s, "/api/workflows/hive-default/1/simulate", map[string]any{ + "kind": "pr", "repo": "myorg/repo1", "number": 7, + "facts": map[string]any{ + "labels": []string{"role/trusted"}, "open": true, + "author": map[string]any{"login": "andy", "kind": "human", "role": "trusted"}, + "pr": map[string]any{"checks": "green", "review": "approved", "dco": "yes"}, + }, + }) + if code != http.StatusOK { + t.Fatalf("%d %s", code, raw) + } + d := out.Decision + if d.Abstain || !d.Covered || d.Stage != 80 || d.State != "awaiting-merge-admission" || d.Column == "" || !strings.HasPrefix(d.Label, "stage/080") { + t.Fatalf("decision %+v", d) + } + if len(out.Trace) != 1 || !out.Trace[0].Builtin || !out.Trace[0].Matched || out.Trace[0].State != d.State { + t.Fatalf("trace %+v", out.Trace) + } + if out.Facts.PR == nil || out.Facts.PR.Lane == nil || out.Facts.PR.Lane.Lane != "trusted-author" || len(out.Notes) != 1 { + t.Fatalf("simulated lane not reported: %+v notes=%v", out.Facts.PR, out.Notes) + } + if out.Facts.Issue != nil || out.Facts.PR.Review != "approved" || out.Facts.Author.Kind != "human" { + t.Fatalf("facts not normalized: %+v", out.Facts) + } + + // Red CI on the hive's own PR is the fixing stage. + code, out, raw = simulateCall(t, s, "/api/workflows/hive-default/1/simulate", map[string]any{ + "kind": "pr", + "facts": map[string]any{"open": true, "author": map[string]any{"kind": "hive", "selfAuthored": true, "role": "trusted"}, "pr": map[string]any{"checks": "red"}}, + }) + if code != http.StatusOK || out.Decision.Stage != 70 { + t.Fatalf("%d %+v %s", code, out.Decision, raw) + } + + // An issue needs no PR evidence. + code, out, raw = simulateCall(t, s, "/api/workflows/hive-default/1/simulate", map[string]any{ + "kind": "issue", "facts": map[string]any{"open": true, "author": map[string]any{"kind": "human"}, "issue": map[string]any{"bucket": "actionable"}}, + }) + if code != http.StatusOK || out.Decision.Stage != 10 || out.Decision.State != "actionable" { + t.Fatalf("%d %+v %s", code, out.Decision, raw) + } +} + +// A rules model: the trace lists every rule in order, marks the first +// match, shows each predicate's verdict and actual value, and leaves the +// rules after the match unevaluated. +func TestWorkflowSimulateRulesTrace(t *testing.T) { + s := workflowTestServer(t, true) + if rec := wfReq(s, http.MethodPost, "/api/workflows", techdocsModel(t), true); rec.Code != http.StatusCreated { + t.Fatalf("save: %d %s", rec.Code, rec.Body) + } + code, out, raw := simulateCall(t, s, "/api/workflows/cncf-techdocs/1/simulate", map[string]any{ + "kind": "issue", + "facts": map[string]any{ + "labels": []string{"techdocs/request", "techdocs/accepted"}, "open": true, + "author": map[string]any{"login": "nate", "kind": "human"}, + }, + }) + if code != http.StatusOK { + t.Fatalf("%d %s", code, raw) + } + if out.Decision.Abstain || out.Decision.Stage != 20 || out.Decision.Column == "" { + t.Fatalf("decision %+v", out.Decision) + } + var matched, afterMatch int + seenMatch := false + for _, tr := range out.Trace { + if tr.Builtin { + t.Fatalf("rules trace has a builtin entry: %+v", tr) + } + if tr.Matched { + matched++ + seenMatch = true + if tr.State != out.Decision.State { + t.Fatalf("matched rule %s names %s, decision %s", tr.ID, tr.State, out.Decision.State) + } + for _, p := range tr.Predicates { + if !p.Holds || !p.Known || p.Fact == "" || p.Op == "" { + t.Fatalf("matched rule has a failing/unknown predicate: %+v", p) + } + } + continue + } + if seenMatch { + afterMatch++ + if tr.Evaluated || len(tr.Predicates) != 0 { + t.Fatalf("rule after the match was evaluated: %+v", tr) + } + } else if tr.Evaluated { + failing := 0 + for _, p := range tr.Predicates { + if !p.Holds { + failing++ + } + } + if failing == 0 { + t.Fatalf("evaluated, unmatched rule has no failing predicate: %+v", tr) + } + } + } + if matched != 1 || afterMatch == 0 { + t.Fatalf("matched=%d afterMatch=%d trace=%+v", matched, afterMatch, out.Trace) + } + + // Uncovered facts abstain with the not-covered reason. + code, out, raw = simulateCall(t, s, "/api/workflows/cncf-techdocs/1/simulate", map[string]any{ + "kind": "issue", "facts": map[string]any{"labels": []string{"bug"}, "open": true}, + }) + if code != http.StatusOK || !out.Decision.Abstain || out.Decision.Covered || !strings.HasPrefix(out.Decision.Reason, stageengine.ReasonNotCovered) { + t.Fatalf("%d %+v %s", code, out.Decision, raw) + } + for _, tr := range out.Trace { + if tr.Matched { + t.Fatalf("uncovered run has a matched rule: %+v", tr) + } + } +} + +func TestWorkflowSimulateErrors(t *testing.T) { + s := workflowTestServer(t, true) + for _, c := range []struct { + name string + path string + body any + code int + }{ + {"bad kind", "/api/workflows/hive-default/1/simulate", map[string]any{"kind": "epic", "facts": map[string]any{}}, http.StatusBadRequest}, + {"no kind", "/api/workflows/hive-default/1/simulate", map[string]any{"facts": map[string]any{}}, http.StatusBadRequest}, + {"bad json", "/api/workflows/hive-default/1/simulate", "{", http.StatusBadRequest}, + {"unknown model", "/api/workflows/nope/1/simulate", map[string]any{"kind": "pr", "facts": map[string]any{}}, http.StatusNotFound}, + {"neither facts nor item", "/api/workflows/hive-default/1/simulate", map[string]any{"kind": "pr"}, http.StatusBadRequest}, + {"item without forge client", "/api/workflows/hive-default/1/simulate", map[string]any{"kind": "pr", "repo": "myorg/repo1", "number": 3}, http.StatusUnprocessableEntity}, + } { + if code, _, raw := simulateCall(t, s, c.path, c.body); code != c.code { + t.Errorf("%s: %d want %d: %s", c.name, code, c.code, raw) + } + } + if rec := wfReq(workflowTestServer(t, false), http.MethodPost, "/api/workflows/hive-default/1/simulate", map[string]any{"kind": "pr", "facts": map[string]any{}}, false); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no store: %d", rec.Code) + } +} + +// repo+number without facts fills from the forge snapshot and reports +// what was filled and what stays unknown; the kind comes from the item. +func TestWorkflowSimulateFetchesItem(t *testing.T) { + s := workflowTestServer(t, true) + var asked string + s.deps.WorkflowItemSnapshot = func(_ context.Context, repo string, number int) (ghpkg.WorkflowItemSnapshot, error) { + asked = repo + "#" + string(rune('0'+number)) + if number == 9 { + return ghpkg.WorkflowItemSnapshot{}, errors.New("boom") + } + return ghpkg.WorkflowItemSnapshot{ + Kind: "pr", Labels: []string{"role/untrusted", "hold", "origin/scanner"}, Open: true, Author: "nate", + HeadSHA: "abc", BaseRef: "main", ConflictKnown: true, + Checks: ghpkg.CommitChecksSummary{Total: 2, Green: 2, DCO: 1}, ChecksKnown: true, + }, nil + } + code, out, raw := simulateCall(t, s, "/api/workflows/hive-default/1/simulate", map[string]any{"repo": "myorg/repo1", "number": 4}) + if code != http.StatusOK { + t.Fatalf("%d %s", code, raw) + } + if asked != "myorg/repo1#4" { + t.Fatalf("asked %q", asked) + } + f := out.Facts + if f.PR == nil || f.Issue != nil || f.PR.Checks != "green" || f.PR.DCO != "yes" || f.PR.Review != "pending" || f.Author.Role != "untrusted" || f.Author.Login != "nate" || f.Origin != "origin/scanner" { + t.Fatalf("facts %+v pr=%+v", f, f.PR) + } + if len(f.Park.Holds) != 1 || f.Park.Holds[0] != "hold" { + t.Fatalf("holds %v", f.Park.Holds) + } + has := func(list []string, want string) bool { + for _, s := range list { + if s == want { + return true + } + } + return false + } + if !has(out.Filled, "pr.checks") || !has(out.Filled, "labels") || !has(out.Unknown, "pr.review") || !has(out.Unknown, "author.kind") || has(out.Unknown, "pr.conflicting") { + t.Fatalf("filled=%v unknown=%v", out.Filled, out.Unknown) + } + if out.Decision.Abstain { + t.Fatalf("decision %+v", out.Decision) + } + + if code, _, raw := simulateCall(t, s, "/api/workflows/hive-default/1/simulate", map[string]any{"kind": "issue", "repo": "myorg/repo1", "number": 4}); code != http.StatusBadRequest || !strings.Contains(raw, "is a pr") { + t.Fatalf("kind mismatch: %d %s", code, raw) + } + if code, _, _ := simulateCall(t, s, "/api/workflows/hive-default/1/simulate", map[string]any{"repo": "myorg/repo1", "number": 9}); code != http.StatusBadGateway { + t.Fatalf("forge error: %d", code) + } +} diff --git a/src/pkg/dashboard/api_workflows_versions.go b/src/pkg/dashboard/api_workflows_versions.go new file mode 100644 index 0000000000..a902ea7646 --- /dev/null +++ b/src/pkg/dashboard/api_workflows_versions.go @@ -0,0 +1,186 @@ +package dashboard + +import ( + "errors" + "fmt" + "net/http" + "time" + + "github.com/hivecommons/hive/pkg/stagelabels" + "github.com/hivecommons/hive/pkg/workflowmodel" + wfstore "github.com/hivecommons/hive/pkg/workflowmodel/store" +) + +// Workflow-model versions, diff, confirm and label preview +// (hivecommons/hive#17620, RFC phase 4, backend). These sit beside the +// library routes in api_workflows.go: reads are open, confirm is +// owner-gated and audited as workflow_confirm. +func (s *Server) registerWorkflowVersionRoutes() { + s.mux.HandleFunc("GET /api/workflows/{name}/versions", s.handleWorkflowVersions) + s.mux.HandleFunc("GET /api/workflows/{name}/diff", s.handleWorkflowDiff) + s.mux.HandleFunc("POST /api/workflows/{name}/{version}/confirm", s.handleWorkflowConfirm) + s.mux.HandleFunc("GET /api/workflows/{name}/{version}/labels", s.handleWorkflowLabels) +} + +// handleWorkflowVersions lists every saved version of a model, newest +// save first; hive-default answers its single virtual entry. +func (s *Server) handleWorkflowVersions(w http.ResponseWriter, r *http.Request) { + st := s.workflowStore(w) + if st == nil { + return + } + entries, err := st.Versions(r.PathValue("name")) + if err != nil { + workflowStoreError(w, err) + return + } + jsonResponse(w, map[string]interface{}{"versions": entries}) +} + +// handleWorkflowDiff compares two saved versions. from/to are versions of +// {name}; fromName/toName override the model name on either side so a +// local model can be compared against hive-default. +func (s *Server) handleWorkflowDiff(w http.ResponseWriter, r *http.Request) { + st := s.workflowStore(w) + if st == nil { + return + } + q := r.URL.Query() + from, to := q.Get("from"), q.Get("to") + if from == "" || to == "" { + jsonError(w, "from and to query parameters are required", http.StatusBadRequest) + return + } + name := r.PathValue("name") + fromName, toName := q.Get("fromName"), q.Get("toName") + if fromName == "" { + fromName = name + } + if toName == "" { + toName = name + } + a, ae, err := st.Get(fromName, from) + if err != nil { + workflowStoreError(w, err) + return + } + b, be, err := st.Get(toName, to) + if err != nil { + workflowStoreError(w, err) + return + } + jsonResponse(w, map[string]interface{}{"from": ae, "to": be, "diff": workflowmodel.Diff(a, b)}) +} + +// handleWorkflowConfirm saves a copy of {name}@{version} with every +// inferred element confirmed and provenance local, as the next integer +// version. hive-default is read-only and answers 409. +func (s *Server) handleWorkflowConfirm(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + st := s.workflowStore(w) + if st == nil { + return + } + name, version := r.PathValue("name"), r.PathValue("version") + if wfstore.IsBuiltin(name) { + jsonError(w, fmt.Sprintf("%s is built-in and read-only; save a copy under another name to edit it", name), http.StatusConflict) + return + } + m, _, err := st.Get(name, version) + if err != nil { + workflowStoreError(w, err) + return + } + inferred := workflowmodel.Inferred(m) + next, err := st.NextVersion(name) + if err != nil { + workflowStoreError(w, err) + return + } + c := workflowmodel.Confirm(m, requestUser(r)) + c.Spec.Meta.Version = next + e, err := st.Save(c, requestUser(r), time.Now().UTC()) + if err != nil { + workflowStoreError(w, err) + return + } + s.auditFromRequest(r, "workflow_confirm", fmt.Sprintf("%s@%s -> %s confirmed=%d compiled=%v", name, version, e.Key(), inferred, e.Compiled), "") + jsonStatusResponse(w, http.StatusCreated, map[string]interface{}{"entry": e, "confirmed": inferred, "from": name + "@" + version}) +} + +// workflowStateCommands is the human-gate verb list one state offers. +type workflowStateCommands struct { + State string `json:"state"` + Verbs []string `json:"verbs"` +} + +// workflowStageLabel is the built-in stage label a numbered state maps +// onto when the model declares no label of its own. +type workflowStageLabel struct { + State string `json:"state"` + Number int `json:"number"` + Label string `json:"label"` +} + +// handleWorkflowLabels previews what binding {name}@{version} regenerates: +// the labels the engine reconciles into a bound repository and the +// commands (human-gate verbs) each state offers. A draft that does not +// compile answers 400 with its diagnostics. +func (s *Server) handleWorkflowLabels(w http.ResponseWriter, r *http.Request) { + st := s.workflowStore(w) + if st == nil { + return + } + name, version := r.PathValue("name"), r.PathValue("version") + p, err := st.Profile(name, version) + if err != nil { + if errors.Is(err, wfstore.ErrNotCompiled) { + _, e, gerr := st.Get(name, version) + if gerr == nil { + jsonStatusResponse(w, http.StatusBadRequest, map[string]interface{}{"error": err.Error(), "diagnostics": e.Diagnostics}) + return + } + } + workflowStoreError(w, err) + return + } + jsonResponse(w, workflowLabelsPreview(p)) +} + +func workflowLabelsPreview(p *workflowmodel.Profile) map[string]interface{} { + labels := p.LabelDefs(stagelabels.StageColor) + if labels == nil { + labels = []workflowmodel.LabelDef{} + } + for i := range labels { + labels[i].Description = stagelabels.ClampLabelDescription(labels[i].Description) + } + commands := []workflowStateCommands{} + stages := []workflowStageLabel{} + for _, col := range p.Columns { + for _, sid := range col.States { + if verbs := p.Replies[sid]; len(verbs) > 0 { + commands = append(commands, workflowStateCommands{State: sid, Verbs: append([]string(nil), verbs...)}) + } + st, ok := p.States[sid] + if !ok || st == nil || st.Lifecycle.Number == 0 { + continue + } + for _, def := range stagelabels.Table { + if def.Number == st.Lifecycle.Number { + stages = append(stages, workflowStageLabel{State: sid, Number: def.Number, Label: def.Stage.Label()}) + break + } + } + } + } + return map[string]interface{}{ + "name": p.Name, "version": p.Version, "hash": p.Hash, + "prefix": p.Labels.Prefix, + "labels": labels, + "commands": commands, + "stages": stages, + } +} diff --git a/src/pkg/dashboard/api_workflows_versions_test.go b/src/pkg/dashboard/api_workflows_versions_test.go new file mode 100644 index 0000000000..6b0f72061d --- /dev/null +++ b/src/pkg/dashboard/api_workflows_versions_test.go @@ -0,0 +1,219 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/stagelabels" + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/hivedefault" + wfstore "github.com/hivecommons/hive/pkg/workflowmodel/store" +) + +func TestWorkflowVersionRoutesNeedStore(t *testing.T) { + s := workflowTestServer(t, false) + for _, c := range []struct{ method, path string }{ + {http.MethodGet, "/api/workflows/x/versions"}, + {http.MethodGet, "/api/workflows/x/diff?from=1&to=2"}, + {http.MethodPost, "/api/workflows/x/1/confirm"}, + {http.MethodGet, "/api/workflows/x/1/labels"}, + } { + if rec := wfReq(s, c.method, c.path, nil, true); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("%s %s: %d %s", c.method, c.path, rec.Code, rec.Body) + } + } + s = workflowTestServer(t, true) + if rec := wfReq(s, http.MethodPost, "/api/workflows/x/1/confirm", nil, false); rec.Code != http.StatusForbidden { + t.Fatalf("confirm owner gate: %d", rec.Code) + } +} + +func TestWorkflowVersionsAndDiff(t *testing.T) { + s := workflowTestServer(t, true) + m := techdocsModel(t) + if rec := wfReq(s, http.MethodPost, "/api/workflows", m, true); rec.Code != http.StatusCreated { + t.Fatal(rec.Body.String()) + } + m2 := techdocsModel(t) + m2.Spec.Meta.Version = "2" + m2.Spec.States[0].Label = "Requested!" + m2.Spec.States = append(m2.Spec.States, workflowmodel.State{ID: "extra", Label: "Extra", Owner: workflowmodel.OwnerHuman}) + if rec := wfReq(s, http.MethodPost, "/api/workflows", m2, true); rec.Code != http.StatusCreated { + t.Fatal(rec.Body.String()) + } + + rec := wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/versions", nil, false) + var vs struct{ Versions []wfstore.Entry } + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &vs) != nil || len(vs.Versions) != 2 { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + if rec := wfReq(s, http.MethodGet, "/api/workflows/ghost/versions", nil, false); rec.Code != http.StatusNotFound { + t.Fatalf("unknown: %d", rec.Code) + } + rec = wfReq(s, http.MethodGet, "/api/workflows/hive-default/versions", nil, false) + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &vs) != nil || len(vs.Versions) != 1 || !vs.Versions[0].ReadOnly { + t.Fatalf("builtin: %d %s", rec.Code, rec.Body) + } + + for _, bad := range []string{"", "?from=1", "?to=2"} { + if rec := wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/diff"+bad, nil, false); rec.Code != http.StatusBadRequest { + t.Fatalf("diff %q: %d", bad, rec.Code) + } + } + if rec := wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/diff?from=1&to=9", nil, false); rec.Code != http.StatusNotFound { + t.Fatalf("diff missing to: %d", rec.Code) + } + if rec := wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/diff?from=9&to=1", nil, false); rec.Code != http.StatusNotFound { + t.Fatalf("diff missing from: %d", rec.Code) + } + rec = wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/diff?from=1&to=2", nil, false) + var d struct { + From, To wfstore.Entry + Diff workflowmodel.ModelDiff + } + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &d) != nil { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + if d.From.Version != "1" || d.To.Version != "2" || d.Diff.Empty || len(d.Diff.States.Added) != 1 || len(d.Diff.States.Changed) != 1 || d.Diff.Summary != "+1 state, 2 changed" { + t.Fatalf("diff %+v", d.Diff) + } + // same version both sides is empty; cross-model diff against hive-default works + rec = wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/diff?from=1&to=1", nil, false) + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &d) != nil || !d.Diff.Empty { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + rec = wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/diff?from=1&toName=hive-default&to="+hivedefault.Version, nil, false) + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &d) != nil || d.To.Name != hivedefault.Name || len(d.Diff.States.Added) == 0 { + t.Fatalf("%d %s", rec.Code, rec.Body) + } +} + +func TestWorkflowConfirm(t *testing.T) { + s := workflowTestServer(t, true) + m := techdocsModel(t) + m.Spec.Meta.Provenance = workflowmodel.ProvenanceIngestedUnreviewed + m.Spec.States[0].Lifecycle.Confidence = workflowmodel.ConfidenceInferred + m.Spec.Events[0].Confidence = workflowmodel.ConfidenceInferred + if rec := wfReq(s, http.MethodPost, "/api/workflows", m, true); rec.Code != http.StatusCreated { + t.Fatal(rec.Body.String()) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/hive-default/1/confirm", nil, true); rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "read-only") { + t.Fatalf("builtin: %d %s", rec.Code, rec.Body) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/cncf-techdocs/9/confirm", nil, true); rec.Code != http.StatusNotFound { + t.Fatalf("missing: %d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/Bad%20Name/1/confirm", nil, true); rec.Code != http.StatusBadRequest { + t.Fatalf("bad name: %d", rec.Code) + } + + rec := wfReq(s, http.MethodPost, "/api/workflows/cncf-techdocs/1/confirm", nil, true) + var out struct { + Entry wfstore.Entry + Confirmed int + From string + } + if rec.Code != http.StatusCreated || json.Unmarshal(rec.Body.Bytes(), &out) != nil { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + if out.Entry.Version != "2" || out.Entry.Provenance != workflowmodel.ProvenanceLocal || out.Confirmed != 2 || out.From != "cncf-techdocs@1" || !out.Entry.Compiled || out.Entry.SavedBy != "andy" { + t.Fatalf("%+v", out) + } + for _, dg := range out.Entry.Diagnostics { + if dg.Code == workflowmodel.CodeInferred { + t.Fatalf("confirmed version still warns inferred: %+v", out.Entry.Diagnostics) + } + } + saved, _, err := s.deps.WorkflowStore.Get("cncf-techdocs", "2") + if err != nil || workflowmodel.Inferred(saved) != 0 || saved.Metadata.Annotations[workflowmodel.AnnotationConfirmedBy] != "andy" { + t.Fatalf("%v %+v", err, saved.Metadata) + } + // the original is untouched + orig, _, err := s.deps.WorkflowStore.Get("cncf-techdocs", "1") + if err != nil || workflowmodel.Inferred(orig) != 2 { + t.Fatalf("%v inferred=%d", err, workflowmodel.Inferred(orig)) + } + // confirming again allocates the next version + rec = wfReq(s, http.MethodPost, "/api/workflows/cncf-techdocs/2/confirm", nil, true) + if rec.Code != http.StatusCreated || json.Unmarshal(rec.Body.Bytes(), &out) != nil || out.Entry.Version != "3" || out.Confirmed != 0 { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + if !auditHasAction(t, s, "workflow_confirm") { + t.Fatal("no workflow_confirm audit entry") + } +} + +func auditHasAction(t *testing.T, s *Server, action string) bool { + t.Helper() + rec := wfReq(s, http.MethodGet, "/api/audit?limit=50", nil, true) + if rec.Code != 200 { + t.Fatalf("audit: %d %s", rec.Code, rec.Body) + } + return strings.Contains(rec.Body.String(), `"`+action+`"`) +} + +func TestWorkflowLabelsPreview(t *testing.T) { + s := workflowTestServer(t, true) + m := techdocsModel(t) + m.Spec.Labels = workflowmodel.LabelSet{Prefix: "docs", Entries: []workflowmodel.LabelEntry{ + {Name: "docs/010-requested", State: "requested"}, + {Name: "docs/020-analysing", Color: "123456", Description: strings.Repeat("x", 200), State: "analysing"}, + }} + if rec := wfReq(s, http.MethodPost, "/api/workflows", m, true); rec.Code != http.StatusCreated { + t.Fatal(rec.Body.String()) + } + rec := wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/1/labels", nil, false) + var out struct { + Name, Version, Prefix string + Labels []workflowmodel.LabelDef + Commands []struct { + State string + Verbs []string + } + Stages []struct { + State string + Label string + } + } + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &out) != nil { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + if out.Name != "cncf-techdocs" || out.Prefix != "docs" || len(out.Labels) != 2 { + t.Fatalf("%+v", out) + } + if out.Labels[0].Color != stagelabels.StageColor || out.Labels[0].Description != m.Spec.States[0].Label || out.Labels[1].Color != "123456" || len(out.Labels[1].Description) >= 200 { + t.Fatalf("labels %+v", out.Labels) + } + verbs := map[string][]string{} + for _, c := range out.Commands { + verbs[c.State] = c.Verbs + } + if len(verbs["analysing"]) != 1 || verbs["analysing"][0] != "hand back" || len(verbs["requested"]) != 0 { + t.Fatalf("commands %+v", out.Commands) + } + if len(out.Stages) == 0 || out.Stages[0].Label == "" { + t.Fatalf("stages %+v", out.Stages) + } + + // hive-default previews the built-in family + rec = wfReq(s, http.MethodGet, "/api/workflows/hive-default/"+hivedefault.Version+"/labels", nil, false) + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &out) != nil || len(out.Labels) == 0 || out.Labels[0].Color != stagelabels.StageColor { + t.Fatalf("builtin: %d %s", rec.Code, rec.Body) + } + if rec := wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/9/labels", nil, false); rec.Code != http.StatusNotFound { + t.Fatalf("missing: %d", rec.Code) + } + // a draft that does not compile answers 400 with its diagnostics + draft := techdocsModel(t) + draft.Spec.Meta.Version = "draft" + draft.Spec.Gates[0].ReleasedBy = "" + if rec := wfReq(s, http.MethodPost, "/api/workflows", draft, true); rec.Code != http.StatusCreated { + t.Fatal(rec.Body.String()) + } + rec = wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/draft/labels", nil, false) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), `"diagnostics"`) { + t.Fatalf("draft: %d %s", rec.Code, rec.Body) + } +} diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index b796186b11..762b20cc9a 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -37,6 +37,7 @@ import ( "github.com/hivecommons/hive/pkg/scheduler" "github.com/hivecommons/hive/pkg/stageengine" "github.com/hivecommons/hive/pkg/tokens" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/cluster" ) type ChatOutbound struct { @@ -65,6 +66,24 @@ type Dependencies struct { // files (repo-relative slash path -> contents) and the ref read, for // POST /api/workflows/ingest. Nil disables ingest from a repository. WorkflowRepoFiles func(ctx context.Context, repo string) (files map[string][]byte, ref string, err error) + // WorkflowItemSnapshot reads one issue or PR head-level for + // POST /api/workflows/{name}/{version}/simulate (RFC #17620 phase 3c) + // so the operator can simulate a real item. Nil disables the fetch; the + // handler then answers 422 and the form still accepts hand-set facts. + WorkflowItemSnapshot func(ctx context.Context, repo string, number int) (ghpkg.WorkflowItemSnapshot, error) + // WorkflowRepoFile fetches one default-branch file (a written process + // spec) for POST /api/workflows/ingest/spec {repo, path}. Nil disables + // repo-path spec ingest. + WorkflowRepoFile func(ctx context.Context, repo, path string) (string, error) + // WorkflowClusterRunner runs kubectl for POST /api/workflows/ingest/cluster + // (RFC #17620 phase 6). Nil means no kube access is configured on this + // spoke and the route answers 422. + WorkflowClusterRunner cluster.Runner + // WorkflowSpecLLM is the reviewer-gateway turn behind the optional LLM + // refinement of POST /api/workflows/ingest/spec. It is the only LLM use + // in the workflow-model stack; nil or an unconfigured gateway disables + // it and the route reports spec.llm=false in capabilities. + WorkflowSpecLLM delegatedecision.TurnRunner // AgentMgr is the agent runtime (hivecommons/hive#13536). The dashboard // programs against the crux.Runtime surface, not *agent.Manager; tests that // need manager-only seams unwrap it with agentManager. diff --git a/src/pkg/dashboard/import_count_test.go b/src/pkg/dashboard/import_count_test.go index 5bda3be660..b752cb31ac 100644 --- a/src/pkg/dashboard/import_count_test.go +++ b/src/pkg/dashboard/import_count_test.go @@ -76,7 +76,7 @@ func TestDashboardInternalImportCountRatchet(t *testing.T) { // plus pkg/workflowmodel/store, the spoke-local library of workflow // models behind the /api/workflows routes and the tracker's per-repo // column block (RFC hivecommons/hive#17620, phase 2). - const maxDashboardInternalImports = 68 + const maxDashboardInternalImports = 69 entries, err := os.ReadDir(".") if err != nil { diff --git a/src/pkg/dashboard/job_terminal.go b/src/pkg/dashboard/job_terminal.go index 7434279171..032f93eff8 100644 --- a/src/pkg/dashboard/job_terminal.go +++ b/src/pkg/dashboard/job_terminal.go @@ -48,6 +48,26 @@ import ( // recorded exit code (a completed Job is deleted by the launcher as soon // as the kick finishes, so this is the normal state between kicks); // - neither: the agent has not run yet. +// +// OpenShell runtime (hivecommons/hive#17645). An agent with +// sandbox.runtime: openshell has no tmux session either: each kick runs in +// a sandbox an OpenShell gateway owns. The same page, route and three +// states apply; only the live source differs. Instead of the Job pod log it +// follows the newest sandbox the launcher created for the agent through +// sandbox.OpenShellLogReader, which keeps the OpenShell SDK out of this +// package, and the launcher deletes the sandbox when the kick ends, so +// between kicks the pane shows the persisted transcript exactly as above. +// +// The live source is weaker than the pod log, and the pane says so. The +// gateway's sandbox log holds supervisor, policy (OCSF) and tracing +// events; the kick runs as an exec session whose output reaches neither +// that log nor the container log (OpenShell docs, "Accessing Logs"), and +// Sandbox.Status.ExitCode is the sandbox main process's, not the kick's. +// So while a kick runs the pane shows what the sandbox did around it +// (network and filesystem decisions, supervisor state) and then points the +// operator at the transcript, which is where the kick's own output and +// exit code land once the launcher's Exec().Run returns. Only the job pod +// log shows the kick's stdout live. const ( // Both live under terminalPathPrefix so the terminal handoff code, @@ -78,6 +98,7 @@ type jobPodLogReader interface { // fields take the in-cluster defaults. type jobTerminalDeps struct { reader jobPodLogReader + openshell sandbox.OpenShellLogReader workspaceRoot string pollInterval time.Duration } @@ -96,6 +117,28 @@ func (s *Server) jobPodLogReader() jobPodLogReader { return &kubejob.Launcher{} } +// openshellLogReader is the OpenShell counterpart of jobPodLogReader: a +// launcher pointed at the gateway in the agent's effective openshell block. +// Only the fields a read needs are set (where the gateway is and how to +// authenticate to it); the kick-shaping ones belong to the agent manager's +// launcher, which this package does not import (crux boundary ratchet). +func (s *Server) openshellLogReader(name string) sandbox.OpenShellLogReader { + if s.jobTerminal.openshell != nil { + return s.jobTerminal.openshell + } + var opts sandbox.OpenShellOptions + if s.deps != nil && s.deps.Config != nil { + if agentCfg, ok := s.deps.Config.Agents[name]; ok { + oc := agentCfg.SandboxOpenShell(s.deps.Config.AgentSandbox) + opts = sandbox.OpenShellOptions{GatewayURL: oc.GatewayURL, Workspace: oc.Namespace, InsecureTLS: oc.InsecureTLS} + if env := strings.TrimSpace(oc.TokenEnv); env != "" { + opts.Token = os.Getenv(env) + } + } + } + return &sandbox.OpenShellLauncher{Options: opts} +} + func (s *Server) jobTerminalWorkspaceRoot() string { if s.jobTerminal.workspaceRoot != "" { return s.jobTerminal.workspaceRoot @@ -116,26 +159,35 @@ func (s *Server) jobTerminalPollInterval() time.Duration { return defaultJobTerminalPollInterval } +// sandboxRuntimeWithoutTmux reports whether an agent on this sandbox +// runtime runs its kicks outside the hive pod, so it has no tmux session +// and its terminal is this read-only viewer. +func sandboxRuntimeWithoutTmux(rt string) bool { + return rt == config.SandboxRuntimeJob || rt == config.SandboxRuntimeOpenShell +} + // jobTerminalAgent resolves the path's agent and checks that its effective -// sandbox runtime is job. The status code says which precondition failed: -// 404 for an unknown agent, 409 for one that has a tmux terminal instead. -func (s *Server) jobTerminalAgent(r *http.Request) (string, int, string) { +// sandbox runtime has no tmux session (job or openshell), returning that +// runtime. The status code says which precondition failed: 404 for an +// unknown agent, 409 for one that has a tmux terminal instead. +func (s *Server) jobTerminalAgent(r *http.Request) (string, string, int, string) { if s.deps == nil || s.deps.Config == nil { - return "", http.StatusServiceUnavailable, "configuration unavailable" + return "", "", http.StatusServiceUnavailable, "configuration unavailable" } name := s.resolveAgentParam(r.PathValue("name")) cfg := s.deps.Config agentCfg, ok := cfg.Agents[name] if !ok { - return "", http.StatusNotFound, fmt.Sprintf("agent %q not found", name) + return "", "", http.StatusNotFound, fmt.Sprintf("agent %q not found", name) } if !agentCfg.SandboxEnabled(cfg.AgentSandbox) { - return "", http.StatusConflict, fmt.Sprintf("agent %q is not sandboxed; its terminal is the tmux session at /terminal/?arg=hive-%s", name, name) + return "", "", http.StatusConflict, fmt.Sprintf("agent %q is not sandboxed; its terminal is the tmux session at /terminal/?arg=hive-%s", name, name) } - if rt := agentCfg.SandboxRuntime(cfg.AgentSandbox); rt != config.SandboxRuntimeJob { - return "", http.StatusConflict, fmt.Sprintf("agent %q runs sandbox kicks with runtime %q, not as a Kubernetes Job; there is no Job pod log to show", name, rt) + rt := agentCfg.SandboxRuntime(cfg.AgentSandbox) + if !sandboxRuntimeWithoutTmux(rt) { + return "", "", http.StatusConflict, fmt.Sprintf("agent %q runs sandbox kicks with runtime %q, not as a Kubernetes Job or OpenShell sandbox; there is no sandbox log to show", name, rt) } - return name, http.StatusOK, "" + return name, rt, http.StatusOK, "" } // handleJobTerminalPage serves the viewer document: a
 the inline
@@ -144,21 +196,23 @@ func (s *Server) jobTerminalAgent(r *http.Request) (string, int, string) {
 // 'unsafe-inline' policy (securityHeaders) rather than the SPA's hash
 // allowlist; no on*= attributes are used, matching the rest of the server.
 func (s *Server) handleJobTerminalPage(w http.ResponseWriter, r *http.Request) {
-	name, code, msg := s.jobTerminalAgent(r)
+	name, rt, code, msg := s.jobTerminalAgent(r)
 	if code != http.StatusOK {
 		http.Error(w, msg, code)
 		return
 	}
+	terms := sandboxTermsFor(rt)
 	esc := html.EscapeString(name)
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
 	w.Header().Set("Cache-Control", "no-store")
-	_, _ = fmt.Fprintf(w, jobTerminalPageHTML, esc, esc, esc)
+	_, _ = fmt.Fprintf(w, jobTerminalPageHTML, esc, terms.title, esc, terms.title, terms.about, esc)
 }
 
-// %s placeholders: agent (title), agent (heading), agent (stream URL path
-// segment). The stream URL is relative so the page works behind any prefix
-// the dashboard host is served at.
-const jobTerminalPageHTML = `%s — sandbox job log
+// %s placeholders: agent and log kind (title), agent and log kind
+// (heading), what the log is, agent (stream URL path segment). The stream
+// URL is relative so the page works behind any prefix the dashboard host is
+// served at.
+const jobTerminalPageHTML = `%s — %s
 
-

▶ %s — sandbox job log

read-only: the kick runs as a Kubernetes Job in its own pod, so this is the pod's log, not a shellconnecting…
+

▶ %s — %s

read-only: %s, not a shellconnecting…