diff --git a/changelog.d/added-integration-orchestration.md b/changelog.d/added-integration-orchestration.md new file mode 100644 index 0000000000..f4634907d0 --- /dev/null +++ b/changelog.d/added-integration-orchestration.md @@ -0,0 +1 @@ +- Add default-off, explicitly scoped integration orchestration with durable change intents, separate candidate observations, workflow-job accounting, and bounded admission above the existing preparation and GitHub queue authorities. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index af69cfdd77..c7a0757390 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -86,6 +86,57 @@ } ], "paths": { + "/api/integration-orchestration": { + "get": { + "tags": ["System"], + "summary": "Inspect integration coordination evidence and unknowns (owner only)", + "description": "Reports source/schema/runtime identity, configured mode, immutable candidates, canonical intents, live reservations, physical workflow jobs and cohort coverage. This ledger never authorizes merges or substitutes integration evidence for author review.", + "responses": { + "200": {"description": "Configuration and ledger, or explicit unavailable/unknown state"}, + "403": {"description": "Owner role required"} + } + } + }, + "/api/config/integration-orchestration": { + "put": { + "tags": ["System"], + "summary": "Replace bounded integration orchestration configuration (owner only)", + "description": "Full off|advisory|enforce configuration with exact repository/base scopes, static execution/admission budgets, bounded lease/window/age/retries and optional pinned workflow contracts. JSON durations use nanoseconds. Disabling is independent of evidence-store availability and never releases authoritative claims/native entries.", + "requestBody": {"required": true, "content": {"application/json": {"schema": {"type": "object"}}}}, + "responses": { + "200": {"description": "Validated effective configuration"}, + "400": {"description": "Invalid mode, scope, contract or bounds"}, + "403": {"description": "Owner role required"}, + "500": {"description": "Configuration could not be persisted"}, + "503": {"description": "Enforcement ledger unavailable; disable remains permitted"} + } + } + }, + "/api/integration-orchestration/intents": { + "post": { + "tags": ["System"], + "summary": "Record a versioned canonical change intent (owner only)", + "description": "Requires canonical work identity, exact enrolled destination, monotonically increasing generation, plan/node/revision, anticipated scope and provenance. Enforceable prerequisites must match the complete bound approved plan DAG; semantic and same-file hints never grant authority.", + "requestBody": {"required": true, "content": {"application/json": {"schema": {"type": "object"}}}}, + "responses": { + "200": {"description": "Durably recorded intent"}, + "400": {"description": "Malformed intent or unenrolled scope"}, + "403": {"description": "Owner role required"}, + "409": {"description": "Unapproved dependencies, generation conflict, cycle or unavailable journal"} + } + } + }, + "/api/integration-orchestration/reconcile": { + "post": { + "tags": ["System"], + "summary": "Reconcile a bounded workflow observation batch (owner only)", + "responses": { + "200": {"description": "Batch observed or minimum interval not elapsed; inspect per-scope completeness"}, + "403": {"description": "Owner role required"}, + "503": {"description": "Authoritative workflow observations unavailable"} + } + } + }, "/api/audit": { "get": { "tags": [ diff --git a/src/cmd/hive/claimswire.go b/src/cmd/hive/claimswire.go index 274b20956a..3371a51580 100644 --- a/src/cmd/hive/claimswire.go +++ b/src/cmd/hive/claimswire.go @@ -194,11 +194,20 @@ func buildClaimsLedger(cfg *config.Config, logger *slog.Logger) *claims.Ledger { // installClaimAdmissionCheck gates every automated ledger producer on current // labels, including renewals and forced takeovers. Resolve the client per call // because App setup can finish after boot. -func installClaimAdmissionCheck(ctx context.Context, ledger *claims.Ledger, client func() *github.Client) { +func installClaimAdmissionCheck(ctx context.Context, ledger *claims.Ledger, client func() *github.Client, integrationCheck ...func(string, int) string) { if ledger == nil || client == nil { return } ledger.SetAdmissionCheck(func(req claims.Request) (string, error) { + if _, held := ledger.Lookup(req.Repo, req.Issue); !held && !req.PRStarted { + for _, check := range integrationCheck { + if check != nil { + if reason := check(req.Repo, req.Issue); reason != "" { + return reason, nil + } + } + } + } ctx, cancel := context.WithTimeout(ctx, claimsGitHubTimeout) defer cancel() gh := client() diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index dc9cf334f9..adced8111f 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -3379,7 +3379,7 @@ func (b *boot) bootDashboardAPIWith(deps bootDashboardAPIDeps) { b.dashSrv.InstallClaimHooks(gfiClaimHooks(b.ctx, b.cfg, b.gfiState(), ghClientFn, b.dashSrv, b.logger, githubClaimHooks(b.ctx, b.cfg, ghClientFn, b.logger))) b.dashSrv.SetGoodFirstIssueStats(func() any { return b.gfiState().sweeper.Stats() }) - installClaimAdmissionCheck(b.ctx, b.issueClaims, func() *github.Client { return b.ghClient }) + installClaimAdmissionCheck(b.ctx, b.issueClaims, func() *github.Client { return b.ghClient }, b.dashSrv.IntegrationClaimBlockReason) releaseClaimsOnAgentRelaunch(b.agentMgr, b.dashSrv.IssueClaims, b.logger) installContributorClaimGuard(b.cfg, b.dashSrv, b.logger) // Forge App tab inventory: the resolved active key path and the per-app-id diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go index 5c1b115f4a..90474d6a7e 100644 --- a/src/cmd/hive/main_eval_cycle.go +++ b/src/cmd/hive/main_eval_cycle.go @@ -939,6 +939,10 @@ func runEvalCycle( // policy, dashboard status, PR/review dispatch, escalation, and every path // not explicitly enrolled. kickActionable := applyConvergenceKickAdmission(cfg, dashSrv, actionable, notifier, logger) + if err := dashSrv.ReconcileIntegration(ctx); err != nil { + logger.Warn("integration observations incomplete", "error", err) + } + kickActionable = dashSrv.IntegrationKickAdmission(kickActionable) sched.SetLastActionable(kickActionable) // The filter compacts agentsDue in place, so snapshot it first to report diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index ae7134fa02..ab95b8e83e 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -814,16 +814,18 @@ func runAutoMergeSweepIfDue(ctx context.Context, ghClient *github.Client, cfg *c ctx, sweepSpan := tracing.StartSpan(ctx, "governor.automerge") defer sweepSpan.End() opts := automerge.Options{ - Logger: logger, - MergerAuthorizer: trustedMergerFunc(cfg), - HumanChangeRequests: review.DefaultHumanChangeRequests(), - TrustedAuthorizer: trustedAuthorFunc(cfg), - TrustedAuthorPolicy: trustedAuthorPolicyFunc(cfg), - ContributorLanePolicy: contributorLanePolicyFunc(cfg), - ContributorLaneVerdict: contributorLaneVerdictFunc(cfg, logger), - ContributorRegistryTier: contributorRegistryTierFunc(), - StageLabelPolicy: stageLabelPolicyFunc(cfg), - StageEngine: stageEngineOptions(cfg, logger), + IntegrationAdmission: dashSrv.IntegrationNativeAdmission, + IntegrationFilesObserved: dashSrv.IntegrationObserveFiles, + Logger: logger, + MergerAuthorizer: trustedMergerFunc(cfg), + HumanChangeRequests: review.DefaultHumanChangeRequests(), + TrustedAuthorizer: trustedAuthorFunc(cfg), + TrustedAuthorPolicy: trustedAuthorPolicyFunc(cfg), + ContributorLanePolicy: contributorLanePolicyFunc(cfg), + ContributorLaneVerdict: contributorLaneVerdictFunc(cfg, logger), + ContributorRegistryTier: contributorRegistryTierFunc(), + StageLabelPolicy: stageLabelPolicyFunc(cfg), + StageEngine: stageEngineOptions(cfg, logger), } syncStageEngineMode(cfg, ghClient, logger) // hivecommons/hive#12436: the contribute hub is the stage-072 nudge sink diff --git a/src/docs/api-reference/agents-and-controls.md b/src/docs/api-reference/agents-and-controls.md index c31c3f8d46..207dac3a7e 100644 --- a/src/docs/api-reference/agents-and-controls.md +++ b/src/docs/api-reference/agents-and-controls.md @@ -23,19 +23,19 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/model-advisor` | Dashboard auth/session | Model Advisor | `pkg/dashboard/api.go:172` | | `GET` | `/api/governor/pr-models` | Dashboard auth/session | Agent-authored PR distribution by normalized attribution model/backend for `window=7d`, `30d`, or `all`, including per-model rework stats and the top 10 most-reworked PRs | `pkg/dashboard/api.go:173` | | `GET` | `/api/reviewer/accuracy` | Dashboard auth/session | Reviewer verdict calibration for the configured window: false-approve/false-block rates and confidence buckets per perspective and reviewer model | `pkg/dashboard/api.go:234` | -| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:501` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:504` | -| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:505` | -| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:509` | +| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:502` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:505` | +| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/keep-open` | Owner only | Keep a superseded PR open: adds `hive/keep-open`, which the supersession sweep never auto-closes | `pkg/dashboard/api.go:506` | +| `GET` | `/api/review/pipeline` | Dashboard auth/session | Review pipeline: one card per review-queue PR placed in its stage (`unreviewed`, `reviewing`, `changes_requested`, `fixing`, `human_hold`, `approved`, `merged`, `abandoned`) with `since`, `reviewers`, P0–P3 `severity` counts, `loop_count`/`loop_cap`, `next_action` and `reasons`; filtered by `repo` and `stage`, paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:510` | | `GET` | `/api/review/dispatch/events` | Dashboard auth/session | Webhook review dispatch queue: `enabled`, `webhooks_configured`, `debounce_s`, `pending` (soonest first) and `recent` (newest first) dispatches with repo, number, head SHA, action, timestamps and coalesced count, plus `stats` counters; in memory, never calls GitHub | `pkg/dashboard/api_review_event_dispatch.go:16` | -| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](../review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:513` | -| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:514` | -| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:516` | -| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `///issues/.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:519` | -| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:313` | -| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:314` | -| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:315` | -| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:316` | +| `GET` | `/api/review/evidence` | Merger or owner | [Review evidence bundle](../review-evidence.md) for one PR head. Query `repo` (`owner/name`), `number`, optional `head` (full SHA or unique prefix of at least 7 characters; `409` when ambiguous; default is the newest head), `format=json\|zip` (default `json`) and `download=1` for an attachment. `json` returns the bundle bytes exactly as sealed (verdicts, posted reviews, policy, CI check runs, sentinel findings, human actions, merge event, `hash`, optional `signature`); `zip` adds `verdicts.json`, `review-links.json` and a `manifest.json` of SHA-256s plus the signing public key when readable. `400` for a bad `repo`/`number`; `404` with `status: "expired"`/`expired: true` when retention removed a bundle Hive can prove existed, otherwise `status: "not_found"` | `pkg/dashboard/api.go:514` | +| `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:515` | +| `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:517` | +| `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `///issues/.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:520` | +| `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:314` | +| `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:315` | +| `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:316` | +| `DELETE` | `/api/agents/{name}` | Owner only | Agent Delete | `pkg/dashboard/api.go:317` | | `GET` | `/api/agents/{name}/log` | Dashboard auth/session | Agent Full Log | `pkg/dashboard/api.go:123` | | `GET` | `/api/agents/{name}/terminal-urls` | Dashboard auth/session | Agent Terminal URLs | `pkg/dashboard/api.go:127` | diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index ae1ec65d17..6606c7d703 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1649` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1650` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1651` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1650` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1651` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1652` | diff --git a/src/docs/api-reference/beads.md b/src/docs/api-reference/beads.md index a99adea4c9..b47bb6ce42 100644 --- a/src/docs/api-reference/beads.md +++ b/src/docs/api-reference/beads.md @@ -5,8 +5,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:485` | -| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:486` | -| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:487` | -| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:488` | -| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:489` | +| `GET` | `/api/beads` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:486` | +| `GET` | `/api/beads/{agent}` | Dashboard auth/session | Beads List | `pkg/dashboard/api.go:487` | +| `POST` | `/api/beads/{agent}` | Owner only | Beads Create | `pkg/dashboard/api.go:488` | +| `POST` | `/api/beads/reset` | Owner only | Beads Reset | `pkg/dashboard/api.go:489` | +| `POST` | `/api/beads/reset/{agent}` | Owner only | Beads Reset Agent | `pkg/dashboard/api.go:490` | diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index 8d938b0c39..9c19c94844 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -11,8 +11,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/config/dashboard/theme` | Owner only | Dashboard theme selection, overrides, effective theme, and built-in catalog | `pkg/dashboard/api.go:56` | | `PUT` | `/api/config/dashboard/theme` | Owner only | Persist `dashboard.theme` and `dashboard.theme_overrides` with validation, CSS/background guardrails, and audit logging | `pkg/dashboard/api.go:57` | | `GET` | `/api/config/download` | Owner only | Config Download | `pkg/dashboard/api.go:50` | -| `GET` | `/api/config/sla` | Owner only | Delivery SLA targets: `{targets:{start_to_pr_h,pr_to_merge_h,open_to_merge_h}, effective_targets:{...}, defaults:{...}}`, hours, from `sla.targets` in `hive.yaml` with defaults 4/24/72 | `pkg/dashboard/api.go:255` | -| `PUT` | `/api/config/sla` | Owner only | Replace `sla.targets` (`{targets:{start_to_pr_h,pr_to_merge_h,open_to_merge_h}}`; each 0 (default) to 8760, validated before anything is persisted); the Throughput Delivery SLA panel recolours on the next status poll | `pkg/dashboard/api.go:256` | +| `GET` | `/api/config/sla` | Owner only | Delivery SLA targets: `{targets:{start_to_pr_h,pr_to_merge_h,open_to_merge_h}, effective_targets:{...}, defaults:{...}}`, hours, from `sla.targets` in `hive.yaml` with defaults 4/24/72 | `pkg/dashboard/api.go:256` | +| `PUT` | `/api/config/sla` | Owner only | Replace `sla.targets` (`{targets:{start_to_pr_h,pr_to_merge_h,open_to_merge_h}}`; each 0 (default) to 8760, validated before anything is persisted); the Throughput Delivery SLA panel recolours on the next status poll | `pkg/dashboard/api.go:257` | | `GET` | `/api/config/export` | Owner only | Redacted JSON export of the effective config, layer files, side files, and provenance | `pkg/dashboard/api_config_export.go` | | `GET` | `/api/config/provenance` | Owner only | Config Provenance | `pkg/dashboard/api.go:53` | | `GET` | `/api/config/variables` | Dashboard auth/session | Variables List | `pkg/dashboard/api.go:54` | @@ -53,36 +53,36 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/config/governor/backup` | Owner only | Backup Key Status (presence + safe source label; never the key value) | `pkg/dashboard/backup_key.go` | | `PUT` | `/api/config/governor/backup` | Owner only | Backup Key Set (64-hex AES-256 key; stored 0600, path-only in `hive.yaml`) | `pkg/dashboard/backup_key.go` | | `DELETE` | `/api/config/governor/backup` | Owner only | Backup Key Clear (backups are refused again) | `pkg/dashboard/backup_key.go` | -| `GET` | `/api/config/governor/bob` | Dashboard auth/session | Governor Bob Status | `pkg/dashboard/api.go:284` | -| `PUT` | `/api/config/governor/bob` | Owner only | Governor Bob Key | `pkg/dashboard/api.go:285` | -| `DELETE` | `/api/config/governor/bob` | Owner only | Governor Bob Key Clear | `pkg/dashboard/api.go:286` | +| `GET` | `/api/config/governor/bob` | Dashboard auth/session | Governor Bob Status | `pkg/dashboard/api.go:285` | +| `PUT` | `/api/config/governor/bob` | Owner only | Governor Bob Key | `pkg/dashboard/api.go:286` | +| `DELETE` | `/api/config/governor/bob` | Owner only | Governor Bob Key Clear | `pkg/dashboard/api.go:287` | | `GET` | `/api/config/governor/cadence-scope` | Owner only | Governor Cadence Scope Get (`aggregate` or `per_repo`) | `pkg/dashboard/api.go:212` | | `PUT` | `/api/config/governor/cadence-scope` | Owner only | Governor Cadence Scope Set (`cadenceScope`/`cadence_scope`: `aggregate`, `per_repo`, or empty for default) | `pkg/dashboard/api.go:213` | -| `POST` | `/api/config/governor/bob/test` | Dashboard auth/session | Governor Bob Key Test | `pkg/dashboard/api.go:288` | -| `POST` | `/api/config/governor/litellm/test` | Dashboard auth/session | Governor Lite LLMTest | `pkg/dashboard/api.go:289` | -| `GET` | `/api/config/governor/gateways` | Dashboard auth/session | Governor Gateways List | `pkg/dashboard/api.go:292` | -| `PUT` | `/api/config/governor/gateways` | Owner only | Governor Gateways Upsert | `pkg/dashboard/api.go:293` | -| `DELETE` | `/api/config/governor/gateways/{name}` | Owner only | Governor Gateways Delete | `pkg/dashboard/api.go:294` | -| `POST` | `/api/config/governor/gateways/{name}/test` | Dashboard auth/session | Governor Gateways Test | `pkg/dashboard/api.go:295` | -| `POST` | `/api/config/governor/gateways/discover` | Owner only | Governor Gateways Discover | `pkg/dashboard/api.go:296` | -| `POST` | `/api/config/governor/agents` | Owner only | Governor Add Agent | `pkg/dashboard/api.go:301` | -| `DELETE` | `/api/config/governor/agents/{name}` | Owner only | Governor Remove Agent | `pkg/dashboard/api.go:302` | -| `PUT` | `/api/config/governor/repos` | Owner only | Governor Repos | `pkg/dashboard/api.go:303` | -| `POST` | `/api/config/governor/repos/check-access` | Dashboard auth/session | Governor Repo Check Access | `pkg/dashboard/api.go:307` | -| `PUT` | `/api/config/github` | Owner only | Config GitHub | `pkg/dashboard/api.go:308` | -| `GET` | `/api/config/github/forge-apps` | Dashboard auth/session | Config GitHub Forge Apps | `pkg/dashboard/api.go:311` | -| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:360` | -| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:361` | -| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:362` | -| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](../compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:352` | -| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](../compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:353` | -| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:354` | -| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](../compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:355` | -| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:356` | -| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:357` | -| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:358` | -| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:526` | +| `POST` | `/api/config/governor/bob/test` | Dashboard auth/session | Governor Bob Key Test | `pkg/dashboard/api.go:289` | +| `POST` | `/api/config/governor/litellm/test` | Dashboard auth/session | Governor Lite LLMTest | `pkg/dashboard/api.go:290` | +| `GET` | `/api/config/governor/gateways` | Dashboard auth/session | Governor Gateways List | `pkg/dashboard/api.go:293` | +| `PUT` | `/api/config/governor/gateways` | Owner only | Governor Gateways Upsert | `pkg/dashboard/api.go:294` | +| `DELETE` | `/api/config/governor/gateways/{name}` | Owner only | Governor Gateways Delete | `pkg/dashboard/api.go:295` | +| `POST` | `/api/config/governor/gateways/{name}/test` | Dashboard auth/session | Governor Gateways Test | `pkg/dashboard/api.go:296` | +| `POST` | `/api/config/governor/gateways/discover` | Owner only | Governor Gateways Discover | `pkg/dashboard/api.go:297` | +| `POST` | `/api/config/governor/agents` | Owner only | Governor Add Agent | `pkg/dashboard/api.go:302` | +| `DELETE` | `/api/config/governor/agents/{name}` | Owner only | Governor Remove Agent | `pkg/dashboard/api.go:303` | +| `PUT` | `/api/config/governor/repos` | Owner only | Governor Repos | `pkg/dashboard/api.go:304` | +| `POST` | `/api/config/governor/repos/check-access` | Dashboard auth/session | Governor Repo Check Access | `pkg/dashboard/api.go:308` | +| `PUT` | `/api/config/github` | Owner only | Config GitHub | `pkg/dashboard/api.go:309` | +| `GET` | `/api/config/github/forge-apps` | Dashboard auth/session | Config GitHub Forge Apps | `pkg/dashboard/api.go:312` | +| `GET` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Get | `pkg/dashboard/api.go:361` | +| `PUT` | `/api/config/sidebar` | Dashboard auth/session | Sidebar Set | `pkg/dashboard/api.go:362` | +| `GET` | `/api/config/backends` | Dashboard auth/session | Backends | `pkg/dashboard/api.go:363` | +| `GET` | `/api/compliance/status` | Merger or owner | Evaluated compliance controls for the frameworks in `compliance.frameworks` (`compliance.Report`, JSON): per control `meets`/`deviates`/`off`/`not_covered` with current vs recommended value per mapped setting, a summary, and the non-certification disclaimer — see [Compliance](../compliance.md) ([#11078](https://github.com/hivecommons/hive/issues/11078)) | `pkg/dashboard/api.go:353` | +| `GET` | `/api/compliance/posture` | Merger or owner | Posture-check catalogue, effective interval / window / history retention, and the latest run (`compliance.PostureRun`: schedule/manual trigger, `triggered_by` identity, and per-check `pass`/`fail`/`skip`/`error`, detail, control ids, evidence refs); `?since=` (RFC 3339, `YYYY-MM-DD` or a look-back such as `168h`) adds every run since then, oldest first, capped at 1000 — see [Compliance](../compliance.md#posture-checks-catalogue) ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:354` | +| `POST` | `/api/compliance/posture/run` | Merger or owner | Run the posture checks now, attributed to the authenticated user, and record the pass in the history; audited as `compliance_posture_run`; `409` when no framework is selected or a pass is already running ([#11079](https://github.com/hivecommons/hive/issues/11079)) | `pkg/dashboard/api.go:355` | +| `GET` | `/api/compliance/posture/history` | Owner only | Posture history for `?since=`/`?until=` (default the last 30 days) as one series per check: points (`at`, `status`), status counts, `last_fail_at` and the latest result with detail and evidence refs; at most the 1000 most recent runs — see [Compliance](../compliance.md#evidence-exports-and-attestations) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:356` | +| `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:357` | +| `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:358` | +| `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:359` | +| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:527` | | `GET` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Get | `pkg/dashboard/api.go:215` | | `PUT` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Set | `pkg/dashboard/api.go:216` | @@ -91,8 +91,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/governor/watchdog` | Owner only | Governor Watchdog | `pkg/dashboard/api.go:225` | | `GET` | `/api/config/escalation` | Owner only | Escalation Config Get | `pkg/dashboard/api.go:228` | | `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:229` | -| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:522` | -| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:523` | +| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:523` | +| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:524` | | `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:232` | | `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:235` | | `PUT` | `/api/config/governor/features` | Owner only | Governor Features | `pkg/dashboard/api.go:241` | @@ -101,15 +101,15 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/config/auto-merge` | Owner only | Auto-Merge Config Get | `pkg/dashboard/api.go:247` | | `PUT` | `/api/config/auto-merge` | Owner only | Auto-Merge Config Set | `pkg/dashboard/api.go:248` | | `GET` | `/api/config/convergence` | Owner only | Convergence Config Get | `pkg/dashboard/api.go:253` | -| `PUT` | `/api/config/convergence` | Owner only | Convergence Config Set | `pkg/dashboard/api.go:254` | -| `GET` | `/api/config/governor/advisory` | Owner only | Governor Advisory Get | `pkg/dashboard/api.go:258` | -| `PUT` | `/api/config/governor/advisory` | Owner only | Governor Advisory Set | `pkg/dashboard/api.go:259` | -| `GET` | `/api/config/governor/replan` | Owner only | Governor Replan Get | `pkg/dashboard/api.go:260` | -| `PUT` | `/api/config/governor/replan` | Owner only | Governor Replan Set | `pkg/dashboard/api.go:261` | -| `GET` | `/api/config/governor/work-source` | Owner only | Governor Work Source Get (Jira secrets/TLS fields reported as set/unset only) | `pkg/dashboard/api.go:271` | -| `PUT` | `/api/config/governor/work-source` | Owner only | Governor Work Source Set (including Jira Data Center TLS settings) | `pkg/dashboard/api.go:272` | -| `PUT` | `/api/config/governor/security` | Owner only | Governor Security | `pkg/dashboard/api.go:273` | -| `GET` | `/api/config/governor/project-observability` | Owner only | Governor Project Observability Get | `pkg/dashboard/api.go:274` | -| `PUT` | `/api/config/governor/project-observability` | Owner only | Governor Project Observability Set | `pkg/dashboard/api.go:275` | -| `GET` | `/api/config/governor/inference-auth` | Owner only | Governor Inference Auth Get | `pkg/dashboard/api.go:290` | -| `PUT` | `/api/config/governor/inference-auth` | Owner only | Governor Inference Auth Set | `pkg/dashboard/api.go:291` | +| `PUT` | `/api/config/convergence` | Owner only | Convergence Config Set | `pkg/dashboard/api.go:255` | +| `GET` | `/api/config/governor/advisory` | Owner only | Governor Advisory Get | `pkg/dashboard/api.go:259` | +| `PUT` | `/api/config/governor/advisory` | Owner only | Governor Advisory Set | `pkg/dashboard/api.go:260` | +| `GET` | `/api/config/governor/replan` | Owner only | Governor Replan Get | `pkg/dashboard/api.go:261` | +| `PUT` | `/api/config/governor/replan` | Owner only | Governor Replan Set | `pkg/dashboard/api.go:262` | +| `GET` | `/api/config/governor/work-source` | Owner only | Governor Work Source Get (Jira secrets/TLS fields reported as set/unset only) | `pkg/dashboard/api.go:272` | +| `PUT` | `/api/config/governor/work-source` | Owner only | Governor Work Source Set (including Jira Data Center TLS settings) | `pkg/dashboard/api.go:273` | +| `PUT` | `/api/config/governor/security` | Owner only | Governor Security | `pkg/dashboard/api.go:274` | +| `GET` | `/api/config/governor/project-observability` | Owner only | Governor Project Observability Get | `pkg/dashboard/api.go:275` | +| `PUT` | `/api/config/governor/project-observability` | Owner only | Governor Project Observability Set | `pkg/dashboard/api.go:276` | +| `GET` | `/api/config/governor/inference-auth` | Owner only | Governor Inference Auth Get | `pkg/dashboard/api.go:291` | +| `PUT` | `/api/config/governor/inference-auth` | Owner only | Governor Inference Auth Set | `pkg/dashboard/api.go:292` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 57fd430b5e..b94fd39fb8 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/audit` | Read-write role | Audit Log — `{"entries": [...]}` envelope, newest first, capped at 200; response shape and the serve-time `user_name` field in [audit-log.md](../audit-log.md#get-apiaudit) | `pkg/dashboard/api.go:60` | -| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](../agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:496` | +| `GET` | `/api/watchdog/activity` | Read-write role | Watchdog activity readout for the Health tab (#7254): `watchdog-*` audit actions over `?days=` (default 30, clamped to 90) — total / taken / observed / `byAction`, a zero-filled per-day `daily` histogram, per-agent `agents` liveness, and the Observe → Heal `promotion` hint; see [agent-watchdog.md](../agent-watchdog.md#watchdog-activity-strip) | `pkg/dashboard/api.go:497` | | `POST` | `/api/presence` | Dashboard auth/session | Presence | `pkg/dashboard/api.go:63` | | `POST` | `/api/upgrade/announce` | Owner role | Announce a pod roll before it happens ([#15788](https://github.com/hivecommons/hive/issues/15788)): body `{"target": "", "source": "hub"|"operator", "reason": "..."}` (`version` is an alias of `target`; all fields optional). Marks `/api/status`/`/api/version` `upgradeProgress` as `state: pulling, phase: announced` with the target and publishes an SSE `upgrading` event so every open dashboard shows the upgrade animation; a SIGTERM afterwards (or with no announce at all) publishes `phase: restarting` from the same record. The hub sends the same announce through the heartbeat response (`upgrade_announce`) for hub-managed hives; an announce with no kill expires after 15 minutes. Returns `{ok, upgradeProgress}` | `pkg/dashboard/api.go:66` | | `GET` | `/api/prompt-history` | Dashboard auth/session | Prompt History | `pkg/dashboard/api.go:64` | @@ -20,19 +20,19 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/pr-throughput` | Dashboard auth/session | Throughput: pull/merge requests and issues across tracked forges. Counts hive-created PRs/issues, hive/observed reviews, observed PR merges/closures, and issue closes over `?hours=N` (default 24, capped at audit retention; `hours=0` = all time from durable counters in `/data/pr-throughput-counters.json`) with optional exact case-insensitive `?repo=owner/name` and `?role=created\|reviewed\|merged` for the actor trend. The response keeps legacy `opened`/`merged`/`closed`, `merged_by_path`, `buckets`, velocity metrics, previous-window trends, top agents/repos/reasons, and `recorded_since`, and adds `by_actor` (`pr`/`issue` × `created`/`reviewed`/`merged`/`closed` × `hive`/`human`/`other`) plus `series` points (`t`, `hive`, `human`, `other`) for the selected role. | `pkg/dashboard/api.go:87` | -| `GET` | `/api/convergence/soak` | Owner only | Convergence Soak Status | `pkg/dashboard/api.go:257` | -| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:448` | -| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:451` | -| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:452` | -| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:454` | -| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:455` | +| `GET` | `/api/convergence/soak` | Owner only | Convergence Soak Status | `pkg/dashboard/api.go:258` | +| `POST` | `/api/plan/from-issue` | Dashboard auth/session | Plan From Issue | `pkg/dashboard/api.go:449` | +| `GET` | `/api/plan/{epicID}` | Dashboard auth/session | Plan Tree | `pkg/dashboard/api.go:452` | +| `POST` | `/api/plan/{epicID}/approve` | Owner only | Plan Approve | `pkg/dashboard/api.go:453` | +| `POST` | `/api/plan/{epicID}/reject` | Owner only | Plan Reject | `pkg/dashboard/api.go:455` | +| `POST` | `/api/plan/{epicID}/child/{childID}` | Owner only | Plan Child Action | `pkg/dashboard/api.go:456` | | `POST` | `/api/linear/agent/install` | Owner only | Linear Agent Install (OAuth start) | `pkg/dashboard/api_linear_agent.go:215` | | `GET` | `/api/linear/agent/status` | Owner only | Linear Agent Status | `pkg/dashboard/api_linear_agent.go:216` | | `POST` | `/api/linear/agent/disconnect` | Owner only | Linear Agent Disconnect | `pkg/dashboard/api_linear_agent.go:217` | -| `GET` | `/api/upstream-watch` | Owner only | Upstream-watch divergence: per watched fork the upstream followed, the watermark and last-run time, surfaced/ported/dismissed/skipped counts, and the recent upstream refs with their upstream URL, diff URL, fork issue number and state. Read-only — it reads the config and the watch state file, never GitHub. Also carries `interval`, `project_repos` and each repo's configured entry (`config`) for the editor. | `pkg/dashboard/api.go:299` | -| `PUT` | `/api/config/upstream-watch` | Owner only | Upstream Watch Config Update: edit `upstream_watch` from the Features tab. Body `{enabled?, interval?, repos?: {: {upstream?, sources?, pr_labels?, max_issues_per_run?, label?} \| null}}` — absent keys are unchanged, `null` removes a repo, keys are stored as the bare repo name. Validated with the loader's rules before the config changes (400 on failure), then persisted and audited as `config_upstream_watch`. | `pkg/dashboard/api.go:300` | +| `GET` | `/api/upstream-watch` | Owner only | Upstream-watch divergence: per watched fork the upstream followed, the watermark and last-run time, surfaced/ported/dismissed/skipped counts, and the recent upstream refs with their upstream URL, diff URL, fork issue number and state. Read-only — it reads the config and the watch state file, never GitHub. Also carries `interval`, `project_repos` and each repo's configured entry (`config`) for the editor. | `pkg/dashboard/api.go:300` | +| `PUT` | `/api/config/upstream-watch` | Owner only | Upstream Watch Config Update: edit `upstream_watch` from the Features tab. Body `{enabled?, interval?, repos?: {: {upstream?, sources?, pr_labels?, max_issues_per_run?, label?} \| null}}` — absent keys are unchanged, `null` removes a repo, keys are stored as the bare repo name. Validated with the loader's rules before the config changes (400 on failure), then persisted and audited as `config_upstream_watch`. | `pkg/dashboard/api.go:301` | -| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:533` | +| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:534` | | `GET` | `/api/widget` | Dashboard auth/session | Widget | `pkg/dashboard/api.go:115` | | `GET` | `/api/pane/{agent}` | Dashboard auth/session | Pane | `pkg/dashboard/api.go:120` | | `GET` | `/api/role` | Dashboard auth/session | Role | `pkg/dashboard/api.go:134` | @@ -42,11 +42,11 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/summaries` | Dashboard auth/session | Summaries | `pkg/dashboard/api.go:187` | | `GET` | `/api/stage-tracker` | Dashboard auth/session | Stage Tracker: every open issue and pull request of the watched repos for the Hive activity monitor — actionable, held, claimed, in-progress, covered-by-PR, filtered (`needs-*`), draft and exempt-labelled items alike, each with `stage`, `stage_since`, `labels`, `created_at`, `author_kind`, `modifiers`, `claimant: {kind: hive-agent|clanker|person, login, agent, hive, since, source: claim-ledger|claim-comment|assignee|kick-history|author}` naming who holds a claimed / in-progress issue or a PR under review or rework (absent when nobody is named; mirrored onto `linked_prs[]`), (PRs) `sentinel_alert` / `sentinel_findings` / `sentinel_reason` when the sentinel flagged the PR, `decision_needed` on a row parked on a person (085 / 045 / 015, the needs-human pill; mirrored onto the issue's `linked_prs[]` entry): `{question, asked_by, since, blocker, blocker_reason, options: [{action, label, enabled, reason_if_disabled}]}` — the reviewer's own decision question when the verdict record has one, the blocker in plain words, and the applicable actions under the row context menu's action ids (`decide:delegate` first), (PRs) `lineage` on the hive's copy of an adopted PR — `{kind: adopted|superseded|rebased-by-hive, from_number, from_url, from_author, from_author_kind, to_number, to_url, at, reason}`, derived from the stage store's adoption record or the `hive/adopt-` head branch, mirrored onto the issue's `linked_prs[]` entry; absent on every other row — and (issues) `linked_prs`; items with no stage yet report stage `0` "Open"; `needs_human_labels` is the "a person must act" label set behind the monitor's Needs human facet (the same list as `/api/status.labelPresets[needs-human]`); `recently_merged` lists the PRs (`repo`, `number`, `merged_at`) whose `pr_merged` audit entry is under 30 minutes old, so a client diffing two payloads can tell a merge from a close; a PR in a conflict-aware merge train carries `merge_train` `{id, state, front, position, size, attempt, max_attempts, overlap_files, ahead, length, head_number, head_state, head_reason, head_since, eta_s, eta_at, eta_basis, late, stalled, stalled_s}` — `head_state` one of `needs-human` / `ci` / `soaking` / `ready` / `merging` / `fixing` / `unknown` with `head_reason` in plain words; `eta_s` / `eta_at` the estimated wait for this member's own merge attempt (own remaining soak + each PR ahead's remaining soak or one sweep + the next sweep tick; absent while the head waits on a person or CI), `eta_at` anchored to the member's first estimate at its position so a countdown survives polls, `eta_basis` its short reading ("2 ahead · head soaking 12m · next sweep in 40s"), `late` once now has overrun the first estimate twice over, `stalled` / `stalled_s` once the head has sat longer than `governor.merge_train_stall_warn_s` ([#15395](https://github.com/hivecommons/hive/issues/15395)); a PR the queue holds because another open PR declares the same `Fixes #N` carries no `merge_train` and a `merge_gate` of `{state: "duplicate-of:", duplicate_of, duplicate_issue}` plus the `needs-decision` modifier (the queue keeps one PR per issue: green first, then the earlier-opened); a PR GitHub reports conflicting on its own, or one under repair (stage/070, 071, 072, 074, 085, or a red required check), carries no `merge_train` and is not counted in any member's `size` / `length` — a repaired member the view dropped carries `left_train: {why: fixing|needs-author|returned-to-contributor|adopted-by-hive|needs-decision|checks-failing, note}` instead (mirrored onto the issue's `linked_prs[]` entry), and a train left with one member reports no `merge_train` at all; a PR short of the merge queue (below stage/080) the planner ordered rides no train either — it carries `overlap: {behind, with, files, note}` ("overlaps #15637 (sidebar.html, index.html) — will queue behind it") and no `merge_train`, and counts in no member's `size` — the state slugs (`front`, `queued-behind`) are payload values only, the SPA renders ordinal places ("Merge queued · 1st of 2", with the train's tag when more than one train is on the page). Served from a server-side cache rebuilt at most every `dashboard.stage_tracker.cache_ttl_s` (default `governor.eval_interval_s`) from the spoke's existing snapshot — a request never calls GitHub. Carries `generated_at`, `source` (`cache` or `fresh`), `cache_ttl_s`, an `ETag` and `Cache-Control: private, max-age=`; `If-None-Match` yields 304. Each hit is recorded under REST caller `dashboard:stage-tracker` so `/api/gh-rate-limits` shows it charged nothing. | `pkg/dashboard/api.go:188` | | `POST` | `/api/prs/{owner}/{repo}/{number}/queue-automerge` | Merger/owner role | Queue PRAuto Merge | `pkg/dashboard/api.go:189` | -| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:365` | -| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:427` | -| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:428` | -| `POST` | `/api/chat` | Dashboard auth/session | Chat | `pkg/dashboard/api.go:457` | -| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:491` | +| `GET` | `/api/inference/models/{backend}` | Dashboard auth/session | Inference Models | `pkg/dashboard/api.go:366` | +| `GET` | `/api/hive-id` | Dashboard auth/session | Hive IDGet | `pkg/dashboard/api.go:428` | +| `PUT` | `/api/hive-id` | Owner only | Hive IDSet | `pkg/dashboard/api.go:429` | +| `POST` | `/api/chat` | Dashboard auth/session | Chat | `pkg/dashboard/api.go:458` | +| `GET` | `/api/auth/token` | Public | Auth Token | `pkg/dashboard/api.go:492` | | `GET` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:288` | | `POST` | `/api/v1/` | GitHub token | Contributor v1 API dispatcher | `pkg/dashboard/api_contribute.go:289` | | `GET` | `/api/docs` | Dashboard auth/session | APIDocs | `pkg/dashboard/api_contribute.go:290` | @@ -64,4 +64,4 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:315` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:316` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:317` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1656` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1657` | diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index 9492f45186..81330e9160 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -8,17 +8,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:44` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:45` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:67` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1629` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1630` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1631` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1637` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`) | `pkg/dashboard/server.go:1642` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1643` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1644` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1645` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1646` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1647` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1648` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1630` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1631` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1632` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1638` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`) | `pkg/dashboard/server.go:1643` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1644` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1645` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1646` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1647` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1648` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1649` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/inception.md b/src/docs/api-reference/inception.md index fd323af0aa..dcedf022fc 100644 --- a/src/docs/api-reference/inception.md +++ b/src/docs/api-reference/inception.md @@ -5,17 +5,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:430` | -| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:431` | -| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:432` | -| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:433` | -| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:434` | -| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:435` | -| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:436` | -| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:437` | -| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:438` | -| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:439` | -| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:440` | -| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:441` | -| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:442` | -| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:443` | +| `POST` | `/api/inception/start` | Owner only | Inception Start | `pkg/dashboard/api.go:431` | +| `POST` | `/api/inception/scan` | Owner only | Inception Scan | `pkg/dashboard/api.go:432` | +| `GET` | `/api/inception/state` | Dashboard auth/session | Inception State | `pkg/dashboard/api.go:433` | +| `POST` | `/api/inception/questions` | Owner only | Inception Set Questions | `pkg/dashboard/api.go:434` | +| `POST` | `/api/inception/answer` | Owner only | Inception Answer | `pkg/dashboard/api.go:435` | +| `POST` | `/api/inception/facts` | Owner only | Inception Record Facts | `pkg/dashboard/api.go:436` | +| `GET` | `/api/inception/scaffold` | Dashboard auth/session | Inception Scaffold | `pkg/dashboard/api.go:437` | +| `POST` | `/api/inception/approve` | Owner only | Inception Approve | `pkg/dashboard/api.go:438` | +| `POST` | `/api/inception/reset` | Owner only | Inception Reset | `pkg/dashboard/api.go:439` | +| `GET` | `/api/inception/ideation-facts` | Dashboard auth/session | Inception Ideation Facts | `pkg/dashboard/api.go:440` | +| `GET` | `/api/inception/download` | Dashboard auth/session | Inception Download | `pkg/dashboard/api.go:441` | +| `GET` | `/api/inception/has-files` | Dashboard auth/session | Inception Has Files | `pkg/dashboard/api.go:442` | +| `PUT` | `/api/inception/wiki-name` | Owner only | Inception Rename Wiki | `pkg/dashboard/api.go:443` | +| `POST` | `/api/inception/import` | Owner only | Inception Import | `pkg/dashboard/api.go:444` | diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index 1bfbdc0fb2..f443a7564c 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -5,37 +5,37 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:367` | -| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:368` | -| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; 404 otherwise | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](../public-knowledge-mcp.md)) | `pkg/dashboard/api.go:376` | -| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:369` | -| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:370` | -| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](../knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:371` | -| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:372` | -| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:377` | -| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:378` | -| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:379` | -| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:380` | -| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:381` | -| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:382` | -| `POST` | `/api/knowledge/create` | Dashboard auth/session | Knowledge Create | `pkg/dashboard/api.go:383` | -| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:384` | -| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:390` | -| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:391` | -| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:392` | -| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:393` | -| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:394` | -| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:395` | -| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:396` | -| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:397` | -| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:398` | -| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:399` | -| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:400` | -| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:401` | -| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:402` | -| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:403` | -| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:404` | -| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:405` | +| `GET` | `/api/knowledge` | Dashboard auth/session | Paginated knowledge facts: `type` filter, `limit` (default 50, max 200; invalid/non-positive values use the default), `offset` (default 0). Returns `enabled`, `facts`, `count` (this page), `total`, `limit`, `offset`, `has_more`; ordered by layer then slug. Offsets are not snapshot cursors: concurrent edits may shift pages. | `pkg/dashboard/api.go:368` | +| `GET` | `/api/knowledge/export` | Dashboard auth/session | Knowledge Export | `pkg/dashboard/api.go:369` | +| `POST` | `/mcp/knowledge` | Public (read-only) when the dashboard setting or `HIVE_PUBLIC_KNOWLEDGE` is on; 404 otherwise | Public Knowledge MCP endpoint — `initialize`, `tools/list`, `tools/call` for `knowledge_search`/`knowledge_toc`/`knowledge_get`/`knowledge_export` ([docs](../public-knowledge-mcp.md)) | `pkg/dashboard/api.go:377` | +| `GET` | `/api/knowledge/search` | Dashboard auth/session | Knowledge Search | `pkg/dashboard/api.go:370` | +| `GET` | `/api/knowledge/toc` | Dashboard auth/session | Scoped knowledge table of contents: capped, body-free entries (`id`, `title`, `type`, `layer`, `repo`, `tags`, `status`, `confidence`, `updated`, `size_bytes`, `source`) plus `total`/`returned`/`truncated`. Query: `layers`, `repos`, `types`, `tags` (comma-separated), `limit` (default 50, max 200), `include_states` (operator opt-in for `draft`/`deprecated`/`superseded`/`all`), `format=prompt` with `max_chars` for a bounded markdown rendering, `agent` to apply that agent's `knowledge.agent_scopes` entry (the other parameters can only narrow it). Approved knowledge only by default ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:371` | +| `GET` | `/api/knowledge/effective` | Dashboard auth/session | Effective knowledge for one agent (`agent` required): the agent's `knowledge.agent_scopes` entry (unrestricted when it has none) intersected with the request scope (approved only unless `include_states` is set). Returns `scope` (resolved scope applied, with `configured`), `included` TOC entries, `excluded` entries each with a `reason` (`lifecycle`, `layer`, `repo`, `type` or `tag` — first failing check, in that order), `included_total`/`excluded_total`/`included_returned`/`excluded_returned` and `truncated`; `limit` caps each list (default 50, max 200) ([docs](../knowledge-toc.md#effective-knowledge)) | `pkg/dashboard/api.go:372` | +| `GET` | `/api/knowledge/entry/{id}` | Dashboard auth/session | Full read of one knowledge entry: `markdown` (front-matter plus body) and the fact. Takes the same scope, `include_states` and `agent` query as the table of contents and returns 404 for an entry outside it ([docs](../knowledge-toc.md)) | `pkg/dashboard/api.go:373` | +| `GET` | `/api/knowledge/public` | Dashboard auth/session | Public Knowledge MCP sharing status (`enabled`, `tags`, `source`, endpoint URL) | `pkg/dashboard/api.go:378` | +| `PUT` | `/api/knowledge/public` | Owner only | Persist public Knowledge MCP sharing setting; overrides env vars until changed | `pkg/dashboard/api.go:379` | +| `GET` | `/api/knowledge/health` | Dashboard auth/session | Knowledge Health | `pkg/dashboard/api.go:380` | +| `GET` | `/api/knowledge/stats` | Dashboard auth/session | Knowledge Stats | `pkg/dashboard/api.go:381` | +| `GET` | `/api/knowledge/graph` | Dashboard auth/session | Knowledge Graph | `pkg/dashboard/api.go:382` | +| `GET` | `/api/knowledge/fact-history` | Dashboard auth/session | Fact History | `pkg/dashboard/api.go:383` | +| `POST` | `/api/knowledge/create` | Dashboard auth/session | Knowledge Create | `pkg/dashboard/api.go:384` | +| `POST` | `/api/knowledge/import` | Dashboard auth/session | Knowledge Import | `pkg/dashboard/api.go:385` | +| `POST` | `/api/knowledge/promote` | Dashboard auth/session | Knowledge Promote | `pkg/dashboard/api.go:391` | +| `GET` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs List | `pkg/dashboard/api.go:392` | +| `POST` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Add | `pkg/dashboard/api.go:393` | +| `DELETE` | `/api/knowledge/subscriptions` | Dashboard auth/session | Knowledge Subs Remove | `pkg/dashboard/api.go:394` | +| `PUT` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Update | `pkg/dashboard/api.go:395` | +| `DELETE` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Delete | `pkg/dashboard/api.go:396` | +| `GET` | `/api/knowledge/{layer}` | Dashboard auth/session | Knowledge Layer | `pkg/dashboard/api.go:397` | +| `GET` | `/api/knowledge/{layer}/{slug}` | Dashboard auth/session | Knowledge Fact | `pkg/dashboard/api.go:398` | +| `PUT` | `/api/knowledge/enabled` | Owner only | Knowledge Toggle | `pkg/dashboard/api.go:399` | +| `GET` | `/api/knowledge/bead-synthesizer` | Dashboard auth/session | Bead Synth Status | `pkg/dashboard/api.go:400` | +| `PUT` | `/api/knowledge/bead-synthesizer/enabled` | Owner only | Bead Synth Toggle | `pkg/dashboard/api.go:401` | +| `GET` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults List | `pkg/dashboard/api.go:402` | +| `POST` | `/api/knowledge/vaults` | Owner only | Vaults Connect | `pkg/dashboard/api.go:403` | +| `DELETE` | `/api/knowledge/vaults` | Dashboard auth/session | Vaults Disconnect | `pkg/dashboard/api.go:404` | +| `POST` | `/api/knowledge/vaults/reindex` | Dashboard auth/session | Vaults Reindex | `pkg/dashboard/api.go:405` | +| `GET` | `/api/knowledge/vaults/{name}/facts` | Dashboard auth/session | Vault Facts | `pkg/dashboard/api.go:406` | | `GET` | `/api/config/knowledge/connectors` | Dashboard auth/session | Knowledge connectors config (name, type, layer, scope, env/file auth reference) and registered types | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/connectors` | Owner only | Replace `knowledge.connectors` after validation; inline secrets rejected | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/connectors/status` | Dashboard auth/session | Per-connector status pill, last sync, pages/facts, last error, plus the `publish` mirror status | `pkg/dashboard/api.go` | @@ -45,19 +45,19 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/knowledge/agent-scopes` | Owner only | Replace `knowledge.agent_scopes` after validating layer and state names; persisted and audited as `config_knowledge_agent_scopes` | `pkg/dashboard/api.go` | | `GET` | `/api/config/knowledge/pipeline` | Owner only | Scalar bead synthesizer, curator, code map and primer settings, plus the read-only engine, vault paths and primer merge strategy ([docs](../settings.md#knowledge)) | `pkg/dashboard/api.go` | | `PUT` | `/api/config/knowledge/pipeline` | Owner only | Update those settings after `config.ValidateKnowledgePipeline`; 0 or empty restores the default; the bead synthesizer switch applies at once, the rest after a restart; audited as `config_knowledge_pipeline` | `pkg/dashboard/api.go` | -| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:415` | -| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:416` | -| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:417` | -| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:418` | -| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:419` | -| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:420` | -| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:421` | -| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:422` | -| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:423` | -| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:424` | -| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:425` | -| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:388` | -| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:389` | +| `GET` | `/api/knowledge/git-sources` | Dashboard auth/session | Git Sources List | `pkg/dashboard/api.go:416` | +| `POST` | `/api/knowledge/git-sources` | Owner only | Git Sources Connect | `pkg/dashboard/api.go:417` | +| `DELETE` | `/api/knowledge/git-sources` | Owner only | Git Sources Disconnect | `pkg/dashboard/api.go:418` | +| `POST` | `/api/knowledge/obsidian/sync` | Dashboard auth/session | Obsidian Sync | `pkg/dashboard/api.go:419` | +| `GET` | `/api/knowledge/documents` | Dashboard auth/session | Documents List | `pkg/dashboard/api.go:420` | +| `POST` | `/api/knowledge/documents` | Dashboard auth/session | Documents Import | `pkg/dashboard/api.go:421` | +| `GET` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Get | `pkg/dashboard/api.go:422` | +| `DELETE` | `/api/knowledge/documents/{slug}` | Dashboard auth/session | Document Delete | `pkg/dashboard/api.go:423` | +| `POST` | `/api/knowledge/documents/{slug}/reimport` | Dashboard auth/session | Document Reimport | `pkg/dashboard/api.go:424` | +| `GET` | `/api/knowledge/context7/search` | Dashboard auth/session | Context7 Search | `pkg/dashboard/api.go:425` | +| `POST` | `/api/knowledge/cleanup-orphans` | Dashboard auth/session | Cleanup Orphans | `pkg/dashboard/api.go:426` | +| `GET` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channels List | `pkg/dashboard/api.go:389` | +| `POST` | `/api/knowledge/channels` | Dashboard auth/session | Knowledge Channel Create | `pkg/dashboard/api.go:390` | | `GET` | `/api/knowledge/connectors` | Dashboard auth/session | Knowledge connector statuses and publish mirror status (`{"connectors": [...], "publish": {...}\|null}`, see [knowledge-connectors.md](../knowledge-connectors.md)) | `pkg/dashboard/api_knowledge_connectors.go:23` | | `POST` | `/api/knowledge/connectors/{name}/sync` | Owner only | Start one connector sync now (202; 404 unknown, 409 running) | `pkg/dashboard/api_knowledge_connectors.go:24` | | `POST` | `/api/knowledge/publish/sync` | Owner only | Run a publish mirror batch now (202 with publish status; 503 mirror not running, 409 batch in progress) | `pkg/dashboard/api_knowledge_connectors.go:25` | diff --git a/src/docs/api-reference/nous.md b/src/docs/api-reference/nous.md index 6bc6ea82e3..fa3d46b69b 100644 --- a/src/docs/api-reference/nous.md +++ b/src/docs/api-reference/nous.md @@ -5,28 +5,28 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:459` | -| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:460` | -| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:461` | -| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:462` | -| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:463` | -| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:464` | -| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:465` | -| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:466` | -| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:467` | -| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:468` | -| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:469` | -| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:470` | -| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:471` | -| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:472` | -| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:473` | -| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:474` | -| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:475` | -| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:476` | -| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:477` | -| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:478` | -| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:479` | -| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:480` | -| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:481` | -| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:482` | -| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:483` | +| `GET` | `/api/nous/status` | Dashboard auth/session | Nous Status — includes `experimentLoop` (`dir`, `lastWriteAt`, `running`, `idleAfterSeconds`): whether the external experiment loop, the only producer of proposals, has written under `/var/run/nous/governor` within 24 evaluation cycles; ready cards then carry `experimentLoopIdle` ([#12541](https://github.com/hivecommons/hive/issues/12541)) | `pkg/dashboard/api.go:460` | +| `GET` | `/api/nous/ledger` | Dashboard auth/session | Nous Ledger | `pkg/dashboard/api.go:461` | +| `GET` | `/api/nous/principles` | Dashboard auth/session | Nous Principles | `pkg/dashboard/api.go:462` | +| `POST` | `/api/nous/approve` | Owner only | Nous Approve | `pkg/dashboard/api.go:463` | +| `POST` | `/api/nous/abort` | Owner only | Nous Abort | `pkg/dashboard/api.go:464` | +| `PUT` | `/api/nous/mode` | Owner only | Nous Mode | `pkg/dashboard/api.go:465` | +| `PUT` | `/api/nous/scope` | Owner only | Nous Scope | `pkg/dashboard/api.go:466` | +| `GET` | `/api/nous/phase` | Dashboard auth/session | Nous Phase | `pkg/dashboard/api.go:467` | +| `PUT` | `/api/nous/gate-decision` | Owner only | Nous Gate Decision | `pkg/dashboard/api.go:468` | +| `GET` | `/api/nous/gate-pending` | Dashboard auth/session | Nous Gate Pending | `pkg/dashboard/api.go:469` | +| `POST` | `/api/nous/gate-respond` | Owner only | Nous Gate Respond | `pkg/dashboard/api.go:470` | +| `GET` | `/api/nous/gate-response` | Dashboard auth/session | Nous Gate Response | `pkg/dashboard/api.go:471` | +| `GET` | `/api/nous/config` | Dashboard auth/session | Nous Config Get | `pkg/dashboard/api.go:472` | +| `PUT` | `/api/nous/config/goals` | Owner only | Nous Config Goals — legacy; the dashboard no longer exposes the research question in the ⚙ dialog, use `PATCH /api/nous/campaigns/{id}` | `pkg/dashboard/api.go:473` | +| `PUT` | `/api/nous/config/repos` | Owner only | Nous Config Repos — the allow-list of repos experiments may target (empty = any) | `pkg/dashboard/api.go:474` | +| `PUT` | `/api/nous/config/output` | Owner only | Nous Config Output | `pkg/dashboard/api.go:475` | +| `PUT` | `/api/nous/config/fast-fail` | Owner only | Nous Config Fast Fail | `pkg/dashboard/api.go:476` | +| `PUT` | `/api/nous/config/schedule` | Owner only | Nous Config Schedule | `pkg/dashboard/api.go:477` | +| `PUT` | `/api/nous/config/controllables` | Owner only | Nous Config Controllables | `pkg/dashboard/api.go:478` | +| `PUT` | `/api/nous/config/principles` | Owner only | Nous Config Principles | `pkg/dashboard/api.go:479` | +| `DELETE` | `/api/nous/principles/{id}` | Owner only | Nous Delete Principle | `pkg/dashboard/api.go:480` | +| `POST` | `/api/nous/campaign/start` | Owner only | Nous Campaign Start — archives the active campaign and starts a new experiment whose baseline restarts at 0 ([#12501](https://github.com/hivecommons/hive/issues/12501)) | `pkg/dashboard/api.go:481` | +| `GET` | `/api/nous/campaigns` | Any | Nous Campaigns List — every experiment with its per-campaign baseline, scope, results and stage ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:482` | +| `POST` | `/api/nous/campaigns` | Owner only | Nous Campaigns Create — starts an experiment alongside the running ones; 409 `scope_conflict` when a running experiment shares a repo target or both are hive-wide ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:483` | +| `PATCH` | `/api/nous/campaigns/{id}` | Owner only | Nous Campaign Patch — pause / resume / archive, or edit question, hypothesis and scope. Hypothesis/tag edits save in place (`status: "updated"`); a changed question or repo set needs `restart: true` (else 409 `restart_required`) and archives the campaign as version N while creating version N+1 with `restartedFrom` (`status: "restarted", campaign, archived`); resume, scope widening and restarts re-run the conflict guard ([#12532](https://github.com/hivecommons/hive/issues/12532)) | `pkg/dashboard/api.go:484` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index ee8d28634a..9b905f0de6 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -5,25 +5,25 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| -| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:318` | -| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:319` | -| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:320` | +| `GET` | `/api/packs` | Dashboard auth/session | Packs List | `pkg/dashboard/api.go:319` | +| `POST` | `/api/packs/{level}/apply` | Owner only | Pack Apply | `pkg/dashboard/api.go:320` | +| `PUT` | `/api/packs/level` | Owner only | Set the ACMM level. Body `{"level": N}` persists and applies the pack; optional `{"release_level_holds": true}` performs a deliberate one-off release of Hive App level-applied `hold` labels for the transition. Level holds are otherwise never auto-released. When the change makes the L6 self-authored merge sweep eligible, the success body adds `self_merge_sweep_active`, `level_changed_at`, `level_holds_pending: [{repo, number, title, url, agent}]`, and `repos: [{repo, auto_merge}]` so the dashboard can show the owner an informational notice and repo toggles. | `pkg/dashboard/api.go:321` | | `POST` | `/api/repos/rescan` | Dashboard auth/session | Re-enumerate watched repositories' open issues/PRs for the dashboard Projects cards. The scan is read-only with respect to agents/governor actions, collapses concurrent requests, and debounces repeated presses for 30 seconds (`repoRescanDebounce`) so the button cannot hammer GitHub | `pkg/dashboard/api.go`, `pkg/dashboard/api_repos_rescan.go` | | `POST` | `/api/repos/pause` | Owner | Quiet ONE repository without stopping the hive: agents stop writing to it and stop being handed work on it, while it keeps its dashboard card and ACMM eval. Body `{"repo": "...", "reason": "..."}` — the repo travels in the body because a `project.repos` entry may be an explicit cross-org `owner/name`. Records who/when/why; `changed: false` marks a no-op re-pause and leaves the original provenance intact; `persisted: false` means the pause is in force but could not be written to config. Rejects a repo outside `project.repos`. See [per-repo agent pause](../repo-pause.md) | `pkg/dashboard/api_repo_pause.go` | | `POST` | `/api/repos/resume` | Owner | Lift a repository's pause. Unlike pause, does not require the repo to be in `project.repos`, so a pause left behind by a removed repo can still be cleared | `pkg/dashboard/api_repo_pause.go` | | `GET` | `/api/repos/pauses` | Dashboard auth/session | Every repo pause with its provenance. Read-only — the same state is already on the repository cards | `pkg/dashboard/api_repo_pause.go` | -| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:335`, `pkg/dashboard/api_repo_hold.go` | +| `GET` | `/api/repos/{owner}/{repo}/hold-permission` | Dashboard auth/session | Reports whether the signed-in user can toggle hold chips for a repository card. Owners are allowed; otherwise the server checks GitHub `repos/{owner}/{repo}/collaborators/{user}/permission` and briefly caches the effective permission | `pkg/dashboard/api.go:336`, `pkg/dashboard/api_repo_hold.go` | | `GET` | `/api/repos/{owner}/{repo}/items/{number}` | Dashboard session | Per-item detail for the activity monitor's decision badge: `{repo, number, stage, stage_slug, operator_actions[], operator_action_count, last_operator_action}`. `operator_actions[]` is the server-side ledger of every row action submitted from the dashboard (hold/release, decision, decide-delegated, send-to-human, accept-triage — recorded by `recordOperatorAction` around those routes) as `{action, actor, at, note, target, outcome: pending|applied|no-op|failed, detail, head_sha, stage}`; `last_operator_action` adds the derived `state` (`waiting|addressed|stuck|pending|failed`), `stuck_after_seconds` and `followed[]` (the stage hops after the action). The same `last_operator_action` / `operator_action_count` ride every stage-tracker row and linked PR. Persisted in `/data/operator-actions.json`; `HIVE_OPERATOR_ACTION_STUCK_AFTER` (default `30m`) sets the stuck window. Rows also carry `rebase_merge: {state: approved|rebasing|ci-pending|merged|failed, actor, base, text, fail_reason, conflict_files[], armed_at}` while a *Rebase, then merge as maintainer* follow-through is live or just settled (persisted in `/data/rebase-merge-followups.json`; `fail_reason: foreign-head` when the head moved after the approval by someone other than the hive, so nothing merged), and `rebase: {attempt, max, exhausted, head, text}` on a conflicting PR the fixer is rebasing under the two-attempt loop guard (`/data/rebase-attempts.json`). | `pkg/dashboard/api.go`, `pkg/dashboard/operator_actions.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/` label; `{"held": false}` removes exactly the labels causing the hold (`hive-pause/` and/or generic hold labels) and never removes `hive/` provenance. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-.json` with audit-backed, provenance-only, and ambiguous legacy `hive/` classifications. | `pkg/dashboard/api.go:339`, `pkg/dashboard/api_repo_hold.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix"|"adopt"|"reroute"|"re-author"|"merge-now"|"queue-merge", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458", "force": true, "to": ""|"any"}` (`force` only with `adopt`, `to` only with `reroute`; an unknown `decision` is a 400). On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision ` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR, and on a contributor's clanker PR (`author/clanker` / hive attribution markers) where the text is posted to the contributor's hive and the PR moves to `stage/072-returned-to-contributor` (`status: "direction-returned-to-contributor"`); a person's PR, which no hive can carry direction to, answers 409. The close variants post a comment naming the operator (and the superseding PR when `superseded_by` is given) and close the item as not planned. `adopt` and `re-author` require `type: "pr"` and queue adoption via the `adoption/supersede` label (409 unless `governor.labels.stage.adoption` is enabled). `adopt` with `"force": true` is *Adopt now* ([#15038](https://github.com/hivecommons/hive/issues/15038)): it bypasses the author's presence, the grace window and `maintainer_policy`, marks the stage record and nudges the automerge sweep so the adoption engine supersedes the head at once (responds `status: "already-adopted"` when nothing is left to do; 400 on the hive's own PR). `reroute` (`type: "pr"`, `"to": ""|"any"`) hands the PR's follow-up work to another online clanker through the contributor fix queue, posts a `` note and arms a pickup timer that falls back to Adopt now after `governor.reroute.pickup_timeout`; answers `status: "rerouted"`, 409 when no clanker is online or adoption is disabled. On a PR at 090, `queue-merge` (*Approve and let hive merge*, the lead action) records an approval attributed to the operator and applies the automerge label, returning `status: "queued-for-merge"` (on the hive's own PR, which the App cannot review, the queue note is a comment and the operator's approval is recorded as `approve` records it — `released` lists `approval-recorded`, `admitted`); hive's merge sweep merges on its next tick once CI is green and the head has soaked, and that recorded operator approval is the one thing that lets the sweep merge an untrusted-author PR (the merge comment names the operator). `merge-now` (*Merge now as maintainer*, secondary) posts an attributed comment and merges immediately through the hive's GitHub App, returning `status: "merged"`. Both require `type: "pr"` and a verified owner or `merger` role in addition to the route's repo permission check; both refuse closed/merged PRs and the operator's own PR. `merge-now` on a PR whose `mergeable_state` is `dirty`/`behind` instead answers `202` with `status: "rebase-queued"` (approval recorded, rebase kicked, merge follows via the follow-through once the hive's own rebase is green; a head pushed by anyone else fails the follow-through closed; `head-not-pushable` refusal on a fork without maintainer edits). `approve` on such a PR adds `rebase: {attempt, max, exhausted, head, text}` for the rebase episode it opened. Every decision writes an audit-log entry (`repo_item_decision_*`, e.g. `repo_item_decision_adopt_now`, `repo_item_decision_reroute`). | `pkg/dashboard/api.go:340`, `pkg/dashboard/api_repo_item_decision.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/` label; `{"held": false}` removes exactly the labels causing the hold (`hive-pause/` and/or generic hold labels) and never removes `hive/` provenance. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-.json` with audit-backed, provenance-only, and ambiguous legacy `hive/` classifications. | `pkg/dashboard/api.go:340`, `pkg/dashboard/api_repo_hold.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix"|"adopt"|"reroute"|"re-author"|"merge-now"|"queue-merge", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458", "force": true, "to": ""|"any"}` (`force` only with `adopt`, `to` only with `reroute`; an unknown `decision` is a 400). On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision ` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR, and on a contributor's clanker PR (`author/clanker` / hive attribution markers) where the text is posted to the contributor's hive and the PR moves to `stage/072-returned-to-contributor` (`status: "direction-returned-to-contributor"`); a person's PR, which no hive can carry direction to, answers 409. The close variants post a comment naming the operator (and the superseding PR when `superseded_by` is given) and close the item as not planned. `adopt` and `re-author` require `type: "pr"` and queue adoption via the `adoption/supersede` label (409 unless `governor.labels.stage.adoption` is enabled). `adopt` with `"force": true` is *Adopt now* ([#15038](https://github.com/hivecommons/hive/issues/15038)): it bypasses the author's presence, the grace window and `maintainer_policy`, marks the stage record and nudges the automerge sweep so the adoption engine supersedes the head at once (responds `status: "already-adopted"` when nothing is left to do; 400 on the hive's own PR). `reroute` (`type: "pr"`, `"to": ""|"any"`) hands the PR's follow-up work to another online clanker through the contributor fix queue, posts a `` note and arms a pickup timer that falls back to Adopt now after `governor.reroute.pickup_timeout`; answers `status: "rerouted"`, 409 when no clanker is online or adoption is disabled. On a PR at 090, `queue-merge` (*Approve and let hive merge*, the lead action) records an approval attributed to the operator and applies the automerge label, returning `status: "queued-for-merge"` (on the hive's own PR, which the App cannot review, the queue note is a comment and the operator's approval is recorded as `approve` records it — `released` lists `approval-recorded`, `admitted`); hive's merge sweep merges on its next tick once CI is green and the head has soaked, and that recorded operator approval is the one thing that lets the sweep merge an untrusted-author PR (the merge comment names the operator). `merge-now` (*Merge now as maintainer*, secondary) posts an attributed comment and merges immediately through the hive's GitHub App, returning `status: "merged"`. Both require `type: "pr"` and a verified owner or `merger` role in addition to the route's repo permission check; both refuse closed/merged PRs and the operator's own PR. `merge-now` on a PR whose `mergeable_state` is `dirty`/`behind` instead answers `202` with `status: "rebase-queued"` (approval recorded, rebase kicked, merge follows via the follow-through once the hive's own rebase is green; a head pushed by anyone else fails the follow-through closed; `head-not-pushable` refusal on a fork without maintainer edits). `approve` on such a PR adds `rebase: {attempt, max, exhausted, head, text}` for the rebase episode it opened. Every decision writes an audit-log entry (`repo_item_decision_*`, e.g. `repo_item_decision_adopt_now`, `repo_item_decision_reroute`). | `pkg/dashboard/api.go:341`, `pkg/dashboard/api_repo_item_decision.go` | | `POST` / `GET` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated` | Same as `…/hold` (owner or repo write), signed-in GitHub user; `decisions.delegate.enabled` (403 when off) | *Let hive decide*: enqueue a delegated-decision job in which the hive decides the parked item itself as the owner's delegate under `pkg/delegatedecision.PolicyText` and records it as a human decision (`/hive decision` comment with the `` marker, park labels off, `triage/accepted`, queue approval on a PR head). Body `{"type": "issue"|"pr", "reason", "caption", "sentinel_reason"}`. `POST` answers 202 with the job (`{id, status: "running"|"done"|"failed", decision: {verdict, rule, text, rationale, actions, keeps_park, ladder?: {decision, reason, ruled_out[]}}, notes, error}`; `ladder` is set on contributor-PR decisions and names the rung that decided — `already-fixed` → `mergeable-as-is` → `rebase-then-merge` → `return-to-contributor` → `contributor-offline` — and why each rung above it did not apply); 409 with the running job while one is in flight; `GET` reads it. Rails: never merges, refuses items wearing `hold`, never grants secrets/permissions (`needs-owner` + `hold` instead), recommends only on hive policy/governance questions (keeps 085). Audited as `repo_item_decision_delegated[_failed]`. See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:329`, `pkg/dashboard/api_repo_item_decide_delegated.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:343`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:344` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:345` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:540`, `pkg/dashboard/api_repo_item_accept_triage.go` | -| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:347`, `pkg/dashboard/api_acmm_eval.go` | -| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:348` | -| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](../acmm-advisor.md) | `pkg/dashboard/api.go:349` | -| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](../advisory.md#owner-advice) | `pkg/dashboard/api.go:351`, `pkg/dashboard/api_hive_advice.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:344`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:345` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:346` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:541`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:348`, `pkg/dashboard/api_acmm_eval.go` | +| `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:349` | +| `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](../acmm-advisor.md) | `pkg/dashboard/api.go:350` | +| `GET` | `/api/hive-advice` | Dashboard auth/session | Weekly owner advice (`hiveadvisor.Result`, JSON) shared with the advisory digest: recommendations with `items`, `links` (Overview export paths) and `bands`; `bands[]` rule text; `counts` naming the governor actionable queue vs. the Overview total. Which recommendations show is frozen for 7 days; their numbers refresh every call — see [Owner advice](../advisory.md#owner-advice) | `pkg/dashboard/api.go:352`, `pkg/dashboard/api_hive_advice.go` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 7d6fc9459d..6d098cdf84 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:46` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1709` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1710` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:80` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:81` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:82` | diff --git a/src/docs/integration-orchestration.md b/src/docs/integration-orchestration.md new file mode 100644 index 0000000000..27cbedde0e --- /dev/null +++ b/src/docs/integration-orchestration.md @@ -0,0 +1,219 @@ +# Integration orchestration + +This layer coordinates authoring admission above existing coordinated preparation +and GitHub's native queue. It is not a merge engine and does not activate either +authority. Every existing CI, review, trust, DCO, coverage, soak, hold and +current-base guard remains authoritative. Nothing deploys automatically. + +## Configuration and rollback + +```yaml +integration_orchestration: + mode: off # off | advisory | enforce; default off + scopes: # exact allowlist; no wildcard or empty enrollment + - repo: example/project + base: release + lease_ttl: 5m # 1–15m, clipped to the underlying live claim + window: 1h # 1m–24h + max_active: 2 # 1–32 per repository/base + max_runner_minutes: 60 + max_jobs: 20 + max_attempts: 2 + max_observation_age: 30m # 1m–1h + admission_runner_minutes: 10 # explicit outstanding allocation per admission + admission_jobs: 1 # explicit fan-out allocation, not predicted capacity + policies: [] # optional pinned existing workflow reader contracts +``` + +OFF preserves dispatch and never observes workflow events or polls. ADVISORY +records intents and observations but cannot withhold work. ENFORCE additionally +withholds scoped new authoring admissions on unknown scope, validated unsatisfied/unknown DAG +edges, incomplete/stale CI accounting, exhausted fixed budgets or active slots. +Native admission affects only **new** Hive-owned enqueues: it does not cancel, +reorder or withdraw queued work. Direct merges do not wait on a capacity budget +after they have already completed their mandatory checks. + +The fixed budgets include both observed execution and outstanding admission +allocations. These are admission charges, not control over GitHub's physical +fan-out or queue build window. A confirmed, producer-bound failed combined +candidate reduces future Hive admission slots, bounded between one and the +configured maximum, for the accounting window. It does not change configuration, +cancel existing work, reorder native entries, or increase capacity. Exact-head +failed/canceled/timed-out attempts also stop new admissions at `max_attempts`, +leaving the existing owner-first repair/cooldown path responsible. + +Settings → Features exposes the existing configuration dialog controls and +owner-only evidence inspector. Scope ambiguity across multiple bases is unknown, +not inferred from an issue. The existing contributor queue priorities, own-work +preference and failure routing remain ahead of within-band intent aging. + +Every configuration transition releases only this layer's reservations. Claim and +native queue state is never deleted. Evidence and the append-only journal remain +for diagnosis. A corrupt journal refuses observations/enforcement rather than +resetting to empty. Runtime configuration writes report persistence failure. +Disabling remains possible if the journal is unavailable. Re-enabling enforcement +requires healthy durable state; cleanup failure never deletes the underlying claim. + +## Owner API + +- `GET /api/integration-orchestration`: configured mode, runtime commit, schema, + source, writer limitations, intents, exact candidate observations, jobs, + coverage, drift and live reservations. Actual configured preparation/native + flags are distinct from the explicitly unknown effective merge writer. + Window CI cost, observed/complete merged denominators and shared-job + attribution are separate; an unknown denominator is null, never zero. +- `PUT /api/config/integration-orchestration`: validated full configuration. + Duration JSON values are nanoseconds; YAML uses Go duration strings. +- `POST /api/integration-orchestration/intents`: versioned canonical work key, + repository/base, generation, plan/node/revision, anticipated paths, components, + contracts, generated outputs, scope confidence/provenance and prerequisites. + Hard prerequisite edges must match bound child nodes in the current approved + plan DAG. Textual/semantic hints never authorize a hard edge. + Prerequisites distinguish `developed`, `integrated` (the default) and `deployed`. + Closure only satisfies an explicitly developed edge; integration requires a + real merged-PR receipt linked by the approved plan. Deployment stays unknown + without an authoritative deployment producer. +- `POST /api/integration-orchestration/reconcile`: bounded observation repair; + shared with the governor's three-minute minimum reconciliation cadence. + +Dispatch creates unknown-scope intents for previously unseen canonical work and +attaches an existing approved DAG when a canonical issue/external-ref binding is +available. Ambiguous/missing bindings remain unknown, not invented dependencies. +An owner/planner can replace one with a higher generation. The original issue +claim/lease remains ownership authority; shared prerequisite work is not cloned. +Reservations renew only with a live authoritative lease, expire after a bounded +TTL, and are released on intent generation changes, failed lease persistence, +claim release/takeover or rollback. Same-file and cross-file semantic relations +are distinct advisory hints, never global file locks. Shadow evaluation compares +existing live-claim scopes without taking reservations. The inspector's +`dispatch_advice` exposes the last actual internal/contributor evaluations, +including semantic hints, false-positive assessments and would-withhold reasons. +These bounded, boot-local diagnostics are historical advice, not durable merge +proof or a new queue; OFF records none. The durable ledger retains the underlying +intent, claim-bound reservation and candidate evidence contracts separately. + +The existing head-keyed merge-train file probe feeds observed scope drift without +another GitHub file scanner. Its head freshness is explicitly unverified, so drift +does not become a hard dependency or passing proof. Cached hotspot analysis is +included in the inspector. An intent's optional `relations` records a canonical +related `key`, `kind` (`semantic` or `file`), `disposition` (`unreviewed`, +`confirmed`, `false-positive`) and `provenance`; assessments annotate actual +dispatch hints and never create locks. Approved-plan changes, intent generations +and holder/generation-fenced releases prevent stale owners deleting new leases. + +## Candidate and CI contracts + +The versioned candidate identity includes ordered PR/full-head membership, +repository/base, authority/ref, actual tested commit/tree, workflow, policy, +producer, attempt and generation. It is deliberately separate from convergence's +single-PR base observation. Removing a predecessor produces a different subject: +no proof is copied to the successor. Hold/review/member-head changes supersede +affected observations; a merged native prefix is not automatically discarded. + +Signed `merge_group` observations read the bounded authoritative queue prefix, +its tested tree and synthetic ref. A queue entry ID alone is not group identity. +Membership observations are **not** a passing policy attestation: until the +trusted group-aware producer supplies complete workflow/check-suite/job/policy +binding, policy evidence remains unknown. Native readiness/activation and +external DCO provider support are separately owned prerequisites; this layer +cannot manufacture them. + +A pinned reader contract contains exact `repo`, `base`, `engine` +(`native` or `direct`), workflow path, full workflow `blob` SHA, producer +`app_id`, and exact required `jobs`. The reader verifies event/source/path, +workflow blob on both the tested revision and trusted target base, workflow +run/attempt, source/target repositories, suite, physical job membership and check +App/head/suite/name. One receipt is **one workflow**, not all CI/review/trust gates. +Changing/removing a policy supersedes affected historical receipts durably. +`observed_state` retains the producer outcome separately from derived +supersession. Bounded reconciliation can complete unavailable proof at the same +workflow update timestamp; late waiting/unknown events cannot regress completed +proof, and contradictory equal-time terminal evidence fails conservatively. +Destroyed-group and member-change tombstones prevent out-of-order old receipts +reviving after restart. A documented native prefix promotion retains the original +subject only when the exact remaining chain and tested tree/ref match; it never +copies an A+B receipt onto an unrelated B-only subject. + +Direct-path checkout attestation is an explicit external producer dependency. +The pinned workflow must publish artifact `hive-integration-subject-v1-` +containing only `subject.json`: + +```json +{ + "version": 1, + "repo": "example/project", + "base": "release", + "base_sha": "", + "tested_sha": "", + "tested_tree": "", + "run_id": 123, + "attempt": 1, + "policy": "", + "members": [{"pr": 42, "head": ""}] +} +``` + +The consumer checks the artifact's run/head/attempt, pinned producer contract, +GitHub commit/tree and both merge parents (event base plus exact author head). +Downloads are size-bounded, in memory, without sending GitHub credentials to +artifact storage. No publisher, workflow, provider or queue activation is installed +by this feature. Missing or unsupported attestations remain unknown; +generic author workflow metadata is not actual combined-checkout proof. + +Actual GitHub workflow jobs are keyed by repository/job/run attempt, so shared +jobs count once. Reordered queued deliveries and partial completion metadata +cannot erase already observed physical runner execution. Accounting clips +assigned-runner wall time to the observation +window, includes canceled execution, and excludes queued/skipped work that never +executed. Unfinished work is accounted only through its last observation. +Incomplete cohorts, ambiguous base attribution and bounded pagination overflow +are unknown, not zero. Reconciliation has fixed limits; it does not create one +poller per run or cancel shared/safety-network jobs. + +The shared reconciliation batch has a three-minute minimum interval and +45-second deadline: at most eight scopes, eight relevant runs per scope, +24 attempt-job reads and four policy-run proof reads, with a 100-item page bound. +Scope service rotates between batches. It repairs missed job, merged-PR and +pinned-policy observations, reconstructing native membership from the live +authoritative queue when a group delivery was missed. Creation time is not used +to exclude older completed executions. A repository whose historical listing +exceeds the bounded page remains incomplete; this deliberately blocks that +cohort's enforcement activation rather than claiming scale-grade telemetry. +The merged-PR denominator uses a separately bounded updated-order closed-PR +listing. Reported minutes per merged PR are an observed-window aggregate, not a +claim that every individual change's eventual cost is known. Unfinished execution +and unattributed jobs remain explicit. + +The append-only journal uses local cross-process fencing, suffix replay and +fsync; no remote reads occur under its lock. Replay is bounded to 256 MiB, +100,000 records and 1 MiB per record. At a bound or corruption, new observations +and admission fail explicitly; audit is retained and disabling remains possible. +There is no automatic archive/compaction or unlimited-throughput claim. + +## Existing preparation visibility + +The inspector composes the existing cached +`/api/stage-tracker.base_preparation` DTO and cached hotspot source; it adds no +tracker polling, store reader, queue seats or preparation priority algorithm. +Current preparation ownership/backlog, inactive tickets and unverified tickets +are not ready-train membership or native queue entries. The standalone +preparation visibility and bounded review-owner handoff remain useful with this +layer OFF. Source landing is not publication or runtime rollout. + +## Design evidence and exclusions + +The reviewed primary abstracts are useful challenges, not Hive benchmarks: +[ICSME 2025 prioritization](https://arxiv.org/abs/2508.08342) studies historical +CI/metadata prioritization against FIFO; +[FSE 2023 parallel batch testing](https://2023.esec-fse.org/details/fse-2023-research-papers/14/Accelerating-Continuous-Integration-with-Parallel-Batch-Testing) +exposes nonlinear machine/feedback tradeoffs; and +[ICST 2026 PRediBatch](https://conf.researchr.org/details/icst-2026/icst-2026-industry/8/Predictive-Pull-Request-Batching-to-Accelerate-Merge-Pipelines-in-Continuous-Integrat) +reports a BMW-data simulation, not Hive production results. These support checking +bounded work-in-progress, anti-starvation and invalidation/failed-execution cost, +not skipping validation. This implementation supplies neither ML prioritization, +custom batching, speculative group construction, batch bisection nor another +merge authority. Native group observations only describe groups GitHub created. + +There is no automatic live pilot, adaptive concurrency growth, runner +procurement or claim of 5,000 merged PRs/day. That is an empirical capacity +target requiring observed quality and retained outcomes, not a code milestone. diff --git a/src/pkg/claims/claims.go b/src/pkg/claims/claims.go index 652b5e9525..66871d0a4f 100644 --- a/src/pkg/claims/claims.go +++ b/src/pkg/claims/claims.go @@ -370,13 +370,14 @@ type Ledger struct { // on it, so one run is escalated once however often it is observed. escalated map[string]time.Time // listed holds the kick listings (#10527) by issue key. - listed map[string]Listing - path string - policy Policy - hooks Hooks - now func() time.Time - hive string - admit func(Request) (string, error) + listed map[string]Listing + path string + policy Policy + hooks Hooks + now func() time.Time + hive string + admit func(Request) (string, error) + commitCheck func(Claim) (string, error) } // ErrInvalid is returned for requests missing a repo, issue, holder or kind. @@ -443,6 +444,15 @@ func (l *Ledger) SetAdmissionCheck(check func(Request) (string, error)) { l.mu.Unlock() } +// SetCommitCheck installs a local-only, atomic admission reservation. It runs +// under the claim mutex for a new automated holder, after all remote admission +// reads. It must not call this ledger or perform remote I/O. +func (l *Ledger) SetCommitCheck(check func(Claim) (string, error)) { + l.mu.Lock() + l.commitCheck = check + l.mu.Unlock() +} + // SetNow overrides the clock. Intended for tests. func (l *Ledger) SetNow(fn func() time.Time) { l.mu.Lock() @@ -483,6 +493,22 @@ func (l *Ledger) Claim(req Request) (Result, error) { ClaimedAt: now, ExpiresAt: now.Add(ttl), } prev, held := l.claims[key] + newHolder := !held || (!sameHolder(prev, req) && + (req.Kind.Outranks(prev.Kind) || req.Kind.rank() == prev.Kind.rank() && req.Force)) + if newHolder && req.Kind != KindHuman && l.commitCheck != nil { + reason, err := l.commitCheck(next) + if err != nil || reason != "" { + if len(expired) > 0 { + if saveErr := l.saveLocked(); saveErr != nil && err == nil { + err = saveErr + } + } + hooks := l.hooks + l.mu.Unlock() + fireExpired(hooks, expired) + return Result{Outcome: OutcomeRefused, Reason: reason}, err + } + } var res Result switch { case !held: diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index ac022d77a8..395e65afd5 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -184,7 +184,8 @@ type Config struct { UpstreamWatch UpstreamWatchConfig `yaml:"upstream_watch,omitempty" json:"upstream_watch,omitempty"` // Convergence toggles the convergence-driven admission surfaces // (kubestellar/hive#3845 follow-ons). Default off → zero behaviour change. - Convergence ConvergenceConfig `yaml:"convergence,omitempty" json:"convergence,omitempty"` + Convergence ConvergenceConfig `yaml:"convergence,omitempty" json:"convergence,omitempty"` + IntegrationOrchestration IntegrationOrchestrationConfig `yaml:"integration_orchestration,omitempty" json:"integration_orchestration"` // SLA holds the Delivery SLA span targets the dashboard Overview colours // against (see sla.go). Unset fields use the defaults. SLA SLAConfig `yaml:"sla,omitempty" json:"sla,omitempty"` diff --git a/src/pkg/config/integration_orchestration.go b/src/pkg/config/integration_orchestration.go new file mode 100644 index 0000000000..1d2f9dfecc --- /dev/null +++ b/src/pkg/config/integration_orchestration.go @@ -0,0 +1,175 @@ +package config + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "strings" + "time" +) + +const ( + IntegrationOff = "off" + IntegrationAdvisory = "advisory" + IntegrationEnforce = "enforce" + DefaultIntegrationLease = 5 * time.Minute + DefaultIntegrationWindow = time.Hour + DefaultIntegrationMaxActive = 2 + DefaultIntegrationRunnerMinutes = 60 + DefaultIntegrationMaxJobs = 20 + DefaultIntegrationMaxAttempts = 2 + DefaultIntegrationMaxAge = 30 * time.Minute + DefaultIntegrationAdmissionMinutes = 10 + DefaultIntegrationAdmissionJobs = 1 + MaxIntegrationScopes = 8 +) + +// IntegrationOrchestrationConfig controls admission, never merge authority. +// Scopes must name exact repository/base pairs; an empty list enrolls nothing. +type IntegrationOrchestrationConfig struct { + Mode string `yaml:"mode,omitempty" json:"mode"` + Scopes []IntegrationScope `yaml:"scopes,omitempty" json:"scopes"` + LeaseTTL time.Duration `yaml:"lease_ttl,omitempty" json:"lease_ttl"` + Window time.Duration `yaml:"window,omitempty" json:"window"` + MaxActive int `yaml:"max_active,omitempty" json:"max_active"` + MaxRunnerMinutes float64 `yaml:"max_runner_minutes,omitempty" json:"max_runner_minutes"` + MaxJobs int `yaml:"max_jobs,omitempty" json:"max_jobs"` + MaxAttempts int `yaml:"max_attempts,omitempty" json:"max_attempts"` + MaxObservationAge time.Duration `yaml:"max_observation_age,omitempty" json:"max_observation_age"` + AdmissionRunnerMinutes float64 `yaml:"admission_runner_minutes,omitempty" json:"admission_runner_minutes"` + AdmissionJobs int `yaml:"admission_jobs,omitempty" json:"admission_jobs"` + Policies []IntegrationPolicy `yaml:"policies,omitempty" json:"policies"` +} + +// IntegrationPolicy pins an already-reviewed workflow blob and its exact jobs. +// It is only an evidence-reader contract, never a replacement required context. +type IntegrationPolicy struct { + Repo string `yaml:"repo" json:"repo"` + Base string `yaml:"base" json:"base"` + Engine string `yaml:"engine" json:"engine"` + Workflow string `yaml:"workflow" json:"workflow"` + Blob string `yaml:"blob" json:"blob"` + AppID int64 `yaml:"app_id" json:"app_id"` + Jobs []string `yaml:"jobs" json:"jobs"` +} + +func (p IntegrationPolicy) Fingerprint() string { + p.Repo = strings.ToLower(p.Repo) + data, _ := json.Marshal(p) + sum := sha256.Sum256(data) + return "workflow-policy-v1/" + hex.EncodeToString(sum[:]) +} + +type IntegrationScope struct { + Repo string `yaml:"repo" json:"repo"` + Base string `yaml:"base" json:"base"` +} + +func (c IntegrationOrchestrationConfig) Effective() IntegrationOrchestrationConfig { + if c.Mode == "" { + c.Mode = IntegrationOff + } + if c.AdmissionRunnerMinutes == 0 { + c.AdmissionRunnerMinutes = DefaultIntegrationAdmissionMinutes + } + if c.AdmissionJobs == 0 { + c.AdmissionJobs = DefaultIntegrationAdmissionJobs + } + if c.LeaseTTL == 0 { + c.LeaseTTL = DefaultIntegrationLease + } + if c.Window == 0 { + c.Window = DefaultIntegrationWindow + } + if c.MaxActive == 0 { + c.MaxActive = DefaultIntegrationMaxActive + } + if c.MaxRunnerMinutes == 0 { + c.MaxRunnerMinutes = DefaultIntegrationRunnerMinutes + } + if c.MaxJobs == 0 { + c.MaxJobs = DefaultIntegrationMaxJobs + } + if c.MaxAttempts == 0 { + c.MaxAttempts = DefaultIntegrationMaxAttempts + } + if c.MaxObservationAge == 0 { + c.MaxObservationAge = DefaultIntegrationMaxAge + } + return c +} + +func (c IntegrationOrchestrationConfig) Validate() error { + c = c.Effective() + if len(c.Scopes) > MaxIntegrationScopes || len(c.Policies) > MaxIntegrationScopes || c.AdmissionJobs < 1 || c.AdmissionJobs > c.MaxJobs || + math.IsNaN(c.AdmissionRunnerMinutes) || math.IsInf(c.AdmissionRunnerMinutes, 0) || + c.AdmissionRunnerMinutes <= 0 || c.AdmissionRunnerMinutes > c.MaxRunnerMinutes { + return fmt.Errorf("integration admission allocations must fit scope budgets; at most %d scopes", MaxIntegrationScopes) + } + if c.Mode != IntegrationOff && c.Mode != IntegrationAdvisory && c.Mode != IntegrationEnforce { + return fmt.Errorf("integration_orchestration.mode must be off, advisory or enforce") + } + if c.LeaseTTL < time.Minute || c.LeaseTTL > 15*time.Minute || + c.Window < time.Minute || c.Window > 24*time.Hour || + c.MaxObservationAge < time.Minute || c.MaxObservationAge > time.Hour || + c.MaxActive < 1 || c.MaxActive > 32 || c.MaxJobs < 1 || c.MaxJobs > 1000 || + c.MaxAttempts < 1 || c.MaxAttempts > 10 || math.IsNaN(c.MaxRunnerMinutes) || math.IsInf(c.MaxRunnerMinutes, 0) || c.MaxRunnerMinutes <= 0 || c.MaxRunnerMinutes > 10000 { + return fmt.Errorf("integration_orchestration bounds invalid: lease 1-15m, window 1m-24h, observation age 1m-1h, active 1-32, jobs 1-1000, attempts 1-10, minutes (0,10000]") + } + seen := map[string]bool{} + for _, s := range c.Scopes { + parts := strings.Split(s.Repo, "/") + if len(parts) != 2 || parts[0] == "" || parts[1] == "" || strings.ContainsAny(s.Repo+s.Base, "*? \t\n") || s.Base == "" { + return fmt.Errorf("integration_orchestration scopes require exact owner/repo and base") + } + k := strings.ToLower(s.Repo) + "#" + s.Base + if seen[k] { + return fmt.Errorf("duplicate integration_orchestration scope %s", k) + } + seen[k] = true + } + policies := map[string]bool{} + for _, p := range c.Policies { + enrolled := false + for _, scope := range c.Scopes { + if strings.EqualFold(scope.Repo, p.Repo) && scope.Base == p.Base { + enrolled = true + } + } + if !enrolled || (p.Engine != "direct" && p.Engine != "native") || !strings.HasPrefix(p.Workflow, ".github/workflows/") || + strings.Contains(p.Workflow, "..") || len(p.Blob) != 40 || strings.Trim(p.Blob, "0123456789abcdef") != "" || + p.AppID <= 0 || len(p.Jobs) == 0 || len(p.Jobs) > c.MaxJobs { + return fmt.Errorf("integration policy requires enrolled scope, engine, workflow blob, producer app and bounded exact jobs") + } + key := strings.ToLower(p.Repo) + "#" + p.Base + "/" + p.Engine + "/" + p.Workflow + if policies[key] { + return fmt.Errorf("duplicate integration policy") + } + policies[key] = true + names := map[string]bool{} + for _, name := range p.Jobs { + if strings.TrimSpace(name) == "" || names[name] { + return fmt.Errorf("integration policy jobs must be unique nonempty names") + } + names[name] = true + } + } + if c.Mode != IntegrationOff && len(c.Scopes) == 0 { + return fmt.Errorf("integration_orchestration requires explicit scopes when enabled") + } + return nil +} + +func (c IntegrationOrchestrationConfig) Allows(repo, base string) bool { + if c.Validate() != nil || c.Effective().Mode == IntegrationOff { + return false + } + for _, s := range c.Scopes { + if strings.EqualFold(s.Repo, repo) && s.Base == base { + return true + } + } + return false +} diff --git a/src/pkg/config/integration_orchestration_test.go b/src/pkg/config/integration_orchestration_test.go new file mode 100644 index 0000000000..8861f600ad --- /dev/null +++ b/src/pkg/config/integration_orchestration_test.go @@ -0,0 +1,75 @@ +package config + +import ( + "math" + "strings" + "testing" + "time" +) + +func TestIntegrationConfigDefaultOffAndBounds(t *testing.T) { + if (IntegrationOrchestrationConfig{}).Effective().Mode != IntegrationOff { + t.Fatal("default enabled") + } + + if err := (IntegrationOrchestrationConfig{}).Validate(); err != nil { + t.Fatal(err) + } + good := IntegrationOrchestrationConfig{Mode: IntegrationEnforce, Scopes: []IntegrationScope{{Repo: "o/r", Base: "v5"}}} + if !good.Allows("O/R", "v5") || good.Allows("o/r", "V5") || good.Allows("other/r", "v5") { + t.Fatal("scope matching") + } + for _, change := range []func(*IntegrationOrchestrationConfig){ + func(c *IntegrationOrchestrationConfig) { c.Mode = "shadow" }, func(c *IntegrationOrchestrationConfig) { c.Scopes = nil }, + func(c *IntegrationOrchestrationConfig) { c.MaxActive = -1 }, func(c *IntegrationOrchestrationConfig) { c.MaxActive = 33 }, + func(c *IntegrationOrchestrationConfig) { c.MaxJobs = 1001 }, func(c *IntegrationOrchestrationConfig) { c.MaxRunnerMinutes = math.NaN() }, + func(c *IntegrationOrchestrationConfig) { c.MaxRunnerMinutes = math.Inf(1) }, func(c *IntegrationOrchestrationConfig) { c.LeaseTTL = time.Hour }, + func(c *IntegrationOrchestrationConfig) { c.Window = 25 * time.Hour }, func(c *IntegrationOrchestrationConfig) { c.MaxObservationAge = time.Second }, + func(c *IntegrationOrchestrationConfig) { c.MaxAttempts = 11 }, + func(c *IntegrationOrchestrationConfig) { c.Scopes = []IntegrationScope{{Repo: "o/*", Base: "v5"}} }, + func(c *IntegrationOrchestrationConfig) { c.Scopes = []IntegrationScope{{Repo: "o/r", Base: ""}} }, + func(c *IntegrationOrchestrationConfig) { c.Scopes = append(c.Scopes, c.Scopes[0]) }, + } { + bad := good + bad.Scopes = append([]IntegrationScope(nil), good.Scopes...) + change(&bad) + if bad.Validate() == nil || bad.Allows("o/r", "v5") { + t.Fatalf("invalid config admitted: %+v", bad) + } + } +} + +func TestIntegrationPinnedPolicyValidationAndIdentity(t *testing.T) { + policy := IntegrationPolicy{Repo: "o/r", Base: "v5", Engine: "native", Workflow: ".github/workflows/group.yml", Blob: strings.Repeat("a", 40), AppID: 42, Jobs: []string{"policy"}} + good := IntegrationOrchestrationConfig{Mode: IntegrationAdvisory, Scopes: []IntegrationScope{{Repo: "o/r", Base: "v5"}}, Policies: []IntegrationPolicy{policy}} + if err := good.Validate(); err != nil { + t.Fatal(err) + } + for _, change := range []func(*IntegrationPolicy){ + func(p *IntegrationPolicy) { p.AppID = 0 }, func(p *IntegrationPolicy) { p.Repo = "other/r" }, + func(p *IntegrationPolicy) { p.Engine = "invented" }, func(p *IntegrationPolicy) { p.Workflow = "../../escape.yml" }, + func(p *IntegrationPolicy) { p.Blob = "not-a-full-blob" }, func(p *IntegrationPolicy) { p.Jobs = nil }, + func(p *IntegrationPolicy) { p.Jobs = []string{"policy", "policy"} }, func(p *IntegrationPolicy) { p.Jobs = []string{""} }, + } { + bad := good + bad.Policies = []IntegrationPolicy{policy} + change(&bad.Policies[0]) + if bad.Validate() == nil { + t.Fatalf("unbound policy accepted: %+v", bad) + } + } + changed := policy + changed.Jobs = []string{"policy", "required-extra"} + if changed.Fingerprint() == policy.Fingerprint() { + t.Fatal("policy requirements did not change evidence identity") + } + changed = policy + changed.Repo = "O/R" + if changed.Fingerprint() != policy.Fingerprint() { + t.Fatal("repository case changed policy identity") + } + good.Policies = append(good.Policies, policy) + if good.Validate() == nil { + t.Fatal("duplicate policies accepted") + } +} diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index 4a77a7ad42..f1c91f48d4 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -743,6 +743,20 @@ "scanner": {}, "upstream_watch": {}, "convergence": {}, + "integration_orchestration": { + "mode": "", + "scopes": null, + "lease_ttl": 0, + "window": 0, + "max_active": 0, + "max_runner_minutes": 0, + "max_jobs": 0, + "max_attempts": 0, + "max_observation_age": 0, + "admission_runner_minutes": 0, + "admission_jobs": 0, + "policies": null + }, "sla": { "targets": {} }, diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index a95ad83dfc..0fb16a6c77 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -82,6 +82,9 @@ func hostAllowedByHTTPList(host string, allowlist []string) bool { } func (c *Config) ValidateWithOptions(opts ValidateOptions) error { + if err := c.IntegrationOrchestration.Validate(); err != nil { + return err + } if c.Project.Org == "" { return fmt.Errorf("project.org is required") } diff --git a/src/pkg/convergence/integration/admission.go b/src/pkg/convergence/integration/admission.go new file mode 100644 index 0000000000..f16897c639 --- /dev/null +++ b/src/pkg/convergence/integration/admission.go @@ -0,0 +1,309 @@ +package integration + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +type Cost struct { + RunnerMinutes *float64 `json:"runner_minutes"` + ObservedRunnerMinutes float64 `json:"observed_runner_minutes"` + Jobs int `json:"jobs"` + Running int `json:"running"` + Complete bool `json:"complete"` + Reason string `json:"reason"` +} + +func cost(jobs map[string]Job, coverage Coverage, repo, base string, from, through time.Time) Cost { + out := Cost{Complete: coverage.Complete && !coverage.From.After(from) && !coverage.Through.Before(through), Reason: coverage.Reason} + minutes := 0.0 + for _, j := range jobs { + if !strings.EqualFold(j.Repo, repo) || j.Base != base || j.RunnerID == 0 || j.StartedAt.IsZero() { + continue + } + if j.Status == "queued" || j.Status == "waiting" || j.Status == "pending" || j.Status == "requested" || j.Conclusion == "skipped" { + continue + } + if j.Status == "" || j.Status == "completed" && j.CompletedAt.IsZero() { + out.Complete = false + continue + } + end := j.CompletedAt + if end.IsZero() { + end = j.ObservedAt + if end.After(from) { + out.Running++ + } + // A live job's unobserved future is unknown, not zero execution. + if end.Before(through) { + out.Complete = false + } + } + start := j.StartedAt + if start.Before(from) { + start = from + } + if end.After(through) { + end = through + } + if end.After(start) { + minutes += end.Sub(start).Minutes() + out.Jobs++ + } + } + out.ObservedRunnerMinutes = minutes + if out.Complete { + out.RunnerMinutes = &minutes + } else { + out.Reason = "workflow job coverage incomplete or stale; admission growth frozen" + } + return out +} + +func (s *Store) Cost(repo, base string, from, through time.Time) Cost { + s.mu.Lock() + defer s.mu.Unlock() + return cost(s.jobs, s.coverage[scopeKey(repo, base)], repo, base, from, through) +} + +type Dependency struct { + Known bool + Satisfied bool +} + +// Input consists of authoritative observations obtained outside Store's lock. +// The approved DAG and live lease, not an intent author's assertion, authorize +// the corresponding constraints. +type Input struct { + Key string + Repo string + Base string + PlanApproved bool + PlanRequired bool + PlanRevision int + IntentGeneration uint64 + Dependencies map[string]Dependency + LiveKeys []string + Claim string + ClaimGeneration uint64 + ClaimExpiresAt time.Time + Reserve bool + RenewOnly bool + Head string + Now time.Time +} + +type Decision struct { + Allowed bool `json:"allowed"` + WouldBlock bool `json:"would_block"` + Reason string `json:"reason"` + Hints []string `json:"hints"` + Cost Cost `json:"cost"` + ActiveLimit int `json:"active_limit"` + FailureBackoff int `json:"failure_backoff"` +} + +func intersects(a, b []string) bool { + for _, x := range a { + for _, y := range b { + if x == y { + return true + } + } + } + return false +} + +func (s *Store) Decide(raw config.IntegrationOrchestrationConfig, in Input) (Decision, error) { + cfg := raw.Effective() + out := Decision{Allowed: true, Hints: []string{}} + if !cfg.Allows(in.Repo, in.Base) { + return out, nil + } + s.mu.Lock() + defer s.mu.Unlock() + unlock, err := s.writerLock() + if err != nil { + return Decision{Allowed: cfg.Mode != config.IntegrationEnforce, WouldBlock: true, Reason: "durable admission fencing unavailable"}, err + } + defer unlock() + intent, found := s.intents[in.Key] + block := func(reason string) { + if !out.WouldBlock { + out.WouldBlock, out.Reason = true, reason + } + } + if !found { + out.Hints = append(out.Hints, "scope unknown: no versioned intent; independence not established") + block("qualified change intent unavailable") + } else { + if !strings.EqualFold(intent.Repo, in.Repo) || intent.Base != in.Base { + block("intent repository/base differs from admission scope") + } + if in.IntentGeneration != 0 && in.IntentGeneration != intent.Generation { + block("intent generation changed while gathering prerequisites") + } + if (in.PlanRequired || intent.Plan != "unresolved") && (!in.PlanApproved || in.PlanRevision != intent.PlanRevision) { + block("approved plan generation unavailable or changed") + } + if !intent.ScopeKnown { + out.Hints = append(out.Hints, "anticipated scope unknown; not evidence of independence") + block("anticipated scope unknown; qualified cohort unavailable") + } + if in.Head != "" && intent.Head != "" && in.Head != intent.Head { + block("exact candidate head differs from current intent") + } + if len(intent.Prerequisites) > 0 { + if !in.PlanApproved || in.PlanRevision != intent.PlanRevision { + block("approved plan generation unavailable or changed") + } + for _, p := range intent.Prerequisites { + d := in.Dependencies[p.Key] + if !d.Known { + block("validated prerequisite unknown: " + p.Key) + } else if !d.Satisfied { + block("waiting for prerequisite: " + p.Key) + } + } + } + liveKeys := map[string]bool{} + for key, reservation := range s.reservations { + if reservation.ExpiresAt.After(in.Now) { + liveKeys[key] = true + } + } + for _, key := range in.LiveKeys { + liveKeys[key] = true + } + for key := range liveKeys { + if key == in.Key { + continue + } + other, ok := s.intents[key] + if !ok || !strings.EqualFold(other.Repo, intent.Repo) || other.Base != intent.Base { + continue + } + if intersects(intent.Contracts, other.Contracts) || intersects(intent.Components, other.Components) || intersects(intent.Generated, other.Generated) { + out.Hints = append(out.Hints, "semantic interaction with "+key+" (advisory; not an exclusive lock"+relationDisposition(intent, key, "semantic")+")") + } else if intersects(intent.Paths, other.Paths) { + out.Hints = append(out.Hints, "same-file overlap with "+key+" (advisory; not an exclusive lock"+relationDisposition(intent, key, "file")+")") + } + } + sort.Strings(out.Hints) + } + coverage := s.coverage[scopeKey(in.Repo, in.Base)] + through := coverage.Through + out.Cost = cost(s.jobs, coverage, in.Repo, in.Base, in.Now.Add(-cfg.Window), through) + if through.IsZero() || through.After(in.Now) || in.Now.Sub(through) > cfg.MaxObservationAge { + out.Cost.Complete, out.Cost.RunnerMinutes, out.Cost.Reason = false, nil, "workflow coverage stale or unknown; admission growth frozen" + } + if !out.Cost.Complete { + block(out.Cost.Reason) + } else if *out.Cost.RunnerMinutes >= cfg.MaxRunnerMinutes || out.Cost.Jobs >= cfg.MaxJobs { + block("fixed runner-minute/job admission budget exhausted") + } + active := 0 + reservedMinutes, reservedJobs := 0.0, 0 + for key, r := range s.reservations { + i := s.intents[key] + if key != in.Key && strings.EqualFold(i.Repo, in.Repo) && i.Base == in.Base && r.ExpiresAt.After(in.Now) { + active++ + reservedMinutes += r.RunnerMinutes + reservedJobs += r.Jobs + } + } + if out.Cost.RunnerMinutes != nil && (*out.Cost.RunnerMinutes+reservedMinutes+cfg.AdmissionRunnerMinutes > cfg.MaxRunnerMinutes || + out.Cost.Jobs+reservedJobs+cfg.AdmissionJobs > cfg.MaxJobs) { + block("scoped runner-minute/job allocations exhausted") + } + head := in.Head + if head == "" { + head = intent.Head + } + for _, job := range s.jobs { + if head != "" && strings.EqualFold(job.Repo, in.Repo) && job.Base == in.Base && job.Head == head && job.Attempt >= cfg.MaxAttempts && job.Status == "completed" && + (job.Conclusion == "failure" || job.Conclusion == "cancelled" || job.Conclusion == "timed_out") { + block("bounded CI attempts exhausted for exact head; owner repair required") + } + } + for _, receipt := range s.evidence { + if !strings.EqualFold(receipt.Subject.Repo, in.Repo) || receipt.Subject.Base != in.Base || receipt.State != Rejected || receipt.Subject.Attempt < cfg.MaxAttempts { + continue + } + for _, member := range receipt.Subject.Members { + if head != "" && member.Head == head { + block("bounded integration candidate attempts exhausted; owner-first repair required") + } + } + } + out.ActiveLimit = cfg.MaxActive + for _, receipt := range s.evidence { + if strings.EqualFold(receipt.Subject.Repo, in.Repo) && receipt.Subject.Base == in.Base && receipt.ObservedState == Rejected && + receipt.Source == "github-actions-bound-v1" && len(receipt.Subject.Members) > 1 && receipt.Subject.Validate() == nil && + !receipt.ObservedAt.Before(in.Now.Add(-cfg.Window)) && !receipt.ObservedAt.After(in.Now) { + out.FailureBackoff++ + if out.ActiveLimit > 1 { + out.ActiveLimit-- + } + } + } + if out.FailureBackoff > 0 { + out.Hints = append(out.Hints, "confirmed combined-candidate failure reduces future Hive admission only; no GitHub build-window mutation or shared-job cancellation") + } + if active >= out.ActiveLimit { + block("scoped active admission limit") + } + if existing, ok := s.reservations[in.Key]; ok && existing.ExpiresAt.After(in.Now) && + (existing.Claim != in.Claim || existing.ClaimGeneration != in.ClaimGeneration) && in.Reserve { + block("work already reserved by another live claim") + } + // A live holder's heartbeat is not new admission. Preserve accounting of + // already-running work even while missing telemetry freezes growth. + if existing, ok := s.reservations[in.Key]; ok && in.Reserve && in.RenewOnly && existing.Claim == in.Claim && + existing.ClaimGeneration == in.ClaimGeneration && existing.IntentGeneration == intent.Generation && + strings.EqualFold(intent.Repo, in.Repo) && intent.Base == in.Base && (in.IntentGeneration == 0 || in.IntentGeneration == intent.Generation) && + (in.Head == "" || intent.Head == "" || in.Head == intent.Head) && + existing.ExpiresAt.After(in.Now) && in.ClaimExpiresAt.After(in.Now) { + out.WouldBlock, out.Reason = false, "existing exact lease renewed; no new work admitted" + } + if cfg.Mode == config.IntegrationEnforce && out.WouldBlock { + out.Allowed = false + } + if !out.Allowed || !in.Reserve || !found || cfg.Mode != config.IntegrationEnforce { + return out, nil + } + if in.Claim == "" || in.ClaimGeneration == 0 || !in.ClaimExpiresAt.After(in.Now) { + return out, fmt.Errorf("reservation requires a live authoritative claim generation") + } + expires := in.Now.Add(cfg.LeaseTTL) + if in.ClaimExpiresAt.Before(expires) { + expires = in.ClaimExpiresAt + } + r := Reservation{Key: in.Key, Claim: in.Claim, ClaimGeneration: in.ClaimGeneration, IntentGeneration: intent.Generation, ExpiresAt: expires, + RunnerMinutes: cfg.AdmissionRunnerMinutes, Jobs: cfg.AdmissionJobs} + err = s.appendLocked(Event{Version: SchemaVersion, ID: fmt.Sprintf("reserve/%s/%s/%d/%d", in.Key, in.Claim, in.ClaimGeneration, in.Now.UnixNano()), At: in.Now, Reservation: &r}) + return out, err +} + +// Order is a stable authoring-admission cohort. It never reorders native queue +// entries. Oldest intents receive service first, independently per repo/base. +func (s *Store) Order(keys []string) []string { + s.mu.Lock() + defer s.mu.Unlock() + out := append([]string(nil), keys...) + sort.SliceStable(out, func(a, b int) bool { + i, j := s.intents[out[a]], s.intents[out[b]] + if i.ObservedAt.IsZero() { + return false + } + if j.ObservedAt.IsZero() { + return true + } + return i.ObservedAt.Before(j.ObservedAt) + }) + return out +} diff --git a/src/pkg/convergence/integration/ci_cost_accounting_test.go b/src/pkg/convergence/integration/ci_cost_accounting_test.go new file mode 100644 index 0000000000..f55332d952 --- /dev/null +++ b/src/pkg/convergence/integration/ci_cost_accounting_test.go @@ -0,0 +1,41 @@ +package integration + +import ( + "strings" + "testing" + "time" +) + +func TestAssignedMetadataDoesNotTurnQueuedSkippedOrIncompleteJobsIntoKnownExecution(t *testing.T) { + from, through := epoch.Add(time.Hour), epoch.Add(2*time.Hour) + coverage := Coverage{From: from, Through: through, Complete: true} + for _, tc := range []struct { + name, status, conclusion string + end time.Time + complete bool + minutes float64 + }{ + {"queued", "queued", "", through, true, 0}, + {"waiting", "waiting", "", through, true, 0}, + {"skipped", "completed", "skipped", through, true, 0}, + {"missing-status", "", "success", through, false, 0}, + {"missing-end", "completed", "success", time.Time{}, false, 0}, + {"clipped", "completed", "cancelled", through.Add(time.Hour), true, 60}, + } { + t.Run(tc.name, func(t *testing.T) { + job := Job{Repo: "o/r", Base: "v5", ID: 1, RunID: 2, Attempt: 1, Head: strings.Repeat("a", 40), RunnerID: 3, + Status: tc.status, Conclusion: tc.conclusion, StartedAt: epoch, CompletedAt: tc.end, ObservedAt: through} + if job.CompletedAt.After(job.ObservedAt) { + job.ObservedAt = job.CompletedAt + } + + report := cost(map[string]Job{job.Key(): job}, coverage, "o/r", "v5", from, through) + if report.Complete != tc.complete || report.ObservedRunnerMinutes != tc.minutes { + t.Fatalf("incorrect actual execution: %+v", report) + } + if tc.complete && (report.RunnerMinutes == nil || *report.RunnerMinutes != tc.minutes) || !tc.complete && report.RunnerMinutes != nil { + t.Fatalf("unknown presented as known: %+v", report) + } + }) + } +} diff --git a/src/pkg/convergence/integration/clone.go b/src/pkg/convergence/integration/clone.go new file mode 100644 index 0000000000..90b2bc9259 --- /dev/null +++ b/src/pkg/convergence/integration/clone.go @@ -0,0 +1,24 @@ +package integration + +import "slices" + +func cloneIntent(i Intent) Intent { + i.Components = slices.Clone(i.Components) + i.Contracts = slices.Clone(i.Contracts) + i.Paths = slices.Clone(i.Paths) + i.Generated = slices.Clone(i.Generated) + i.Prerequisites = slices.Clone(i.Prerequisites) + i.Relations = slices.Clone(i.Relations) + return i +} + +func cloneEvidence(e Evidence) Evidence { + e.Subject.Members = slices.Clone(e.Subject.Members) + e.CheckIDs = slices.Clone(e.CheckIDs) + return e +} + +func cloneJob(j Job) Job { + j.Candidates = slices.Clone(j.Candidates) + return j +} diff --git a/src/pkg/convergence/integration/coordination_metrics_test.go b/src/pkg/convergence/integration/coordination_metrics_test.go new file mode 100644 index 0000000000..d5eb233fdc --- /dev/null +++ b/src/pkg/convergence/integration/coordination_metrics_test.go @@ -0,0 +1,136 @@ +package integration + +import ( + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +func TestObservedScopeDriftAndFalsePositiveRemainAdvisoryAndDurable(t *testing.T) { + store := testStore(t) + intent := testIntent(1) + intent.Head = strings.Repeat("a", 40) + intent.Paths = []string{"src/api"} + intent.Generated = []string{"dashboard.js"} + intent.Relations = []RelationAssessment{{Key: "o/r#2", Kind: "semantic", Disposition: "false-positive", Provenance: "owner-reviewed-independent-contracts"}} + putIntent(t, store, intent) + if err := store.ObserveFiles("elsewhere/r", intent.Head, []string{"unrelated"}, epoch); err != nil { + t.Fatal(err) + } + if len(store.Snapshot(epoch).Drifts) != 0 { + t.Fatal("cross-repository file observation attached") + } + files := []string{"src/api/handler.go", "dashboard.js", "outside.go"} + if err := store.ObserveFiles("o/r", intent.Head, files, epoch); err != nil { + t.Fatal(err) + } + if err := store.ObserveFiles("o/r", intent.Head, files, epoch.Add(time.Hour)); err != nil { + t.Fatal("probe replay conflict", err) + } + snapshot := store.Snapshot(epoch) + if len(snapshot.Drifts) != 1 || snapshot.Drifts[0].HeadVerified || len(snapshot.Drifts[0].OutsideAnticipated) != 1 || + snapshot.Drifts[0].OutsideAnticipated[0] != "outside.go" { + t.Fatalf("%+v", snapshot.Drifts) + } + snapshot.Drifts[0].Actual[0] = "mutated" + if store.Snapshot(epoch).Drifts[0].Actual[0] == "mutated" { + t.Fatal("drift snapshot aliases ledger") + } + if !strings.Contains(relationDisposition(intent, "o/r#2", "semantic"), "false-positive") || relationDisposition(intent, "o/r#2", "file") != "" { + t.Fatal("relation assessment lost kind or disposition") + } + restarted, err := Open(store.path) + if err != nil || len(restarted.Snapshot(epoch).Drifts) != 1 { + t.Fatalf("restart=%v", err) + } + bad := intent + bad.Relations = []RelationAssessment{{Key: "o/r#2", Kind: "semantic", Disposition: "permanent-lock", Provenance: "text"}} + if bad.Validate() == nil { + t.Fatal("semantic feedback became a hard control") + } +} + +func TestScopedWindowCostPerMergedPRIncludesSharedAndCanceledExecution(t *testing.T) { + store := testStore(t) + now := epoch.Add(time.Hour) + job := Job{Repo: "o/r", Base: "v5", ID: 1, RunID: 20, Attempt: 1, Head: testSubject().TestedSHA, RunnerID: 30, + StartedAt: now.Add(-30 * time.Minute), CompletedAt: now.Add(-10 * time.Minute), ObservedAt: now, Status: "completed", Conclusion: "cancelled"} + if err := store.Append(Event{Version: 1, ID: "physical-job", At: now, Job: &job}); err != nil { + t.Fatal(err) + } + subject := testSubject() + evidence := Evidence{Subject: subject, State: Rejected, ObservedAt: now, Source: "github-actions-bound-v1", RunID: 20, SuiteID: 2, CheckIDs: []int64{3}} + if err := store.Append(Event{Version: 1, ID: "candidate", At: now, Evidence: &evidence}); err != nil { + t.Fatal(err) + } + evidence.Subject.Policy = "second-policy" + if err := store.Append(Event{Version: 1, ID: "shared-candidate", At: now, Evidence: &evidence}); err != nil { + t.Fatal(err) + } + unknown := job + unknown.ID, unknown.RunID, unknown.StartedAt, unknown.CompletedAt = 2, 99, now.Add(-10*time.Minute), now + if err := store.Append(Event{Version: 1, ID: "unclassified-execution", At: now, Job: &unknown}); err != nil { + t.Fatal(err) + } + merged := IntegrationReceipt{Repo: "o/r", Base: "v5", PR: 1, Head: subject.Members[0].Head, MergeSHA: subject.TestedSHA, MergedAt: now.Add(-5 * time.Minute), Source: "github-merged-pr-rest-v1"} + if err := store.Append(Event{Version: 1, ID: "merged", At: now, Integrated: &merged}); err != nil { + t.Fatal(err) + } + coverage := coverageEvent(now) + coverage.Coverage.MergedComplete = true + if err := store.Append(coverage); err != nil { + t.Fatal(err) + } + report := store.CostReport("O/R", "v5", now.Add(-time.Hour), now) + if report.MergedPRs == nil || *report.MergedPRs != 1 || report.RunnerMinutesPerMergedPR == nil || *report.RunnerMinutesPerMergedPR != 30 || + report.AttributedIntegrationMinutes == nil || *report.AttributedIntegrationMinutes != 20 || report.UnattributedExecutionJobs != 1 { + t.Fatalf("%+v", report) + } + coverage.ID = "incomplete-denominator" + coverage.Coverage.MergedComplete = false + if err := store.Append(coverage); err != nil { + t.Fatal(err) + } + report = store.CostReport("o/r", "v5", now.Add(-time.Hour), now) + if report.MergedPRs != nil || report.RunnerMinutesPerMergedPR != nil || report.ObservedMergedPRs != 1 { + t.Fatal("unknown denominator became zero or a claimed complete ratio") + } +} + +func TestConfirmedCombinedFailureShrinksOnlyBoundedHiveAdmissions(t *testing.T) { + store := testStore(t) + now := epoch.Add(time.Hour) + cfg := configFor(config.IntegrationEnforce) + cfg.MaxActive = 3 + if err := store.Append(coverageEvent(now)); err != nil { + t.Fatal(err) + } + for _, number := range []int{1, 2, 3} { + putIntent(t, store, testIntent(number)) + } + input := Input{Repo: "o/r", Base: "v5", Now: now, PlanApproved: true, PlanRevision: 1, Claim: "owner", ClaimGeneration: 1, ClaimExpiresAt: now.Add(time.Hour), Reserve: true} + for _, key := range []string{"o/r#1", "o/r#2"} { + input.Key = key + if decision, err := store.Decide(cfg, input); err != nil || !decision.Allowed { + t.Fatalf("%+v %v", decision, err) + } + } + rejected := Evidence{Subject: testSubject(), State: Rejected, Source: "github-actions-bound-v1", ObservedAt: now, RunID: 20, SuiteID: 2, CheckIDs: []int64{3}} + if err := store.Append(Event{Version: 1, ID: "combined-red", At: now, Evidence: &rejected}); err != nil { + t.Fatal(err) + } + input.Key, input.Reserve = "o/r#3", false + decision, err := store.Decide(cfg, input) + if err != nil || decision.Allowed || decision.ActiveLimit != 2 || decision.FailureBackoff != 1 { + t.Fatalf("%+v %v", decision, err) + } + if len(store.Snapshot(now).Reservations) != 2 { + t.Fatal("failure canceled existing valid work") + } + cfg.Mode = config.IntegrationAdvisory + if decision, err := store.Decide(cfg, input); err != nil || !decision.Allowed || !decision.WouldBlock { + t.Fatalf("%+v %v", decision, err) + } +} diff --git a/src/pkg/convergence/integration/drift.go b/src/pkg/convergence/integration/drift.go new file mode 100644 index 0000000000..74476bb8f4 --- /dev/null +++ b/src/pkg/convergence/integration/drift.go @@ -0,0 +1,58 @@ +package integration + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// ObserveFiles reuses the merge planner's existing head-keyed file probe. +// The probe is advisory: it does not attest a fresh remote head or grant proof. +func (s *Store) ObserveFiles(repo, head string, files []string, now time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + unlock, err := s.writerLock() + if err != nil { + return err + } + defer unlock() + for _, intent := range s.intents { + if !strings.EqualFold(intent.Repo, repo) || intent.Head != head { + continue + } + drift := Drift{Key: intent.Key, Repo: repo, Head: head, IntentGeneration: intent.Generation, ObservedAt: now, Source: "existing-head-keyed-overlap-probe-v1", Actual: append([]string(nil), files...)} + sort.Strings(drift.Actual) + for _, file := range drift.Actual { + anticipated := false + for _, path := range append(append([]string(nil), intent.Paths...), intent.Generated...) { + if file == path || strings.HasPrefix(file, strings.TrimSuffix(path, "/")+"/") { + anticipated = true + break + } + } + if !anticipated { + drift.OutsideAnticipated = append(drift.OutsideAnticipated, file) + } + } + identity := drift + identity.ObservedAt = time.Time{} + id := "files/" + eventDigest(Event{Drift: &identity}) + if _, exists := s.seen[id]; exists { + continue + } + if err := s.appendLocked(Event{Version: SchemaVersion, ID: id, At: now, Drift: &drift}); err != nil { + return fmt.Errorf("recording overlap scope drift: %w", err) + } + } + return nil +} + +func relationDisposition(intent Intent, key, kind string) string { + for _, assessment := range intent.Relations { + if assessment.Key == key && assessment.Kind == kind { + return "; operator assessment=" + assessment.Disposition + " (" + assessment.Provenance + ")" + } + } + return "" +} diff --git a/src/pkg/convergence/integration/fence.go b/src/pkg/convergence/integration/fence.go new file mode 100644 index 0000000000..1474cc7e8b --- /dev/null +++ b/src/pkg/convergence/integration/fence.go @@ -0,0 +1,85 @@ +package integration + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "syscall" +) + +// writerLock fences multiple hub processes and catches up only the appended +// suffix before making a decision. No forge I/O runs while this lock is held. +func (s *Store) writerLock() (func(), error) { + if err := os.MkdirAll(filepath.Dir(s.path), 0700); err != nil { + return nil, err + } + lock, err := os.OpenFile(s.path+".lock", os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, err + } + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil { + _ = lock.Close() + return nil, err + } + unlock := func() { _ = syscall.Flock(int(lock.Fd()), syscall.LOCK_UN); _ = lock.Close() } + f, err := os.Open(s.path) + if os.IsNotExist(err) && s.offset == 0 { + return unlock, nil + } + if err != nil { + unlock() + return nil, err + } + defer f.Close() + stat, err := f.Stat() + if err != nil { + unlock() + return nil, err + } + if stat.Size() < s.offset { + unlock() + return nil, fmt.Errorf("integration journal truncated; refusing stale writer") + } + if stat.Size() > maxJournalBytes { + unlock() + return nil, fmt.Errorf("integration journal exceeds bounded replay size") + } + if stat.Size() == s.offset { + return unlock, nil + } + var last [1]byte + if _, err := f.ReadAt(last[:], stat.Size()-1); err != nil || last[0] != '\n' { + unlock() + return nil, fmt.Errorf("integration journal has an incomplete concurrent record") + } + if _, err := f.Seek(s.offset, 0); err != nil { + unlock() + return nil, err + } + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 4096), maxJournalRecordBytes) + for scanner.Scan() { + if len(s.seen) > maxJournalRecords { + unlock() + return nil, fmt.Errorf("integration journal exceeds bounded replay index") + } + var event Event + if err := json.Unmarshal(scanner.Bytes(), &event); err != nil { + unlock() + return nil, err + } + if err := s.validate(event); err != nil { + unlock() + return nil, err + } + s.apply(event) + s.offset += int64(len(scanner.Bytes()) + 1) + } + if err := scanner.Err(); err != nil { + unlock() + return nil, err + } + return unlock, nil +} diff --git a/src/pkg/convergence/integration/generation_test.go b/src/pkg/convergence/integration/generation_test.go new file mode 100644 index 0000000000..8f6d3d2755 --- /dev/null +++ b/src/pkg/convergence/integration/generation_test.go @@ -0,0 +1,116 @@ +package integration + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestLateOldWorkflowGenerationCannotReviveCurrentProof(t *testing.T) { + for _, newerFirst := range []bool{false, true} { + store := testStore(t) + old := Evidence{Subject: testSubject(), State: Passed, ObservedAt: epoch.Add(time.Hour), Source: "bound-producer", RunID: 1, SuiteID: 2, CheckIDs: []int64{3}} + newer := old + newer.Subject.Generation++ + newer.Subject.TestedSHA = strings.Repeat("f", 40) + newer.ObservedAt = epoch + newer.RunID = 4 + events := []Event{{Version: 1, ID: "old", At: epoch.Add(time.Hour), Evidence: &old}, {Version: 1, ID: "new", At: epoch, Evidence: &newer}} + if newerFirst { + events[0], events[1] = events[1], events[0] + } + for _, event := range events { + if err := store.Append(event); err != nil { + t.Fatal(err) + } + } + if store.evidence[old.Subject.ID()].State != Superseded || store.evidence[newer.Subject.ID()].State != Passed { + t.Fatalf("generation ordering depended on delivery/update timestamps: %+v", store.evidence) + } + restarted, err := Open(store.path) + if err != nil || restarted.evidence[old.Subject.ID()].State != Superseded { + t.Fatalf("restart revived old generation: %v", err) + } + } +} + +func TestOneBoundWorkflowAttemptHasOneImmutableTestedSubject(t *testing.T) { + store := testStore(t) + receipt := Evidence{Subject: testSubject(), State: Passed, ObservedAt: epoch, Source: "github-actions-bound-v1", RunID: 20, SuiteID: 2, CheckIDs: []int64{3}} + if err := store.Append(Event{Version: 1, ID: "first", At: epoch, Evidence: &receipt}); err != nil { + t.Fatal(err) + } + receipt.Subject.TestedTree = strings.Repeat("f", 40) + if store.Append(Event{Version: 1, ID: "conflicting-subject", At: epoch, Evidence: &receipt}) == nil { + t.Fatal("same physical workflow attempt changed tested result") + } +} + +func TestUnknownBoundObservationCanAcquireExactAttemptIdentity(t *testing.T) { + store := testStore(t) + receipt := Evidence{Subject: testSubject(), State: Unknown, ObservedAt: epoch, Source: "github-actions-bound-v1", RunID: 20, SuiteID: 2} + unknownID := receipt.Subject.ID() + if err := store.Append(Event{Version: 1, ID: "unknown", At: epoch, Evidence: &receipt}); err != nil { + t.Fatal(err) + } + receipt.Subject.TestedTree = strings.Repeat("f", 40) + receipt.State, receipt.CheckIDs = Passed, []int64{3} + if err := store.Append(Event{Version: 1, ID: "bound", At: epoch, Evidence: &receipt}); err != nil { + t.Fatalf("unknown placeholder prevented actual attempt proof: %v", err) + } + if store.evidence[unknownID].State != Superseded { + t.Fatal("completed subject left its placeholder current") + } + receipt.ObservedAt = epoch.Add(time.Minute) + if err := store.Append(Event{Version: 1, ID: "bound-refresh", At: receipt.ObservedAt, Evidence: &receipt}); err != nil { + t.Fatalf("superseded unknown placeholder became an immutable bound attempt: %v", err) + } +} + +func TestFailedAuthorHeadRequiresExactSuiteAndCheckForCandidateContext(t *testing.T) { + store := testStore(t) + receipt := Evidence{Subject: testSubject(), State: Passed, ObservedAt: epoch, Source: "github-actions-attested-v1", RunID: 20, SuiteID: 2, CheckIDs: []int64{3}} + if err := store.Append(Event{Version: 1, ID: "candidate", At: epoch, Evidence: &receipt}); err != nil { + t.Fatal(err) + } + head := receipt.Subject.Members[0].Head + for _, tc := range []struct { + head string + suite, check int64 + want int + }{ + {head, 2, 3, 1}, {head, 2, 0, 0}, {head, 0, 3, 0}, {head, 2, 99, 0}, {head, 99, 3, 0}, {strings.Repeat("f", 40), 2, 3, 0}, {receipt.Subject.TestedSHA, 0, 0, 0}, {receipt.Subject.TestedSHA, 2, 3, 1}, + } { + if got := len(store.ForFailure("o/r", tc.head, tc.suite, tc.check)); got != tc.want { + t.Fatalf("failure %+v got %d receipts", tc, got) + } + } +} + +func TestReconciliationCanCompleteSameTimestampProofWithoutLateRegression(t *testing.T) { + store := testStore(t) + receipt := Evidence{Subject: testSubject(), State: Unknown, ObservedAt: epoch, Source: "github-actions-bound-v1", RunID: 20, SuiteID: 2, CheckIDs: []int64{3}} + for n, state := range []State{Unknown, Waiting, Running, Passed, Unknown, Running, Rejected, Passed, Superseded, Passed} { + receipt.State = state + if err := store.Append(Event{Version: 1, ID: fmt.Sprintf("progress/%d", n), At: epoch.Add(time.Duration(n) * time.Minute), Evidence: &receipt}); err != nil { + t.Fatal(err) + } + current := store.evidence[receipt.Subject.ID()].State + want := state + switch n { + case 4, 5: + want = Passed + case 7: + want = Rejected + case 9: + want = Superseded + } + if current != want { + t.Fatalf("same-source-time observation %d changed %s to %s", n, want, current) + } + } + if evidenceStateRank("invalid") != -1 { + t.Fatal("invalid state ordered as evidence") + } +} diff --git a/src/pkg/convergence/integration/job_delivery_test.go b/src/pkg/convergence/integration/job_delivery_test.go new file mode 100644 index 0000000000..41755b6709 --- /dev/null +++ b/src/pkg/convergence/integration/job_delivery_test.go @@ -0,0 +1,42 @@ +package integration + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestLateQueuedAndPartialCompletionCannotEraseObservedRunnerExecution(t *testing.T) { + store := testStore(t) + job := Job{Repo: "o/r", Base: "v5", ID: 1, RunID: 2, Attempt: 1, Head: strings.Repeat("a", 40), RunnerID: 3, + Status: "in_progress", StartedAt: epoch, ObservedAt: epoch.Add(time.Minute)} + queued := job + queued.Status, queued.RunnerID, queued.ObservedAt = "queued", 0, epoch.Add(2*time.Minute) + completed := queued + completed.Status, completed.Conclusion, completed.CompletedAt, completed.ObservedAt = "completed", "cancelled", epoch.Add(3*time.Minute), epoch.Add(4*time.Minute) + completed.StartedAt = time.Time{} + partial := completed + partial.CompletedAt, partial.ObservedAt = time.Time{}, epoch.Add(5*time.Minute) + for n, observation := range []Job{job, queued, completed, partial} { + if err := store.Append(Event{Version: 1, ID: fmt.Sprintf("job/%d", n), At: observation.ObservedAt, Job: &observation}); err != nil { + t.Fatal(err) + } + actual := store.jobs[job.Key()] + if n == 1 && actual.Status != "in_progress" { + t.Fatal("late queued delivery erased assigned execution") + } + } + actual := store.jobs[job.Key()] + if actual.RunnerID != 3 || actual.StartedAt != epoch || actual.CompletedAt != completed.CompletedAt { + t.Fatalf("partial terminal delivery erased known physical execution: %+v", actual) + } + report := cost(store.jobs, Coverage{From: epoch, Through: completed.ObservedAt, Complete: true}, "o/r", "v5", epoch, completed.ObservedAt) + if report.RunnerMinutes == nil || *report.RunnerMinutes != 3 { + t.Fatalf("canceled execution lost: %+v", report) + } + restarted, err := Open(store.path) + if err != nil || restarted.jobs[job.Key()].RunnerID != 3 { + t.Fatalf("restart lost assigned execution: %v", err) + } +} diff --git a/src/pkg/convergence/integration/ledger_contract_test.go b/src/pkg/convergence/integration/ledger_contract_test.go new file mode 100644 index 0000000000..23cb6ce64d --- /dev/null +++ b/src/pkg/convergence/integration/ledger_contract_test.go @@ -0,0 +1,136 @@ +package integration + +import ( + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +func TestDestroyedGroupAndPolicyCannotReviveAfterRestart(t *testing.T) { + store := testStore(t) + subject := testSubject() + policy := config.IntegrationPolicy{Repo: subject.Repo, Base: subject.Base, Engine: subject.Engine, Workflow: subject.Workflow, Blob: strings.Repeat("a", 40), AppID: 42, Jobs: []string{"policy"}} + subject.Policy = policy.Fingerprint() + if err := store.SetPolicies([]config.IntegrationPolicy{policy}, epoch); err != nil { + t.Fatal(err) + } + receipt := Evidence{Subject: subject, State: Passed, ObservedAt: epoch, Source: "github-actions-bound-v1", RunID: 1, SuiteID: 2, CheckIDs: []int64{3}} + if err := store.InvalidateGroup(subject.Repo, subject.Authority, subject.TestedSHA, "before-create", "group destroyed", epoch.Add(time.Minute)); err != nil { + t.Fatal(err) + } + if err := store.InvalidateGroup(subject.Repo, subject.Authority, subject.TestedSHA, "before-create", "group destroyed", epoch.Add(time.Hour)); err != nil { + t.Fatal(err) + } + if err := store.Append(Event{Version: 1, ID: "late-pass", At: epoch.Add(time.Hour), Evidence: &receipt}); err != nil { + t.Fatal(err) + } + if store.evidence[subject.ID()].State != Superseded { + t.Fatal("destroy-before-create revived passing group") + } + receipt.Subject.Authority = "another-group" + if err := store.Append(Event{Version: 1, ID: "other-pass", At: epoch, Evidence: &receipt}); err != nil { + t.Fatal(err) + } + if store.evidence[receipt.Subject.ID()].State != Passed { + t.Fatal("independent group blocked") + } + if err := store.SetPolicies(nil, epoch.Add(2*time.Minute)); err != nil { + t.Fatal(err) + } + if store.evidence[receipt.Subject.ID()].State != Superseded { + t.Fatal("removed policy remained passed") + } + receipt.Subject.Authority = "late-old-policy" + receipt.ObservedAt = epoch.Add(time.Hour) + if err := store.Append(Event{Version: 1, ID: "late-old-policy", At: receipt.ObservedAt, Evidence: &receipt}); err != nil { + t.Fatal(err) + } + if store.evidence[receipt.Subject.ID()].State != Superseded { + t.Fatal("old policy revived after removal") + } + restarted, err := Open(store.path) + if err != nil { + t.Fatal(err) + } + if err := restarted.SetPolicies(nil, epoch.Add(3*time.Hour)); err != nil { + t.Fatal(err) + } + if len(restarted.Snapshot(epoch).Evidence) != 3 || restarted.evidence[receipt.Subject.ID()].State != Superseded { + t.Fatal("policy lineage not replayed") + } +} + +func TestJobCandidateCorrelationIsOrderIndependentAndNotDoubleCounted(t *testing.T) { + for _, jobFirst := range []bool{false, true} { + store := testStore(t) + subject := testSubject() + job := Job{Repo: subject.Repo, Base: subject.Base, ID: 10, RunID: 20, Attempt: subject.Attempt, Head: subject.TestedSHA, RunnerID: 30, + StartedAt: epoch, CompletedAt: epoch.Add(time.Minute), ObservedAt: epoch.Add(time.Minute), Status: "completed", Conclusion: "success"} + evidence := Evidence{Subject: subject, State: Passed, ObservedAt: epoch, Source: "bound", RunID: 20, SuiteID: 2, CheckIDs: []int64{3}} + jobEvent := Event{Version: 1, ID: "job", At: job.ObservedAt, Job: &job} + proofEvent := Event{Version: 1, ID: "proof", At: epoch, Evidence: &evidence} + events := []Event{proofEvent, jobEvent} + if jobFirst { + events = []Event{jobEvent, proofEvent} + } + for _, event := range events { + if err := store.Append(event); err != nil { + t.Fatal(err) + } + } + evidence.Subject.Policy = "another-independent-workflow-policy" + if err := store.Append(Event{Version: 1, ID: "shared-proof", At: epoch, Evidence: &evidence}); err != nil { + t.Fatal(err) + } + if len(store.jobs[job.Key()].Candidates) != 2 || len(store.jobs) != 1 { + t.Fatalf("%+v", store.jobs) + } + if len(store.ForTestedHead("O/R", subject.TestedSHA)) != 2 || len(store.ForTestedHead("other/r", subject.TestedSHA)) != 0 { + t.Fatal("head lookup lost repository identity") + } + snapshot := store.Snapshot(epoch) + snapshot.Jobs[0].Candidates[0] = "mutated" + if store.jobs[job.Key()].Candidates[0] == "mutated" { + t.Fatal("snapshot aliased durable job") + } + } +} + +func TestDispatchIntentReconcilesDAGWithoutOverwritingOwnerScope(t *testing.T) { + store := testStore(t) + intent := testIntent(1) + intent.Plan, intent.PlanRevision, intent.ScopeKnown, intent.Provenance = "unresolved", 0, false, "dispatch-enumerator" + if err := store.EnsureIntent(intent); err != nil { + t.Fatal(err) + } + intent.Plan, intent.PlanRevision = "approved-epic", 2 + intent.ObservedAt = epoch.Add(time.Minute) + if err := store.EnsureIntent(intent); err != nil { + t.Fatal(err) + } + current, found := store.Intent(intent.Key) + if !found || current.Generation != 2 || current.PlanRevision != 2 || !current.ObservedAt.Equal(epoch.Add(time.Second)) { + t.Fatalf("%+v", current) + } + if err := store.EnsureIntent(intent); err != nil { + t.Fatal(err) + } + current.Generation, current.ScopeKnown, current.Provenance = 3, true, "owner-approved-scope" + putIntent(t, store, current) + intent.PlanRevision = 4 + if err := store.EnsureIntent(intent); err != nil { + t.Fatal(err) + } + current, _ = store.Intent(intent.Key) + if current.Generation != 3 || !current.ScopeKnown { + t.Fatal("automatic discovery overwrote owner intent") + } + if len(store.Intents()) != 1 { + t.Fatal("canonical work duplicated") + } + if _, present := store.Reservation(intent.Key, epoch); present { + t.Fatal("unknown reservation invented") + } +} diff --git a/src/pkg/convergence/integration/metrics.go b/src/pkg/convergence/integration/metrics.go new file mode 100644 index 0000000000..3d2f131cd1 --- /dev/null +++ b/src/pkg/convergence/integration/metrics.go @@ -0,0 +1,74 @@ +package integration + +import ( + "strings" + "time" +) + +type CostReport struct { + Repo string `json:"repo"` + Base string `json:"base"` + From time.Time `json:"from"` + Through time.Time `json:"through"` + Cost Cost `json:"cost"` + ObservedMergedPRs int `json:"observed_merged_prs"` + MergedPRs *int `json:"merged_prs"` + RunnerMinutesPerMergedPR *float64 `json:"runner_minutes_per_merged_pr"` + AttributedIntegrationMinutes *float64 `json:"attributed_integration_minutes"` + UnattributedExecutionJobs int `json:"unattributed_execution_jobs"` + Attribution string `json:"attribution"` + Reason string `json:"reason"` +} + +func (s *Store) CostReport(repo, base string, from, through time.Time) CostReport { + s.mu.Lock() + defer s.mu.Unlock() + coverage := s.coverage[scopeKey(repo, base)] + out := CostReport{Repo: repo, Base: base, From: from, Through: through, Cost: cost(s.jobs, coverage, repo, base, from, through), + Attribution: "bound tested-candidate job execution only; unclassified jobs and author-review receipts remain separate"} + for _, receipt := range s.integrated { + if strings.EqualFold(receipt.Repo, repo) && receipt.Base == base && !receipt.MergedAt.Before(from) && !receipt.MergedAt.After(through) { + out.ObservedMergedPRs++ + } + } + if coverage.MergedComplete && !coverage.From.After(from) && !coverage.Through.Before(through) { + count := out.ObservedMergedPRs + out.MergedPRs = &count + if count > 0 && out.Cost.RunnerMinutes != nil { + average := *out.Cost.RunnerMinutes / float64(count) + out.RunnerMinutesPerMergedPR = &average + } else { + out.Reason = "no completed merged-PR denominator or complete job cohort" + } + } else { + out.Reason = "merged-PR cohort incomplete: " + coverage.MergeReason + } + for _, job := range s.jobs { + if !strings.EqualFold(job.Repo, repo) || job.Base != base { + continue + } + one := cost(map[string]Job{job.Key(): job}, coverage, repo, base, from, through) + if one.Jobs == 0 { + continue + } + bound := false + for _, id := range job.Candidates { + receipt, exists := s.evidence[id] + if exists && receipt.Subject.Validate() == nil && receipt.Source != "github-workflow-run-v1" && + receipt.RunID == job.RunID && receipt.Subject.Attempt == job.Attempt { + bound = true + break + } + } + if !bound { + out.UnattributedExecutionJobs++ + continue + } + if out.AttributedIntegrationMinutes == nil { + value := 0.0 + out.AttributedIntegrationMinutes = &value + } + *out.AttributedIntegrationMinutes += one.ObservedRunnerMinutes + } + return out +} diff --git a/src/pkg/convergence/integration/policy.go b/src/pkg/convergence/integration/policy.go new file mode 100644 index 0000000000..49519b6158 --- /dev/null +++ b/src/pkg/convergence/integration/policy.go @@ -0,0 +1,51 @@ +package integration + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +func groupChangeKey(repo, authority, head string) string { + return strings.ToLower(repo) + "#" + authority + "#" + head +} + +func appendCandidate(ids []string, id string) []string { + for _, existing := range ids { + if existing == id { + return ids + } + } + return append(ids, id) +} + +func (s *Store) policyActive(subject Subject) bool { + for _, binding := range s.policies.Active { + if strings.EqualFold(binding.Repo, subject.Repo) && binding.Base == subject.Base && binding.Fingerprint == subject.Policy { + return true + } + } + return false +} + +func (s *Store) SetPolicies(policies []config.IntegrationPolicy, now time.Time) error { + set := PolicySet{Active: make([]PolicyBinding, 0, len(policies))} + for _, policy := range policies { + set.Active = append(set.Active, PolicyBinding{Repo: strings.ToLower(policy.Repo), Base: policy.Base, Fingerprint: policy.Fingerprint()}) + } + sort.Slice(set.Active, func(i, j int) bool { return set.Active[i].Fingerprint < set.Active[j].Fingerprint }) + s.mu.Lock() + defer s.mu.Unlock() + unlock, err := s.writerLock() + if err != nil { + return err + } + defer unlock() + if s.policies != nil && eventDigest(Event{PolicySet: s.policies}) == eventDigest(Event{PolicySet: &set}) { + return nil + } + return s.appendLocked(Event{Version: SchemaVersion, ID: fmt.Sprintf("policies/%d", now.UnixNano()), At: now, PolicySet: &set}) +} diff --git a/src/pkg/convergence/integration/regression_test.go b/src/pkg/convergence/integration/regression_test.go new file mode 100644 index 0000000000..16541f0ccc --- /dev/null +++ b/src/pkg/convergence/integration/regression_test.go @@ -0,0 +1,268 @@ +package integration + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +func TestConcurrentWritersFenceCapacityAndReplay(t *testing.T) { + first := testStore(t) + second, err := Open(first.path) + if err != nil { + t.Fatal(err) + } + now := epoch.Add(time.Hour) + for _, intent := range []Intent{testIntent(1), testIntent(2)} { + putIntent(t, first, intent) + } + if err := first.Append(coverageEvent(now)); err != nil { + t.Fatal(err) + } + cfg := configFor(config.IntegrationEnforce) + cfg.MaxActive = 1 + results := make(chan bool, 2) + var workers sync.WaitGroup + for index, writer := range []*Store{first, second} { + workers.Add(1) + go func(index int, writer *Store) { + defer workers.Done() + input := Input{Key: fmt.Sprintf("o/r#%d", index+1), Repo: "o/r", Base: "v5", Now: now, Reserve: true, PlanApproved: true, PlanRevision: 1, + Claim: "worker", ClaimGeneration: 1, ClaimExpiresAt: now.Add(time.Hour)} + decision, err := writer.Decide(cfg, input) + if err != nil { + t.Error(err) + } + results <- err == nil && decision.Allowed + }(index, writer) + } + workers.Wait() + close(results) + admitted := 0 + for allowed := range results { + if allowed { + admitted++ + } + } + if admitted != 1 { + t.Fatalf("%d admissions for one slot across two writers", admitted) + } + restarted, err := Open(first.path) + if err != nil || len(restarted.Snapshot(now).Reservations) != 1 { + t.Fatalf("restart lost fencing: %v", err) + } + if err := os.Truncate(first.path, 0); err != nil { + t.Fatal(err) + } + if first.EnsureIntent(testIntent(3)) == nil { + t.Fatal("stale writer accepted truncated journal") + } +} + +func TestReleaseFencesOldGenerationAndIntentDrift(t *testing.T) { + store := testStore(t) + now := epoch.Add(time.Hour) + intent := testIntent(1) + putIntent(t, store, intent) + if err := store.Append(coverageEvent(now)); err != nil { + t.Fatal(err) + } + input := Input{Key: intent.Key, Repo: intent.Repo, Base: intent.Base, Now: now, Reserve: true, Claim: "owner", ClaimGeneration: 2, ClaimExpiresAt: now.Add(time.Hour), PlanApproved: true, PlanRevision: 1} + if decision, err := store.Decide(configFor(config.IntegrationEnforce), input); err != nil || !decision.Allowed { + t.Fatalf("%+v %v", decision, err) + } + for _, release := range []Event{ + {Version: 1, ID: "old-generation", At: now, Release: intent.Key, ReleaseClaim: "owner", ReleaseGeneration: 1}, + {Version: 1, ID: "wrong-owner", At: now, Release: intent.Key, ReleaseClaim: "other", ReleaseGeneration: 2}, + } { + if err := store.Append(release); err != nil { + t.Fatal(err) + } + if len(store.Snapshot(now).Reservations) != 1 { + t.Fatal("delayed release freed a replacement holder") + } + } + intent.Generation++ + intent.Paths = []string{"new.go"} + putIntent(t, store, intent) + if len(store.Snapshot(now).Reservations) != 0 { + t.Fatal("scope generation retained old reservation") + } + if err := store.EnsureIntent(testIntent(1)); err != nil { + t.Fatal(err) + } + if store.Snapshot(now).Intents[0].Generation != 2 { + t.Fatal("dispatch overwrote owner intent") + } +} + +func TestRetryBudgetRenewalAndQualification(t *testing.T) { + store := testStore(t) + now := epoch.Add(time.Hour) + intent := testIntent(1) + intent.Head = strings.Repeat("a", 40) + putIntent(t, store, intent) + if err := store.Append(coverageEvent(now)); err != nil { + t.Fatal(err) + } + cfg := configFor(config.IntegrationEnforce) + cfg.MaxRunnerMinutes = 15 + cfg.AdmissionRunnerMinutes = 10 + input := Input{Key: intent.Key, Repo: intent.Repo, Base: intent.Base, Now: now, Reserve: true, Claim: "owner", ClaimGeneration: 1, ClaimExpiresAt: now.Add(time.Hour), PlanApproved: true, PlanRevision: 1} + if d, err := store.Decide(cfg, input); err != nil || !d.Allowed { + t.Fatalf("%+v %v", d, err) + } + other := testIntent(2) + putIntent(t, store, other) + second := input + second.Key = other.Key + if d, err := store.Decide(cfg, second); err != nil || d.Allowed || !strings.Contains(d.Reason, "allocations") { + t.Fatalf("%+v %v", d, err) + } + job := Job{Repo: "o/r", Base: "v5", ID: 1, RunID: 2, Attempt: cfg.MaxAttempts, Head: intent.Head, Status: "completed", Conclusion: "failure", ObservedAt: now} + if err := store.Append(Event{Version: 1, ID: "failed-rerun", At: now, Job: &job}); err != nil { + t.Fatal(err) + } + probe := input + probe.Reserve = false + if d, err := store.Decide(cfg, probe); err != nil || d.Allowed || !strings.Contains(d.Reason, "attempts") { + t.Fatalf("%+v %v", d, err) + } + input.Now = now.Add(time.Minute) + input.RenewOnly = true + if d, err := store.Decide(cfg, input); err != nil || !d.Allowed { + t.Fatalf("heartbeat was new admission: %+v %v", d, err) + } + rejected := Evidence{Subject: testSubject(), State: Rejected, ObservedAt: now, Source: "bound-producer"} + rejected.Subject.Attempt = cfg.MaxAttempts + if err := store.Append(Event{Version: 1, ID: "candidate-red", At: now, Evidence: &rejected}); err != nil { + t.Fatal(err) + } + intent.Generation++ + intent.Head = rejected.Subject.Members[1].Head + putIntent(t, store, intent) + probe.Head = intent.Head + if d, err := store.Decide(cfg, probe); err != nil || d.Allowed || !strings.Contains(d.Reason, "candidate attempts") { + t.Fatalf("%+v %v", d, err) + } + intent.ScopeKnown = false + intent.Generation++ + putIntent(t, store, intent) + for _, mode := range []string{config.IntegrationAdvisory, config.IntegrationEnforce} { + cfg.Mode = mode + d, err := store.Decide(cfg, probe) + if err != nil || !d.WouldBlock || d.Allowed != (mode == config.IntegrationAdvisory) { + t.Fatalf("%+v %v", d, err) + } + } + probe.Key = "o/r#999" + if d, err := store.Decide(cfg, probe); err != nil || d.Allowed { + t.Fatalf("unknown scope admitted %+v %v", d, err) + } +} + +func TestInvalidationBeforeObservationAndMergedLifecycle(t *testing.T) { + store := testStore(t) + subject := testSubject() + if err := store.InvalidateMember(subject.Repo, 1, "", "review withdrawn", "early", epoch.Add(time.Minute)); err != nil { + t.Fatal(err) + } + passed := Evidence{Subject: subject, State: Passed, ObservedAt: epoch, Source: "bound-producer", RunID: 1, SuiteID: 2, CheckIDs: []int64{3}} + if err := store.Append(Event{Version: 1, ID: "late-observation", At: epoch.Add(time.Hour), Evidence: &passed}); err != nil { + t.Fatal(err) + } + if got := store.Snapshot(epoch).Evidence[0]; got.State != Superseded || got.InvalidatedBy != "member/early" { + t.Fatalf("%+v", got) + } + if err := store.InvalidateMember(subject.Repo, 1, "", "review withdrawn", "early", epoch.Add(time.Hour)); err != nil { + t.Fatal("delivery replay changed invalidation", err) + } + fresh := passed + fresh.Subject.Generation++ + fresh.ObservedAt = epoch.Add(2 * time.Minute) + if err := store.Append(Event{Version: 1, ID: "fresh-observation", At: fresh.ObservedAt, Evidence: &fresh}); err != nil { + t.Fatal(err) + } + if err := store.InvalidateGroup(subject.Repo, subject.Authority, strings.Repeat("f", 40), "other", "different group destroyed", epoch.Add(time.Hour)); err != nil { + t.Fatal(err) + } + if store.evidence[fresh.Subject.ID()].State != Passed { + t.Fatal("unrelated group destroyed valid prefix") + } + if err := store.InvalidateGroup(subject.Repo, subject.Authority, subject.TestedSHA, "destroyed", "group destroyed", epoch.Add(time.Hour)); err != nil { + t.Fatal(err) + } + if store.evidence[fresh.Subject.ID()].State != Superseded { + t.Fatal("destroyed group proof retained") + } + receipt := IntegrationReceipt{Repo: "o/r", PR: 1, Head: subject.Members[0].Head, MergeSHA: subject.TestedSHA, MergedAt: epoch, Source: "signed-merge"} + if err := store.Append(Event{Version: 1, ID: "merged", At: epoch, Integrated: &receipt}); err != nil { + t.Fatal(err) + } + if _, ok := store.Integrated("o/r", 1); !ok { + t.Fatal("merge receipt missing") + } + if _, ok := store.Integrated("o/r", 2); ok { + t.Fatal("closure inferred integration") + } + restarted, err := Open(store.path) + if err != nil { + t.Fatal(err) + } + if len(restarted.Snapshot(epoch).Integrated) != 1 { + t.Fatal("merge receipt lost across restart") + } +} + +func TestStoreRejectsMalformedObservationsAndFilesystemFailures(t *testing.T) { + store := testStore(t) + for index, event := range []Event{ + {Version: 99, ID: "bad", At: epoch}, + {Version: 1, ID: "bad", At: epoch, Job: &Job{}}, + {Version: 1, ID: "bad", At: epoch, Coverage: &Coverage{}}, + {Version: 1, ID: "bad", At: epoch, Reservation: &Reservation{}}, + {Version: 1, ID: "bad", At: epoch, Integrated: &IntegrationReceipt{}}, + {Version: 1, ID: "bad", At: epoch, MemberChange: &MemberChange{}}, + {Version: 1, ID: "bad", At: epoch, Evidence: &Evidence{Subject: testSubject(), State: "invented"}}, + {Version: 1, ID: "bad", At: epoch, Evidence: &Evidence{State: Unknown}}, + {Version: 1, ID: "bad", At: epoch, Evidence: &Evidence{Subject: testSubject(), State: Passed, ObservedAt: epoch, Source: "unbound"}}, + {Version: 1, ID: "bad", At: epoch, Job: &Job{Repo: "o/r", ID: 1, RunID: 2, Attempt: 1, Head: strings.Repeat("a", 40), ObservedAt: epoch, RunnerID: 3, StartedAt: epoch, CompletedAt: epoch.Add(-time.Hour)}}, + } { + event.ID = fmt.Sprintf("invalid-%d", index) + if store.Append(event) == nil { + t.Fatalf("invalid event %d accepted", index) + } + } + i := testIntent(1) + i.Prerequisites = []Prerequisite{{Key: "o/r#2", PlanNode: "2", Condition: "closed-is-deployed"}} + if i.Validate() == nil { + t.Fatal("ambiguous dependency condition accepted") + } + if err := os.WriteFile(store.path, []byte("{}\n"), 0600); err != nil { + t.Fatal(err) + } + if _, err := Open(store.path); err == nil { + t.Fatal("invalid replay envelope accepted") + } + blocker := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(blocker, []byte("file"), 0600); err != nil { + t.Fatal(err) + } + unwritable, err := Open(filepath.Join(blocker, "journal")) + if err == nil && unwritable.EnsureIntent(testIntent(1)) == nil { + t.Fatal("file parent accepted as directory") + } + lockBlock := testStore(t) + if err := os.Mkdir(lockBlock.path+".lock", 0700); err != nil { + t.Fatal(err) + } + if lockBlock.Append(coverageEvent(epoch)) == nil { + t.Fatal("unavailable lock ignored") + } +} diff --git a/src/pkg/convergence/integration/shadow_advice_test.go b/src/pkg/convergence/integration/shadow_advice_test.go new file mode 100644 index 0000000000..8e2ffa6efd --- /dev/null +++ b/src/pkg/convergence/integration/shadow_advice_test.go @@ -0,0 +1,37 @@ +package integration + +import ( + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +func TestAdvisoryUsesLiveClaimScopesWithoutTakingReservations(t *testing.T) { + store := testStore(t) + a, b, otherBase := testIntent(1), testIntent(2), testIntent(3) + a.Paths, b.Paths = []string{"a.go"}, []string{"b.go"} + a.Contracts, b.Contracts = []string{"wire/v1"}, []string{"wire/v1"} + a.Relations = []RelationAssessment{{Key: b.Key, Kind: "semantic", Disposition: "false-positive", Provenance: "owner-reviewed"}} + otherBase.Base = "release" + for _, intent := range []Intent{a, b, otherBase} { + putIntent(t, store, intent) + } + now := epoch.Add(time.Hour) + for _, live := range [][]string{nil, {a.Key, b.Key, b.Key, otherBase.Key, "missing"}} { + decision, err := store.Decide(configFor(config.IntegrationAdvisory), Input{Key: a.Key, Repo: a.Repo, Base: a.Base, Now: now, LiveKeys: live, PlanApproved: true, PlanRevision: 1}) + if err != nil || !decision.Allowed { + t.Fatalf("shadow advice blocked: %+v %v", decision, err) + } + if len(live) > 0 && (len(decision.Hints) != 1 || !strings.Contains(decision.Hints[0], "semantic interaction") || !strings.Contains(decision.Hints[0], "false-positive")) { + t.Fatalf("lost disjoint-file semantic advice: %+v", decision) + } + if len(live) == 0 && len(decision.Hints) != 0 { + t.Fatal("unclaimed intent treated as active") + } + } + if len(store.Snapshot(now).Reservations) != 0 { + t.Fatal("advisory created an exclusive reservation") + } +} diff --git a/src/pkg/convergence/integration/store.go b/src/pkg/convergence/integration/store.go new file mode 100644 index 0000000000..b981b347a3 --- /dev/null +++ b/src/pkg/convergence/integration/store.go @@ -0,0 +1,603 @@ +package integration + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +const ( + maxJournalRecordBytes = 1 << 20 + maxJournalBytes = 256 << 20 + maxJournalRecords = 100000 +) + +// Store serializes local durable state only. Callers must do all forge/claim +// reads before entering it; no remote I/O belongs under this mutex. +type Store struct { + mu sync.Mutex + path string + seen map[string]string + intents map[string]Intent + evidence map[string]Evidence + jobs map[string]Job + coverage map[string]Coverage + reservations map[string]Reservation + memberChanges map[string]Event + groupChanges map[string]Event + policies *PolicySet + drifts map[string]Drift + integrated map[string]IntegrationReceipt + poison error + offset int64 +} + +func (s *Store) HasEvent(id string) bool { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.seen[id] + return ok +} + +func (s *Store) EnsureIntent(i Intent) error { + s.mu.Lock() + defer s.mu.Unlock() + unlock, err := s.writerLock() + if err != nil { + return err + } + defer unlock() + if previous, ok := s.intents[i.Key]; ok { + if previous.Provenance == "dispatch-enumerator" && i.Provenance == previous.Provenance && !previous.ScopeKnown && + (previous.Plan != i.Plan || previous.PlanNode != i.PlanNode || previous.PlanRevision != i.PlanRevision) { + now := i.ObservedAt + i.Generation, i.ObservedAt = previous.Generation+1, previous.ObservedAt + return s.appendLocked(Event{Version: SchemaVersion, ID: fmt.Sprintf("dispatch-intent/%s/%d", i.Key, i.Generation), At: now, Intent: &i}) + } + return nil + } + return s.appendLocked(Event{Version: SchemaVersion, ID: "dispatch-intent/" + i.Key, At: i.ObservedAt, Intent: &i}) +} + +func (s *Store) Intent(key string) (Intent, bool) { + s.mu.Lock() + defer s.mu.Unlock() + intent, found := s.intents[key] + return cloneIntent(intent), found +} + +func (s *Store) Intents() []Intent { + s.mu.Lock() + defer s.mu.Unlock() + intents := make([]Intent, 0, len(s.intents)) + for _, intent := range s.intents { + intents = append(intents, cloneIntent(intent)) + } + return intents +} + +func (s *Store) Reservation(key string, now time.Time) (Reservation, bool) { + s.mu.Lock() + defer s.mu.Unlock() + reservation, found := s.reservations[key] + return reservation, found && reservation.ExpiresAt.After(now) +} + +func (s *Store) ForTestedHead(repo, head string) []Evidence { + s.mu.Lock() + defer s.mu.Unlock() + var receipts []Evidence + for _, receipt := range s.evidence { + if strings.EqualFold(receipt.Subject.Repo, repo) && receipt.Subject.TestedSHA == head { + receipts = append(receipts, cloneEvidence(receipt)) + } + } + return receipts +} + +func evidenceStateRank(state State) int { + switch state { + case Unknown: + return 0 + case Waiting: + return 1 + case Running: + return 2 + case Passed: + return 3 + case Rejected: + return 4 + case Superseded: + return 5 + default: + return -1 + } +} + +func (s *Store) ForFailure(repo, head string, suite, check int64) []Evidence { + s.mu.Lock() + defer s.mu.Unlock() + var receipts []Evidence + for _, receipt := range s.evidence { + if !strings.EqualFold(receipt.Subject.Repo, repo) { + continue + } + bound := receipt.RunID > 0 || receipt.SuiteID > 0 || len(receipt.CheckIDs) > 0 + checkKnown := false + for _, id := range receipt.CheckIDs { + if id == check && check > 0 { + checkKnown = true + } + } + if bound && (suite <= 0 || receipt.SuiteID != suite || !checkKnown) { + continue + } + matched := receipt.Subject.TestedSHA == head + if !matched && bound { + for _, member := range receipt.Subject.Members { + if member.Head == head { + matched = true + } + } + } + if matched { + receipts = append(receipts, cloneEvidence(receipt)) + } + } + return receipts +} + +func Open(path string) (*Store, error) { + s := &Store{path: path, seen: map[string]string{}, intents: map[string]Intent{}, + evidence: map[string]Evidence{}, jobs: map[string]Job{}, coverage: map[string]Coverage{}, + reservations: map[string]Reservation{}, memberChanges: map[string]Event{}, groupChanges: map[string]Event{}, integrated: map[string]IntegrationReceipt{}, drifts: map[string]Drift{}} + _, err := os.Stat(path) + if os.IsNotExist(err) { + return s, nil + } + if err != nil { + return nil, err + } + unlock, err := s.writerLock() + if err != nil { + return nil, err + } + unlock() + return s, nil +} + +func eventDigest(e Event) string { + b, _ := json.Marshal(e) + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} + +func scopeKey(repo, base string) string { return strings.ToLower(repo) + "#" + base } + +func (s *Store) validate(e Event) error { + if e.Version != SchemaVersion || e.ID == "" || e.At.IsZero() { + return fmt.Errorf("invalid integration event envelope") + } + if old, ok := s.seen[e.ID]; ok && old != eventDigest(e) { + return fmt.Errorf("event identity reused with different content") + } + n := 0 + for _, yes := range []bool{e.Intent != nil, e.Evidence != nil, e.Job != nil, e.Coverage != nil, e.Reservation != nil, e.Release != "", e.Invalidate != "", e.MemberChange != nil, e.Integrated != nil, e.GroupChange != nil, e.PolicySet != nil, e.Drift != nil} { + if yes { + n++ + } + } + if n != 1 { + return fmt.Errorf("event must carry exactly one observation") + } + if change := e.MemberChange; change != nil && (change.Repo == "" || change.PR <= 0) { + return fmt.Errorf("invalid member change") + } + if drift := e.Drift; drift != nil && (drift.Key == "" || drift.Repo == "" || !fullSHA(drift.Head) || drift.IntentGeneration == 0 || drift.Source == "" || drift.ObservedAt.IsZero()) { + return fmt.Errorf("invalid scope drift observation") + } + if change := e.GroupChange; change != nil && (change.Repo == "" || change.Authority == "" || !fullSHA(change.Head)) { + return fmt.Errorf("invalid group invalidation") + } + if set := e.PolicySet; set != nil { + for _, binding := range set.Active { + if binding.Repo == "" || binding.Base == "" || binding.Fingerprint == "" { + return fmt.Errorf("invalid policy binding") + } + } + } + if receipt := e.Integrated; receipt != nil && (receipt.Repo == "" || receipt.PR < 1 || !fullSHA(receipt.Head) || !fullSHA(receipt.MergeSHA) || receipt.MergedAt.IsZero() || receipt.Source == "") { + return fmt.Errorf("invalid integrated receipt") + } + if e.Intent != nil { + if err := e.Intent.Validate(); err != nil { + return err + } + if old, ok := s.intents[e.Intent.Key]; ok && e.Intent.Generation == old.Generation && eventDigest(Event{Intent: &old}) != eventDigest(Event{Intent: e.Intent}) { + return fmt.Errorf("intent generation reused") + } + visiting, done := map[string]bool{}, map[string]bool{} + var visit func(string) bool + visit = func(key string) bool { + if visiting[key] { + return true + } + if done[key] { + return false + } + visiting[key] = true + i := s.intents[key] + if key == e.Intent.Key { + i = *e.Intent + } + for _, p := range i.Prerequisites { + if visit(p.Key) { + return true + } + } + delete(visiting, key) + done[key] = true + return false + } + if visit(e.Intent.Key) { + return fmt.Errorf("intent prerequisite cycle") + } + } + if e.Evidence != nil { + for _, previous := range s.evidence { + next := e.Evidence + if (next.Source == "github-actions-bound-v1" || next.Source == "github-actions-attested-v1") && next.State != Unknown && next.RunID > 0 && + previous.ObservedState != Unknown && strings.EqualFold(previous.Subject.Repo, next.Subject.Repo) && + previous.Source == next.Source && previous.RunID == next.RunID && previous.Subject.Attempt == next.Subject.Attempt && + previous.Subject.Policy == next.Subject.Policy && previous.Subject.Workflow == next.Subject.Workflow && + previous.Subject.Authority == next.Subject.Authority && previous.Subject.ID() != next.Subject.ID() { + return fmt.Errorf("workflow attempt changed immutable tested subject") + } + } + if err := e.Evidence.Subject.Validate(); err != nil && e.Evidence.State != Unknown { + return err + } + if e.Evidence.Subject.Repo == "" || e.Evidence.Subject.Authority == "" { + return fmt.Errorf("unknown candidate still requires authoritative identity") + } + switch e.Evidence.State { + case Unknown, Waiting, Running, Passed, Rejected, Superseded: + default: + return fmt.Errorf("invalid evidence state") + } + if e.Evidence.ObservedAt.IsZero() || e.Evidence.Source == "" { + return fmt.Errorf("evidence observation missing") + } + if e.Evidence.State == Passed && (e.Evidence.RunID <= 0 || e.Evidence.SuiteID <= 0 || len(e.Evidence.CheckIDs) == 0) { + return fmt.Errorf("passed evidence requires run, suite and check identities") + } + } + if j := e.Job; j != nil { + if j.Repo == "" || j.ID <= 0 || j.RunID <= 0 || j.Attempt < 1 || !fullSHA(j.Head) || j.ObservedAt.IsZero() { + return fmt.Errorf("invalid job identity") + } + if !j.CompletedAt.IsZero() && (j.StartedAt.IsZero() || j.CompletedAt.Before(j.StartedAt)) && j.RunnerID > 0 { + return fmt.Errorf("invalid job execution interval") + } + } + if c := e.Coverage; c != nil { + if c.Repo == "" || c.Base == "" || c.Source == "" || c.From.IsZero() || c.Through.Before(c.From) { + return fmt.Errorf("invalid telemetry coverage") + } + } + if r := e.Reservation; r != nil { + if r.Key == "" || r.Claim == "" || r.ClaimGeneration == 0 || r.IntentGeneration == 0 || !r.ExpiresAt.After(e.At) { + return fmt.Errorf("invalid reservation") + } + } + return nil +} + +func (s *Store) apply(e Event) { + if _, ok := s.seen[e.ID]; ok { + return + } + s.seen[e.ID] = eventDigest(e) + if drift := e.Drift; drift != nil { + if previous, found := s.drifts[drift.Key]; !found || drift.ObservedAt.After(previous.ObservedAt) { + next := *drift + next.Actual = append([]string(nil), drift.Actual...) + next.OutsideAnticipated = append([]string(nil), drift.OutsideAnticipated...) + s.drifts[drift.Key] = next + } + } + if receipt := e.Integrated; receipt != nil { + s.integrated[fmt.Sprintf("%s#%d", strings.ToLower(receipt.Repo), receipt.PR)] = *receipt + } + if i := e.Intent; i != nil { + old, found := s.intents[i.Key] + if !found || i.Generation > old.Generation { + s.intents[i.Key] = cloneIntent(*i) + delete(s.reservations, i.Key) + } + } + if v := e.Evidence; v != nil { + k := v.Subject.ID() + supersededBy := "" + var supersedingGeneration uint64 + supersedingAttempt := 0 + for id, prior := range s.evidence { + old, next := prior.Subject, v.Subject + sameProducer := id != k && strings.EqualFold(old.Repo, next.Repo) && old.Base == next.Base && old.Engine == next.Engine && + old.Authority == next.Authority && old.Policy == next.Policy && old.Producer == next.Producer && old.Workflow == next.Workflow + completesUnknown := old.Generation == next.Generation && old.Attempt == next.Attempt && prior.State == Unknown && v.State != Unknown + lateUnknown := old.Generation == next.Generation && old.Attempt == next.Attempt && prior.State != Unknown && v.State == Unknown + if sameProducer && (completesUnknown || next.Generation > old.Generation || next.Generation == old.Generation && next.Attempt > old.Attempt) { + prior.State, prior.InvalidatedBy, prior.Reason = Superseded, e.ID, "authoritative workflow attempt superseded" + s.evidence[id] = prior + } else if sameProducer && (lateUnknown || old.Generation > next.Generation || old.Generation == next.Generation && old.Attempt > next.Attempt) { + if supersededBy == "" || old.Generation > supersedingGeneration || old.Generation == supersedingGeneration && + (old.Attempt > supersedingAttempt || old.Attempt == supersedingAttempt && id < supersededBy) { + supersededBy, supersedingGeneration, supersedingAttempt = id, old.Generation, old.Attempt + } + } + } + old, found := s.evidence[k] + progresses := v.ObservedAt.Equal(old.ObservedAt) && evidenceStateRank(v.State) > evidenceStateRank(old.State) + if !found || old.State != Superseded && (v.ObservedAt.After(old.ObservedAt) || progresses) { + next := cloneEvidence(*v) + next.ObservedState = v.State + if supersededBy != "" { + next.State, next.InvalidatedBy, next.Reason = Superseded, supersededBy, "newer authoritative workflow attempt already observed" + } + for _, member := range next.Subject.Members { + change, known := s.memberChanges[fmt.Sprintf("%s#%d", strings.ToLower(next.Subject.Repo), member.PR)] + if known && !next.ObservedAt.After(change.At) && (change.MemberChange.Head == "" || change.MemberChange.Head != member.Head) { + next.State, next.InvalidatedBy, next.Reason = Superseded, change.ID, change.Reason + } + } + if change, found := s.groupChanges[groupChangeKey(next.Subject.Repo, next.Subject.Authority, next.Subject.TestedSHA)]; found { + next.State, next.InvalidatedBy, next.Reason = Superseded, change.ID, change.Reason + } + if s.policies != nil && strings.HasPrefix(next.Subject.Policy, "workflow-policy-v1/") && !s.policyActive(next.Subject) { + next.State, next.InvalidatedBy, next.Reason = Superseded, "active-policy-set", "producer policy no longer configured" + } + s.evidence[k] = next + } + for key, job := range s.jobs { + if strings.EqualFold(job.Repo, v.Subject.Repo) && job.RunID == v.RunID && job.Attempt == v.Subject.Attempt { + job.Candidates = appendCandidate(job.Candidates, k) + s.jobs[key] = job + } + } + } + if change := e.GroupChange; change != nil { + copyChange := *change + e.GroupChange = ©Change + key := groupChangeKey(change.Repo, change.Authority, change.Head) + s.groupChanges[key] = e + for id, receipt := range s.evidence { + if groupChangeKey(receipt.Subject.Repo, receipt.Subject.Authority, receipt.Subject.TestedSHA) == key { + receipt.State, receipt.InvalidatedBy, receipt.Reason = Superseded, e.ID, e.Reason + s.evidence[id] = receipt + } + } + } + if set := e.PolicySet; set != nil { + s.policies = &PolicySet{Active: append([]PolicyBinding(nil), set.Active...)} + for id, receipt := range s.evidence { + if strings.HasPrefix(receipt.Subject.Policy, "workflow-policy-v1/") && !s.policyActive(receipt.Subject) { + receipt.State, receipt.InvalidatedBy, receipt.Reason = Superseded, e.ID, "producer policy changed or removed" + s.evidence[id] = receipt + } + } + } + if change := e.MemberChange; change != nil { + copyChange := *change + e.MemberChange = ©Change + key := fmt.Sprintf("%s#%d", strings.ToLower(change.Repo), change.PR) + if previous, found := s.memberChanges[key]; !found || e.At.After(previous.At) { + s.memberChanges[key] = e + } + for id, receipt := range s.evidence { + for _, member := range receipt.Subject.Members { + if strings.EqualFold(receipt.Subject.Repo, change.Repo) && member.PR == change.PR && !receipt.ObservedAt.After(e.At) && (change.Head == "" || member.Head != change.Head) { + receipt.State, receipt.InvalidatedBy, receipt.Reason = Superseded, e.ID, e.Reason + s.evidence[id] = receipt + } + } + } + } + if j := e.Job; j != nil { + old, found := s.jobs[j.Key()] + wasRunning := old.Status == "in_progress" || old.Status == "running" + regressesRunning := wasRunning && j.Status != "in_progress" && j.Status != "running" && j.Status != "completed" + if !found || (!j.ObservedAt.Before(old.ObservedAt) && !regressesRunning && (old.Status != "completed" || j.Status == "completed")) { + next := cloneJob(*j) + if old.RunnerID > 0 && (wasRunning || old.Status == "completed" && old.Conclusion != "skipped") { + if next.RunnerID == 0 { + next.RunnerID = old.RunnerID + } + if next.StartedAt.IsZero() { + next.StartedAt = old.StartedAt + } + if next.CompletedAt.IsZero() && old.Status == "completed" { + next.CompletedAt = old.CompletedAt + } + } + for _, id := range old.Candidates { + next.Candidates = appendCandidate(next.Candidates, id) + } + for id, receipt := range s.evidence { + if strings.EqualFold(receipt.Subject.Repo, j.Repo) && receipt.RunID == j.RunID && receipt.Subject.Attempt == j.Attempt { + next.Candidates = appendCandidate(next.Candidates, id) + } + } + s.jobs[j.Key()] = next + } + } + if c := e.Coverage; c != nil { + k := scopeKey(c.Repo, c.Base) + old, found := s.coverage[k] + if !found || !c.Through.Before(old.Through) { + s.coverage[k] = *c + } + } + if r := e.Reservation; r != nil { + s.reservations[r.Key] = *r + } + if e.Release != "" { + if e.Release == "*" { + clear(s.reservations) + } else { + r := s.reservations[e.Release] + if r.Claim == e.ReleaseClaim && r.ClaimGeneration == e.ReleaseGeneration { + delete(s.reservations, e.Release) + } + } + } + if e.Invalidate != "" { + if old, ok := s.evidence[e.Invalidate]; ok { + old.State, old.InvalidatedBy, old.Reason = Superseded, e.ID, e.Reason + s.evidence[e.Invalidate] = old + } + } +} + +func (s *Store) appendLocked(e Event) error { + if s.poison != nil { + return s.poison + } + if err := s.validate(e); err != nil { + return err + } + if _, ok := s.seen[e.ID]; ok { + return nil + } + if len(s.seen) >= maxJournalRecords && e.Release != "*" { + return fmt.Errorf("integration journal index bound reached; observations unavailable, retain audit and disable enforcement") + } + b, err := json.Marshal(e) + if err != nil { + return err + } + if len(b) >= maxJournalRecordBytes { + return fmt.Errorf("integration event exceeds journal record bound") + } + if s.offset+int64(len(b)+1) > maxJournalBytes { + return fmt.Errorf("integration journal byte bound reached; observations unavailable, retain audit and disable enforcement") + } + if err := os.MkdirAll(filepath.Dir(s.path), 0700); err != nil { + return err + } + f, err := os.OpenFile(s.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + return err + } + _, writeErr := f.Write(append(b, '\n')) + if writeErr == nil { + writeErr = f.Sync() + } + closeErr := f.Close() + if writeErr != nil { + s.poison = writeErr + return writeErr + } + if closeErr != nil { + s.poison = closeErr + return closeErr + } + s.apply(e) + s.offset += int64(len(b) + 1) + return nil +} + +func (s *Store) InvalidateGroup(repo, authority, head, delivery, reason string, now time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + unlock, err := s.writerLock() + if err != nil { + return err + } + defer unlock() + if _, exists := s.seen["group/"+delivery]; exists { + return nil + } + return s.appendLocked(Event{Version: SchemaVersion, ID: "group/" + delivery, At: now, Reason: reason, GroupChange: &GroupChange{Repo: repo, Authority: authority, Head: head}}) +} + +func (s *Store) Append(e Event) error { + s.mu.Lock() + defer s.mu.Unlock() + unlock, err := s.writerLock() + if err != nil { + return err + } + defer unlock() + return s.appendLocked(e) +} + +type Snapshot struct { + Version int `json:"version"` + Intents []Intent `json:"intents"` + Evidence []Evidence `json:"evidence"` + Jobs []Job `json:"jobs"` + Coverage []Coverage `json:"coverage"` + Reservations []Reservation `json:"reservations"` + Integrated []IntegrationReceipt `json:"integrated"` + Drifts []Drift `json:"drifts"` +} + +func (s *Store) Snapshot(now time.Time) Snapshot { + s.mu.Lock() + defer s.mu.Unlock() + out := Snapshot{Version: SchemaVersion, Intents: []Intent{}, Evidence: []Evidence{}, Jobs: []Job{}, Coverage: []Coverage{}, Reservations: []Reservation{}} + for _, v := range s.intents { + out.Intents = append(out.Intents, cloneIntent(v)) + } + for _, v := range s.evidence { + out.Evidence = append(out.Evidence, cloneEvidence(v)) + } + for _, v := range s.jobs { + out.Jobs = append(out.Jobs, cloneJob(v)) + } + for _, v := range s.coverage { + out.Coverage = append(out.Coverage, v) + } + for _, v := range s.reservations { + if v.ExpiresAt.After(now) { + out.Reservations = append(out.Reservations, v) + } + } + for _, v := range s.integrated { + out.Integrated = append(out.Integrated, v) + } + for _, v := range s.drifts { + v.Actual = append([]string(nil), v.Actual...) + v.OutsideAnticipated = append([]string(nil), v.OutsideAnticipated...) + out.Drifts = append(out.Drifts, v) + } + return out +} + +func (s *Store) Integrated(repo string, pr int) (IntegrationReceipt, bool) { + s.mu.Lock() + defer s.mu.Unlock() + receipt, ok := s.integrated[fmt.Sprintf("%s#%d", strings.ToLower(repo), pr)] + return receipt, ok +} + +func (s *Store) InvalidateMember(repo string, pr int, head, reason, delivery string, now time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + unlock, err := s.writerLock() + if err != nil { + return err + } + defer unlock() + if _, seen := s.seen["member/"+delivery]; seen { + return nil + } + return s.appendLocked(Event{Version: SchemaVersion, ID: "member/" + delivery, At: now, Reason: reason, MemberChange: &MemberChange{Repo: repo, PR: pr, Head: head}}) +} diff --git a/src/pkg/convergence/integration/store_test.go b/src/pkg/convergence/integration/store_test.go new file mode 100644 index 0000000000..bff1c4c8fa --- /dev/null +++ b/src/pkg/convergence/integration/store_test.go @@ -0,0 +1,328 @@ +package integration + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +var epoch = time.Date(2026, 10, 30, 20, 0, 0, 0, time.UTC) + +func testStore(t *testing.T) *Store { + t.Helper() + s, err := Open(filepath.Join(t.TempDir(), "ledger.jsonl")) + if err != nil { + t.Fatal(err) + } + return s +} + +func testIntent(n int) Intent { + return Intent{Version: SchemaVersion, Key: fmt.Sprintf("o/r#%d", n), Repo: "o/r", Base: "v5", Generation: 1, + Plan: "plan", PlanNode: fmt.Sprint(n), PlanRevision: 1, ScopeKnown: true, Confidence: "declared", Provenance: "approved-plan", ObservedAt: epoch.Add(time.Duration(n) * time.Second)} +} + +func testSubject() Subject { + return Subject{Version: SchemaVersion, Repo: "o/r", Base: "v5", Engine: "native", Authority: "group/ref", + Members: []Member{{PR: 1, Head: strings.Repeat("a", 40), Entry: "entry1", Position: 1}, {PR: 2, Head: strings.Repeat("b", 40), Entry: "entry2", Position: 2}}, + BaseSHA: strings.Repeat("c", 40), TestedSHA: strings.Repeat("d", 40), TestedTree: strings.Repeat("e", 40), + Policy: "policy@1", Producer: "app/42", Workflow: ".github/workflows/test.yml@abc", Attempt: 1, Generation: 1} +} + +func putIntent(t *testing.T, s *Store, i Intent) { + t.Helper() + if err := s.Append(Event{Version: SchemaVersion, ID: fmt.Sprintf("intent/%s/%d", i.Key, i.Generation), At: i.ObservedAt, Intent: &i}); err != nil { + t.Fatal(err) + } +} + +func coverageEvent(now time.Time) Event { + c := Coverage{Repo: "o/r", Base: "v5", From: now.Add(-2 * time.Hour), Through: now, Complete: true, Source: "github-actions"} + return Event{Version: SchemaVersion, ID: "coverage", At: now, Coverage: &c} +} + +func configFor(mode string) config.IntegrationOrchestrationConfig { + return config.IntegrationOrchestrationConfig{Mode: mode, Scopes: []config.IntegrationScope{{Repo: "o/r", Base: "v5"}}}.Effective() +} + +func TestExactSubjectIdentity(t *testing.T) { + s := testSubject() + if err := s.Validate(); err != nil { + t.Fatal(err) + } + for _, modify := range []func(*Subject){ + func(s *Subject) { s.Members = []Member{s.Members[1]}; s.Members[0].Position = 1 }, + func(s *Subject) { + s.Members = []Member{s.Members[1], s.Members[0]} + s.Members[0].Position, s.Members[1].Position = 1, 2 + }, + func(s *Subject) { s.BaseSHA = strings.Repeat("f", 40) }, + func(s *Subject) { s.TestedTree = strings.Repeat("f", 40) }, + func(s *Subject) { s.Policy = "policy@2" }, + func(s *Subject) { s.Producer = "app/99" }, + func(s *Subject) { s.Workflow = "different" }, + func(s *Subject) { s.Attempt++ }, + func(s *Subject) { s.Generation++ }, + } { + next := s + next.Members = append([]Member(nil), s.Members...) + modify(&next) + if next.ID() == s.ID() { + t.Fatal("candidate identity transferred across different tested subject") + } + } + for _, modify := range []func(*Subject){ + func(s *Subject) { s.Version++ }, func(s *Subject) { s.Engine = "invented" }, + func(s *Subject) { s.TestedSHA = "short" }, func(s *Subject) { s.Policy = "" }, + func(s *Subject) { s.Members[1].PR = s.Members[0].PR }, func(s *Subject) { s.Members[1].Position = 9 }, + } { + next := testSubject() + modify(&next) + if next.Validate() == nil { + t.Fatal("invalid exact subject accepted") + } + } +} + +func TestJournalReplayInvalidationAndOutOfOrder(t *testing.T) { + s := testStore(t) + subject := testSubject() + evidence := Evidence{Subject: subject, State: Passed, ObservedAt: epoch, Source: "github", RunID: 1, SuiteID: 2, CheckIDs: []int64{3}} + e := Event{Version: SchemaVersion, ID: "pass", At: epoch, Evidence: &evidence} + if err := s.Append(e); err != nil { + t.Fatal(err) + } + if err := s.Append(e); err != nil { + t.Fatal(err) + } + if err := s.InvalidateMember("o/r", 1, strings.Repeat("f", 40), "head changed", "delivery", epoch.Add(time.Second)); err != nil { + t.Fatal(err) + } + evidence.ObservedAt = epoch.Add(time.Hour) + e.ID = "late-pass" + if err := s.Append(e); err != nil { + t.Fatal(err) + } + reopened, err := Open(s.path) + if err != nil { + t.Fatal(err) + } + snap := reopened.Snapshot(epoch) + if len(snap.Evidence) != 1 || snap.Evidence[0].State != Superseded || snap.Evidence[0].InvalidatedBy == "" { + t.Fatalf("replayed evidence=%+v", snap.Evidence) + } + // Removing predecessor does not produce a receipt for its successor. + successor := subject + successor.Members = []Member{subject.Members[1]} + successor.Members[0].Position = 1 + if _, ok := reopened.evidence[successor.ID()]; ok { + t.Fatal("group proof reused after predecessor removal") + } + e.ID = "pass" + if err := s.Append(e); err == nil { + t.Fatal("reused delivery ID changed its content") + } +} + +func TestIntentDAGGenerationAndRecovery(t *testing.T) { + s := testStore(t) + a, b := testIntent(1), testIntent(2) + a.Prerequisites = []Prerequisite{{Key: b.Key, PlanNode: b.PlanNode}} + putIntent(t, s, a) + putIntent(t, s, b) + b.Generation = 2 + b.Prerequisites = []Prerequisite{{Key: a.Key, PlanNode: a.PlanNode}} + if err := s.Append(Event{Version: SchemaVersion, ID: "cycle", At: epoch, Intent: &b}); err == nil { + t.Fatal("cycle accepted") + } + b.Prerequisites = nil + putIntent(t, s, b) + older := testIntent(2) + if err := s.Append(Event{Version: SchemaVersion, ID: "old-intent", At: epoch, Intent: &older}); err != nil { + t.Fatal(err) + } + if s.intents[b.Key].Generation != 2 { + t.Fatal("intent generation regressed") + } + for _, i := range []Intent{{}, func() Intent { x := a; x.Prerequisites = []Prerequisite{{Key: a.Key, PlanNode: a.PlanNode}}; return x }()} { + if i.Validate() == nil { + t.Fatal("invalid intent accepted") + } + } + invalid := Event{Version: SchemaVersion, ID: "empty", At: epoch} + if s.Append(invalid) == nil { + t.Fatal("empty event accepted") + } + if err := os.WriteFile(filepath.Join(filepath.Dir(s.path), "corrupt"), []byte("{"), 0600); err != nil { + t.Fatal(err) + } + if _, err := Open(filepath.Join(filepath.Dir(s.path), "corrupt")); err == nil { + t.Fatal("corrupt ledger reset to empty") + } + if err := os.WriteFile(filepath.Join(filepath.Dir(s.path), "partial"), []byte(`{}`), 0600); err != nil { + t.Fatal(err) + } + if _, err := Open(filepath.Join(filepath.Dir(s.path), "partial")); err == nil { + t.Fatal("incomplete tail accepted") + } +} + +func TestOffAdvisoryAndValidatedDependencies(t *testing.T) { + s := testStore(t) + i := testIntent(2) + i.Prerequisites = []Prerequisite{{Key: "o/r#1", PlanNode: "1"}} + putIntent(t, s, i) + in := Input{Key: i.Key, Repo: i.Repo, Base: i.Base, Now: epoch.Add(time.Hour)} + for _, mode := range []string{config.IntegrationOff, config.IntegrationAdvisory, config.IntegrationEnforce} { + d, err := s.Decide(configFor(mode), in) + if err != nil || d.Allowed != (mode != config.IntegrationEnforce) { + t.Fatalf("mode=%s decision=%+v err=%v", mode, d, err) + } + } + if err := s.Append(coverageEvent(in.Now)); err != nil { + t.Fatal(err) + } + in.PlanApproved, in.PlanRevision = true, 1 + in.Dependencies = map[string]Dependency{"o/r#1": {Known: true, Satisfied: false}} + d, err := s.Decide(configFor(config.IntegrationEnforce), in) + if err != nil || d.Allowed || !strings.Contains(d.Reason, "prerequisite") { + t.Fatalf("%+v %v", d, err) + } + in.Dependencies["o/r#1"] = Dependency{Known: true, Satisfied: true} + d, err = s.Decide(configFor(config.IntegrationEnforce), in) + if err != nil || !d.Allowed { + t.Fatalf("%+v %v", d, err) + } + in.Base = "other" + d, err = s.Decide(configFor(config.IntegrationEnforce), in) + if err != nil || !d.Allowed { + t.Fatal("out of scope blocked") + } +} + +func TestReservationsRaceRenewExpiryRollbackAndHints(t *testing.T) { + s := testStore(t) + now := epoch.Add(time.Hour) + if err := s.Append(coverageEvent(now)); err != nil { + t.Fatal(err) + } + a, b, c := testIntent(1), testIntent(2), testIntent(3) + a.Paths, b.Paths = []string{"same.go"}, []string{"same.go"} + a.Contracts, c.Contracts = []string{"wire/v1"}, []string{"wire/v1"} + c.Paths = []string{"different.go"} + for _, i := range []Intent{a, b, c} { + putIntent(t, s, i) + } + cfg := configFor(config.IntegrationEnforce) + input := Input{Key: a.Key, Repo: "o/r", Base: "v5", Now: now, Reserve: true, Claim: "owner", ClaimGeneration: 1, ClaimExpiresAt: now.Add(time.Hour), PlanApproved: true, PlanRevision: 1} + d, err := s.Decide(cfg, input) + if err != nil || !d.Allowed { + t.Fatalf("%+v %v", d, err) + } + for _, i := range []Intent{b, c} { + in := input + in.Key = i.Key + in.Reserve = false + d, err := s.Decide(cfg, in) + if err != nil || !d.Allowed || len(d.Hints) != 1 { + t.Fatalf("overlap became lock: %+v %v", d, err) + } + } + var wg sync.WaitGroup + allowed := make(chan bool, 2) + for _, owner := range []string{"new1", "new2"} { + wg.Add(1) + go func(owner string) { + defer wg.Done() + in := input + in.Claim = owner + d, err := s.Decide(cfg, in) + allowed <- err == nil && d.Allowed + }(owner) + } + wg.Wait() + close(allowed) + for ok := range allowed { + if ok { + t.Fatal("competing claim replaced active reservation") + } + } + input.Now = now.Add(time.Minute) + if _, err := s.Decide(cfg, input); err != nil { + t.Fatal(err) + } + reopened, err := Open(s.path) + if err != nil || len(reopened.Snapshot(input.Now).Reservations) != 1 { + t.Fatal("live reservation not replayed", err) + } + if len(reopened.Snapshot(now.Add(time.Hour)).Reservations) != 0 { + t.Fatal("reservation never expired") + } + if err := s.Append(Event{Version: SchemaVersion, ID: "rollback", At: now, Release: "*"}); err != nil { + t.Fatal(err) + } + if len(s.Snapshot(now).Reservations) != 0 || len(s.Snapshot(now).Intents) != 3 { + t.Fatal("rollback dropped evidence or retained reservations") + } + order := s.Order([]string{c.Key, a.Key, b.Key, "o/r#99"}) + if strings.Join(order, ",") != a.Key+","+b.Key+","+c.Key+",o/r#99" { + t.Fatal(order) + } +} + +func TestActualJobAccountingClipsAndDeduplicates(t *testing.T) { + s := testStore(t) + now := epoch.Add(time.Hour) + if err := s.Append(coverageEvent(now)); err != nil { + t.Fatal(err) + } + jobs := []Job{ + {Repo: "o/r", Base: "v5", ID: 1, RunID: 10, Attempt: 1, Head: strings.Repeat("a", 40), RunnerID: 9, Status: "completed", Conclusion: "cancelled", + StartedAt: now.Add(-2 * time.Hour), CompletedAt: now.Add(-30 * time.Minute), ObservedAt: now, Candidates: []string{"a", "b"}}, + {Repo: "o/r", Base: "v5", ID: 2, RunID: 11, Attempt: 1, Head: strings.Repeat("a", 40), Status: "queued", ObservedAt: now}, + {Repo: "o/r", Base: "v5", ID: 3, RunID: 12, Attempt: 1, Head: strings.Repeat("a", 40), Status: "completed", Conclusion: "skipped", ObservedAt: now}, + } + for n, j := range jobs { + if err := s.Append(Event{Version: SchemaVersion, ID: fmt.Sprint(n), At: now, Job: &j}); err != nil { + t.Fatal(err) + } + } + cost := s.Cost("o/r", "v5", now.Add(-time.Hour), now) + if !cost.Complete || cost.RunnerMinutes == nil || *cost.RunnerMinutes != 30 || cost.Jobs != 1 { + t.Fatalf("%+v", cost) + } + // The same physical job delivered twice is still one cost. + j := jobs[0] + if err := s.Append(Event{Version: SchemaVersion, ID: "duplicate-job", At: now, Job: &j}); err != nil { + t.Fatal(err) + } + j.Attempt = 2 + j.StartedAt = now.Add(-10 * time.Minute) + j.CompletedAt = now.Add(-5 * time.Minute) + if err := s.Append(Event{Version: SchemaVersion, ID: "attempt2", At: now, Job: &j}); err != nil { + t.Fatal(err) + } + cost = s.Cost("o/r", "v5", now.Add(-time.Hour), now) + if cost.RunnerMinutes == nil || *cost.RunnerMinutes != 35 || cost.Jobs != 2 { + t.Fatalf("%+v", cost) + } + j.ID = 4 + j.Status = "in_progress" + j.CompletedAt = time.Time{} + j.ObservedAt = now.Add(-time.Minute) + if err := s.Append(Event{Version: SchemaVersion, ID: "unfinished", At: now, Job: &j}); err != nil { + t.Fatal(err) + } + if s.Cost("o/r", "v5", now.Add(-time.Hour), now).Complete { + t.Fatal("unfinished observation counted unknown future as zero") + } + if s.Cost("other", "v5", now.Add(-time.Hour), now).RunnerMinutes != nil { + t.Fatal("missing cohort reported zero") + } +} diff --git a/src/pkg/convergence/integration/types.go b/src/pkg/convergence/integration/types.go new file mode 100644 index 0000000000..2159dcaea5 --- /dev/null +++ b/src/pkg/convergence/integration/types.go @@ -0,0 +1,270 @@ +// Package integration records coordination observations. Its ledger is not a +// merge permission: callers must retain their live exact-head guards. +package integration + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/worksource" +) + +const SchemaVersion = 1 +const fullSHABytes = 20 + +type State string + +const ( + Unknown State = "unknown" + Waiting State = "waiting" + Running State = "running" + Passed State = "passed" + Rejected State = "rejected" + Superseded State = "superseded" +) + +type Member struct { + PR int `json:"pr"` + Head string `json:"head"` + Entry string `json:"entry"` + Position int `json:"position"` + BaseCommit string `json:"base_commit,omitempty"` + HeadCommit string `json:"head_commit,omitempty"` +} + +// Subject is immutable. Order, policy, producer, attempt and actual tested +// result are part of its identity, not mutable annotations on a PR record. +type Subject struct { + Version int `json:"version"` + Repo string `json:"repo"` + Base string `json:"base"` + Engine string `json:"engine"` + Authority string `json:"authority"` + Members []Member `json:"members"` + BaseSHA string `json:"base_sha"` + TestedSHA string `json:"tested_sha"` + TestedTree string `json:"tested_tree"` + Policy string `json:"policy"` + Producer string `json:"producer"` + Workflow string `json:"workflow"` + WorkflowSourceRepo string `json:"workflow_source_repo,omitempty"` + Attempt int `json:"attempt"` + Generation uint64 `json:"generation"` +} + +func fullSHA(s string) bool { + b, err := hex.DecodeString(s) + return err == nil && len(b) == fullSHABytes +} + +func (s Subject) Validate() error { + if s.Version != SchemaVersion || s.Repo == "" || s.Base == "" || + (s.Engine != "direct" && s.Engine != "native") || s.Authority == "" || + !fullSHA(s.BaseSHA) || !fullSHA(s.TestedSHA) || !fullSHA(s.TestedTree) || + s.Policy == "" || s.Producer == "" || s.Workflow == "" || s.Attempt < 1 || s.Generation == 0 || len(s.Members) == 0 { + return fmt.Errorf("incomplete exact integration subject") + } + seen := map[int]bool{} + for i, m := range s.Members { + if m.PR <= 0 || seen[m.PR] || !fullSHA(m.Head) || m.Position != i+1 || m.Entry == "" { + return fmt.Errorf("invalid ordered integration member %d", i) + } + seen[m.PR] = true + } + return nil +} + +func (s Subject) ID() string { + b, _ := json.Marshal(s) + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} + +type Evidence struct { + Subject Subject `json:"subject"` + State State `json:"state"` + ObservedState State `json:"observed_state,omitempty"` + ObservedAt time.Time `json:"observed_at"` + Source string `json:"source"` + RunID int64 `json:"run_id"` + SuiteID int64 `json:"suite_id"` + CheckIDs []int64 `json:"check_ids"` + InvalidatedBy string `json:"invalidated_by,omitempty"` + Reason string `json:"reason,omitempty"` +} + +type Prerequisite struct { + Key string `json:"key"` + // PlanNode is the existing approved DAG node, never an inferred text edge. + PlanNode string `json:"plan_node"` + Condition string `json:"condition,omitempty"` +} + +type Intent struct { + Version int `json:"version"` + Key string `json:"key"` + Repo string `json:"repo"` + Base string `json:"base"` + Generation uint64 `json:"generation"` + Plan string `json:"plan"` + PlanNode string `json:"plan_node"` + PlanRevision int `json:"plan_revision"` + Head string `json:"head,omitempty"` + Components []string `json:"components"` + Contracts []string `json:"contracts"` + Paths []string `json:"paths"` + Generated []string `json:"generated"` + ScopeKnown bool `json:"scope_known"` + Confidence string `json:"confidence"` + Provenance string `json:"provenance"` + Prerequisites []Prerequisite `json:"prerequisites"` + Relations []RelationAssessment `json:"relations,omitempty"` + ObservedAt time.Time `json:"observed_at"` +} + +type RelationAssessment struct { + Key string `json:"key"` + Kind string `json:"kind"` + Disposition string `json:"disposition"` + Provenance string `json:"provenance"` +} + +type Drift struct { + Key string `json:"key"` + Repo string `json:"repo"` + Head string `json:"head"` + IntentGeneration uint64 `json:"intent_generation"` + ObservedAt time.Time `json:"observed_at"` + Source string `json:"source"` + HeadVerified bool `json:"head_verified"` + Actual []string `json:"actual"` + OutsideAnticipated []string `json:"outside_anticipated"` +} + +func (i Intent) Validate() error { + ref, ok := worksource.ParseKey(i.Key) + if !ok || !strings.EqualFold(ref.Repo, i.Repo) || i.Version != SchemaVersion || + i.Base == "" || i.Generation == 0 || i.Plan == "" || i.PlanNode == "" || + i.Provenance == "" || i.ObservedAt.IsZero() || (i.Head != "" && !fullSHA(i.Head)) { + return fmt.Errorf("intent requires canonical work, plan, generation and provenance") + } + keys, nodes := map[string]bool{}, map[string]bool{} + for _, relation := range i.Relations { + if _, valid := worksource.ParseKey(relation.Key); !valid || relation.Key == i.Key || relation.Provenance == "" || + relation.Kind != "semantic" && relation.Kind != "file" || relation.Disposition != "confirmed" && relation.Disposition != "false-positive" && relation.Disposition != "unreviewed" { + return fmt.Errorf("invalid advisory relation assessment") + } + } + for _, p := range i.Prerequisites { + if _, ok := worksource.ParseKey(p.Key); !ok || p.PlanNode == "" || p.Key == i.Key { + return fmt.Errorf("invalid prerequisite %q", p.Key) + } + if p.Condition != "" && p.Condition != "developed" && p.Condition != "integrated" && p.Condition != "deployed" { + return fmt.Errorf("invalid prerequisite lifecycle %q", p.Condition) + } + if keys[p.Key] || nodes[p.PlanNode] { + return fmt.Errorf("duplicate prerequisite ownership") + } + keys[p.Key], nodes[p.PlanNode] = true, true + } + return nil +} + +type Reservation struct { + Key string `json:"key"` + Claim string `json:"claim"` + ClaimGeneration uint64 `json:"claim_generation"` + IntentGeneration uint64 `json:"intent_generation"` + ExpiresAt time.Time `json:"expires_at"` + RunnerMinutes float64 `json:"runner_minutes"` + Jobs int `json:"jobs"` +} + +type Job struct { + Repo string `json:"repo"` + Base string `json:"base"` + ID int64 `json:"id"` + RunID int64 `json:"run_id"` + Attempt int `json:"attempt"` + Head string `json:"head"` + Workflow string `json:"workflow"` + RunnerID int64 `json:"runner_id"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` + ObservedAt time.Time `json:"observed_at"` + // Candidates can share one physical job. Accounting counts the job once. + Candidates []string `json:"candidates"` +} + +func (j Job) Key() string { return fmt.Sprintf("%s/%d/%d", strings.ToLower(j.Repo), j.ID, j.Attempt) } + +type Coverage struct { + Repo string `json:"repo"` + Base string `json:"base"` + From time.Time `json:"from"` + Through time.Time `json:"through"` + Complete bool `json:"complete"` + Source string `json:"source"` + Reason string `json:"reason"` + MergedComplete bool `json:"merged_complete"` + MergeReason string `json:"merge_reason,omitempty"` +} + +type Event struct { + Version int `json:"version"` + ID string `json:"id"` + At time.Time `json:"at"` + Intent *Intent `json:"intent,omitempty"` + Evidence *Evidence `json:"evidence,omitempty"` + Job *Job `json:"job,omitempty"` + Coverage *Coverage `json:"coverage,omitempty"` + Reservation *Reservation `json:"reservation,omitempty"` + Release string `json:"release,omitempty"` + ReleaseClaim string `json:"release_claim,omitempty"` + ReleaseGeneration uint64 `json:"release_generation,omitempty"` + Invalidate string `json:"invalidate,omitempty"` + Reason string `json:"reason,omitempty"` + MemberChange *MemberChange `json:"member_change,omitempty"` + Integrated *IntegrationReceipt `json:"integrated,omitempty"` + GroupChange *GroupChange `json:"group_change,omitempty"` + PolicySet *PolicySet `json:"policy_set,omitempty"` + Drift *Drift `json:"drift,omitempty"` +} + +type GroupChange struct { + Repo string `json:"repo"` + Authority string `json:"authority"` + Head string `json:"head"` +} + +type PolicyBinding struct { + Repo string `json:"repo"` + Base string `json:"base"` + Fingerprint string `json:"fingerprint"` +} + +type PolicySet struct { + Active []PolicyBinding `json:"active"` +} + +type MemberChange struct { + Repo string `json:"repo"` + PR int `json:"pr"` + Head string `json:"head,omitempty"` +} + +type IntegrationReceipt struct { + Repo string `json:"repo"` + Base string `json:"base,omitempty"` + PR int `json:"pr"` + Head string `json:"head"` + MergeSHA string `json:"merge_sha"` + MergedAt time.Time `json:"merged_at"` + Source string `json:"source"` +} diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index b9187e2469..56e4a5ea15 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -251,6 +251,7 @@ func (s *Server) RegisterAPI(deps *Dependencies) { // (#4263 off/shadow/enforce rollout). The soak endpoint is the // fixed-commit telemetry read path (convergence_soak.go). s.mux.HandleFunc("GET /api/config/convergence", s.handleConvergenceConfigGet) + s.registerIntegrationRoutes() s.mux.HandleFunc("PUT /api/config/convergence", s.handleConvergenceConfigPut) s.mux.HandleFunc("GET /api/config/sla", s.handleSLAConfigGet) s.mux.HandleFunc("PUT /api/config/sla", s.handleSLAConfigPut) diff --git a/src/pkg/dashboard/api_github_webhook.go b/src/pkg/dashboard/api_github_webhook.go index 3653077486..2ed1d49a8f 100644 --- a/src/pkg/dashboard/api_github_webhook.go +++ b/src/pkg/dashboard/api_github_webhook.go @@ -105,6 +105,9 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { jsonResponse(w, map[string]any{"status": "ignored", "reason": "unknown repository", "event": event}) return } + if err := s.observeIntegrationWebhook(event, r.Header.Get("X-GitHub-Delivery"), body, receivedAt); err != nil { + s.webhookLogger().Error("integration observation not durable", "event", event, "error", err) + } // HandleWebhookEvent is the legacy writer: it invalidated the PR cache // and dirtied issues for the next cycle. The forgestate writer records // its own sample (WriterForgeState) when it applies the item patch. @@ -126,6 +129,9 @@ func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) { s.deps.WebhookWake(reason) resp["wake"] = reason } + if err := s.integrationFailureIdentity(event, body, res.CIFailure); err != nil { + s.webhookLogger().Warn("integration failure identity unavailable", "error", err) + } if reactions := s.reactToCIFailure(res); len(reactions) > 0 { resp["ci_failure"] = reactions } diff --git a/src/pkg/dashboard/api_github_webhook_ci_failure.go b/src/pkg/dashboard/api_github_webhook_ci_failure.go index acaadf002b..b37940cb85 100644 --- a/src/pkg/dashboard/api_github_webhook_ci_failure.go +++ b/src/pkg/dashboard/api_github_webhook_ci_failure.go @@ -44,6 +44,14 @@ func (s *Server) reactToCIFailure(res github.WebhookEventResult) []ciFailureReac return nil } store := github.CIFailureSignals() + var integrationContext *github.IntegrationFailureContext + memberHead := "" + if owner, context, head := s.integrationFailureOwner(res.Repo, f); owner != nil { + copyFailure := *f + copyFailure.PRs = []github.CIFailurePR{*owner} + f = ©Failure + integrationContext, memberHead = context, head + } if len(f.PRs) == 0 { store.RecordUnmatched() return nil @@ -67,6 +75,9 @@ func (s *Server) reactToCIFailure(res github.WebhookEventResult) []ciFailureReac Conclusion: f.Conclusion, DetailsURL: f.DetailsURL, Title: pr.Title, Author: pr.Author, HiveAuthored: pr.HiveAuthored, At: now, } + if integrationContext != nil { + sig.Integration, sig.HeadSHA = integrationContext, memberHead + } r := ciFailureReaction{Number: pr.Number, Check: f.Check, Route: "contributor"} if pr.HiveAuthored { r.Route = "agent" diff --git a/src/pkg/dashboard/api_governor.go b/src/pkg/dashboard/api_governor.go index 8f5bb2b1fc..8a832b7952 100644 --- a/src/pkg/dashboard/api_governor.go +++ b/src/pkg/dashboard/api_governor.go @@ -226,26 +226,27 @@ func (s *Server) handleGovernorConfigGet(w http.ResponseWriter, r *http.Request) "ttlSeconds": cfg.Governor.Sensing.TTLSeconds, "pullbackSeconds": cfg.Governor.Sensing.PullbackSeconds, }, - "logging": loggingSectionResponse(cfg), - "dashboard": dashboardSettingsResponse(cfg), - "backends": backendsPolicyResponse(cfg), - "litellm": litellmSectionResponse(&cfg.Governor.LiteLLM), - "trajectory": trajectorySectionResponse(&cfg.Governor), - "classifier": classifierSectionResponse(cfg), - "features": s.featuresSectionWithLinked(cfg), - "review": reviewSectionResponse(cfg), - "review_bots": reviewBotsSectionResponse(cfg), - "auto_merge": s.autoMergeSectionResponseWithBots(cfg), - "convergence": s.convergenceSectionResponse(cfg), - "sla": slaSectionResponse(cfg), - "advisory": advisorySectionResponse(cfg), - "project_observability": s.projectObservabilityResponse(cfg), - "replan": replanSectionResponse(cfg), - "question_autoclose": questionAutocloseSectionResponse(cfg), - "fleet_report": s.fleetReportSectionResponse(cfg), - "release_sentinel": releaseSentinelSectionResponse(cfg), - "work_source": workSourceSectionResponse(cfg), - "security": securitySectionResponse(cfg), + "logging": loggingSectionResponse(cfg), + "dashboard": dashboardSettingsResponse(cfg), + "backends": backendsPolicyResponse(cfg), + "litellm": litellmSectionResponse(&cfg.Governor.LiteLLM), + "trajectory": trajectorySectionResponse(&cfg.Governor), + "classifier": classifierSectionResponse(cfg), + "features": s.featuresSectionWithLinked(cfg), + "review": reviewSectionResponse(cfg), + "review_bots": reviewBotsSectionResponse(cfg), + "auto_merge": s.autoMergeSectionResponseWithBots(cfg), + "convergence": s.convergenceSectionResponse(cfg), + "integration_orchestration": s.integrationConfig(), + "sla": slaSectionResponse(cfg), + "advisory": advisorySectionResponse(cfg), + "project_observability": s.projectObservabilityResponse(cfg), + "replan": replanSectionResponse(cfg), + "question_autoclose": questionAutocloseSectionResponse(cfg), + "fleet_report": s.fleetReportSectionResponse(cfg), + "release_sentinel": releaseSentinelSectionResponse(cfg), + "work_source": workSourceSectionResponse(cfg), + "security": securitySectionResponse(cfg), "attribution": map[string]interface{}{ // Effective value (default ON when unset) — the UI renders the // switch from this, so an untouched hive shows it on. diff --git a/src/pkg/dashboard/claims_api.go b/src/pkg/dashboard/claims_api.go index 7d71ae4f4d..9bd5516adc 100644 --- a/src/pkg/dashboard/claims_api.go +++ b/src/pkg/dashboard/claims_api.go @@ -65,8 +65,24 @@ func (s *Server) InstallClaimHooks(base claims.Hooks) { if l == nil { return } + l.SetCommitCheck(s.integrationClaimCommit) prevTaken := base.OnTakenOver + prevClaimed, prevReleased := base.OnClaimed, base.OnReleased + base.OnClaimed = func(c claims.Claim, outcome claims.Outcome) { + s.integrationClaimObserved(c) + if prevClaimed != nil { + prevClaimed(c, outcome) + } + } + base.OnReleased = func(c claims.Claim, reason string) { + s.integrationRelease(c.Key(), c.Holder, uint64(c.ClaimedAt.UnixNano()), reason) + if prevReleased != nil { + prevReleased(c, reason) + } + } base.OnTakenOver = func(now, previous claims.Claim) { + s.integrationRelease(previous.Key(), previous.Holder, uint64(previous.ClaimedAt.UnixNano()), "claim takeover") + s.integrationClaimObserved(now) if prevTaken != nil { prevTaken(now, previous) } diff --git a/src/pkg/dashboard/contribute_admission.go b/src/pkg/dashboard/contribute_admission.go index be1c1010a9..39d5e561c6 100644 --- a/src/pkg/dashboard/contribute_admission.go +++ b/src/pkg/dashboard/contribute_admission.go @@ -175,6 +175,15 @@ const ( // it" signal. External work bypasses that GitHub-only observer and evaluates // the source-native dependency edges carried by its enumeration snapshot. func (h *ContributeWSHub) evaluateContributorNeutralAdmission(sweep *contributorAdmissionSweep, candidate contributorAdmissionCandidate) contributorAdmissionDecision { + if h != nil && h.server != nil { + ref := candidate.ref + if ref == (worksource.Ref{}) { + ref = worksource.Ref{Repo: candidate.repoFull, Number: candidate.number} + } + if d := h.server.integrationDispatchDecision(ref); !d.Allowed { + return contributorAdmissionDecision{reason: "integration_orchestration: " + d.Reason} + } + } if label, ok := h.contributeSkipLabel(candidate.labels); ok { if strings.EqualFold(strings.TrimSpace(label), blockedWorkflowLabel) { return blockedWorkflowAdmissionDecision(label) diff --git a/src/pkg/dashboard/contribute_leases.go b/src/pkg/dashboard/contribute_leases.go index c6006e72de..0dd5bc3292 100644 --- a/src/pkg/dashboard/contribute_leases.go +++ b/src/pkg/dashboard/contribute_leases.go @@ -257,7 +257,23 @@ func (h *ContributeWSHub) recordLeaseForKeyStage(identity, taskID, repo string, expiresAt: now.Add(leaseTTL), } // #5681: a lease the hub issued must outlive the process that issued it. + if h.server != nil { + if err := h.server.integrationLease(key, repo, identity, gen, now.Add(leaseTTL), now, false); err != nil { + if prev != nil { + h.leases[k] = prev + } else { + delete(h.leases, k) + } + for admissionKey, l := range removedAdmissions { + h.leases[admissionKey] = l + } + return err + } + } if err := h.saveLeasesLocked(); err != nil { + if h.server != nil { + h.server.integrationRelease(key, "contributor/"+identity, gen, "authoritative lease persistence failed") + } if prev != nil { h.leases[k] = prev } else { @@ -592,7 +608,8 @@ func (h *ContributeWSHub) renewLease(identity, taskID string, now time.Time) err var saveErr error if l := h.leaseForLocked(identity, taskID); l != nil { l.expiresAt = now.Add(leaseTTL) - renewed = l + copyLease := *l + renewed = ©Lease // #5681: persist the EXTENDED window. Without this a restart would restore // the window as it stood at assignment, so a task that had been progressing // for longer than leaseTTL — the exact case #4260 fixed in memory — would @@ -602,6 +619,11 @@ func (h *ContributeWSHub) renewLease(identity, taskID string, now time.Time) err } } h.leaseMu.Unlock() + if renewed != nil && h.server != nil { + if err := h.server.integrationLease(renewed.key, renewed.repo, identity, renewed.gen, renewed.expiresAt, now, true); err != nil { + h.logger.Warn("integration reservation renewal failed; existing lease retained", "task", taskID, "error", err) + } + } // #8380: the worker claim travels with the lease — renew it too, outside // leaseMu, so a task that outlives the 30m claim TTL stays visibly held. if renewed != nil && renewed.number > 0 { @@ -649,10 +671,12 @@ func (h *ContributeWSHub) revokeLease(identity, taskID string) { revoked := false // #8380: remember which items went so their worker claims go with them. var releasedKeys []string + var releasedLeases []taskLease if taskID != "" { if l, ok := h.leases[leaseKey(identity, taskID)]; ok { if l != nil { releasedKeys = append(releasedKeys, leaseClaimKey(l)) + releasedLeases = append(releasedLeases, *l) } delete(h.leases, leaseKey(identity, taskID)) revoked = true @@ -661,6 +685,7 @@ func (h *ContributeWSHub) revokeLease(identity, taskID string) { for k, l := range h.leases { if l != nil && l.identity == identity { releasedKeys = append(releasedKeys, leaseClaimKey(l)) + releasedLeases = append(releasedLeases, *l) delete(h.leases, k) revoked = true } @@ -680,6 +705,11 @@ func (h *ContributeWSHub) revokeLease(identity, taskID string) { } } h.leaseMu.Unlock() + for _, l := range releasedLeases { + if h.server != nil { + h.server.integrationRelease(leaseClaimKey(&l), "contributor/"+identity, l.gen, "lease revoked") + } + } for _, key := range releasedKeys { h.releaseClaimForLease(identity, key, "lease revoked") } diff --git a/src/pkg/dashboard/contribute_select.go b/src/pkg/dashboard/contribute_select.go index dbabd81741..08a86a1d4b 100644 --- a/src/pkg/dashboard/contribute_select.go +++ b/src/pkg/dashboard/contribute_select.go @@ -1107,6 +1107,11 @@ func (h *ContributeWSHub) selectTaskPass(c *ContributorConnection, skippedUnmint // set, this is a no-op and behaviour is identical to the previous first-eligible pick. ownFirst := make([]candidate, len(candidates)) copy(ownFirst, candidates) + integrationKeys := make([]string, 0, len(candidates)) + for _, c := range candidates { + integrationKeys = append(integrationKeys, c.ref.Key()) + } + integrationRanks := h.server.integrationAgingRanks(integrationKeys) sort.SliceStable(ownFirst, func(i, j int) bool { if ri, rj := orderRank(ownFirst[i]), orderRank(ownFirst[j]); ri != rj { return ri < rj // operator-pinned (lower rank) sorts ahead @@ -1120,6 +1125,9 @@ func (h *ContributeWSHub) selectTaskPass(c *ContributorConnection, skippedUnmint if ownFirst[i].recentFailures != ownFirst[j].recentFailures { return ownFirst[i].recentFailures < ownFirst[j].recentFailures // fewer failures first } + if len(integrationRanks) > 0 { + return integrationRanks[ownFirst[i].ref.Key()] < integrationRanks[ownFirst[j].ref.Key()] + } return false // equal keys → SliceStable preserves original scan order }) diff --git a/src/pkg/dashboard/integration_dispatch_advice.go b/src/pkg/dashboard/integration_dispatch_advice.go new file mode 100644 index 0000000000..b682b83234 --- /dev/null +++ b/src/pkg/dashboard/integration_dispatch_advice.go @@ -0,0 +1,62 @@ +package dashboard + +import ( + "sort" + "time" + + "github.com/hivecommons/hive/pkg/convergence/integration" + "github.com/hivecommons/hive/pkg/worksource" +) + +const integrationDispatchObservationLimit = 128 + +type integrationDispatchObservation struct { + Key string `json:"key"` + Repo string `json:"repo"` + Base string `json:"base"` + Mode string `json:"mode"` + ObservedAt time.Time `json:"observed_at"` + Decision integration.Decision `json:"decision"` +} + +func (s *Server) rememberIntegrationDispatch(ref worksource.Ref, base, mode string, decision integration.Decision) { + s.integration.mu.Lock() + defer s.integration.mu.Unlock() + if s.integration.advice == nil { + s.integration.advice = make(map[string]integrationDispatchObservation) + } + key := ref.Key() + if _, exists := s.integration.advice[key]; !exists && len(s.integration.advice) >= integrationDispatchObservationLimit { + oldest := "" + for candidate, observation := range s.integration.advice { + previous := s.integration.advice[oldest] + if oldest == "" || observation.ObservedAt.Before(previous.ObservedAt) || + observation.ObservedAt.Equal(previous.ObservedAt) && candidate < oldest { + oldest = candidate + } + } + delete(s.integration.advice, oldest) + } + decision.Hints = append([]string(nil), decision.Hints...) + if decision.Cost.RunnerMinutes != nil { + minutes := *decision.Cost.RunnerMinutes + decision.Cost.RunnerMinutes = &minutes + } + s.integration.advice[key] = integrationDispatchObservation{Key: key, Repo: ref.Repo, Base: base, Mode: mode, ObservedAt: time.Now(), Decision: decision} +} + +func (s *Server) integrationDispatchAdvice() []integrationDispatchObservation { + s.integration.mu.Lock() + defer s.integration.mu.Unlock() + out := make([]integrationDispatchObservation, 0, len(s.integration.advice)) + for _, observation := range s.integration.advice { + observation.Decision.Hints = append([]string(nil), observation.Decision.Hints...) + if observation.Decision.Cost.RunnerMinutes != nil { + minutes := *observation.Decision.Cost.RunnerMinutes + observation.Decision.Cost.RunnerMinutes = &minutes + } + out = append(out, observation) + } + sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key }) + return out +} diff --git a/src/pkg/dashboard/integration_dispatch_advice_test.go b/src/pkg/dashboard/integration_dispatch_advice_test.go new file mode 100644 index 0000000000..5b21ab6ef0 --- /dev/null +++ b/src/pkg/dashboard/integration_dispatch_advice_test.go @@ -0,0 +1,72 @@ +package dashboard + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/claims" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/convergence/integration" + ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/worksource" +) + +func TestIntegrationRealAdvisoryDispatchExposesLiveSemanticAdvice(t *testing.T) { + _, server, ledger := claimsHub(t) + server.integration.path = filepath.Join(t.TempDir(), "integration.jsonl") + server.deps.Config.IntegrationOrchestration = config.IntegrationOrchestrationConfig{Mode: config.IntegrationAdvisory, Scopes: []config.IntegrationScope{{Repo: "myorg/repo1", Base: "v5"}}} + store, err := server.integrationStore() + if err != nil { + t.Fatal(err) + } + for _, number := range []int{1, 2} { + intent := integration.Intent{Version: 1, Key: fmt.Sprintf("myorg/repo1#%d", number), Repo: "myorg/repo1", Base: "v5", Generation: 1, Plan: "unresolved", PlanNode: "owner-enrolled", ScopeKnown: true, Provenance: "owner", ObservedAt: time.Now(), Contracts: []string{"wire/v1"}, Paths: []string{fmt.Sprintf("%d.go", number)}} + if err := store.EnsureIntent(intent); err != nil { + t.Fatal(err) + } + } + if result, err := ledger.Claim(claims.Request{Repo: "myorg/repo1", Issue: 2, Holder: "agent", Kind: claims.KindAgent}); err != nil || result.Outcome != claims.OutcomeClaimed { + t.Fatalf("shadow blocked authoritative ownership: %+v %v", result, err) + } + raw := &ghpkg.ActionableResult{Issues: ghpkg.IssueResultFromItems([]ghpkg.Issue{{Repo: "myorg/repo1", Number: 1}})} + if server.IntegrationKickAdmission(raw) != raw { + t.Fatal("advisory changed dispatch") + } + found := false + for _, observation := range server.integrationDispatchAdvice() { + if observation.Key == "myorg/repo1#1" { + found = true + if len(observation.Decision.Hints) != 1 || !strings.Contains(observation.Decision.Hints[0], "semantic interaction") || !observation.Decision.Allowed { + t.Fatalf("actual dispatch advice lost: %+v", observation) + } + } + } + if !found || len(store.Snapshot(time.Now()).Reservations) != 0 { + t.Fatal("shadow advice absent or reserved work") + } +} + +func TestIntegrationDispatchAdviceIsBoundedAndCopiesHints(t *testing.T) { + server := orchestrationServer(t, config.IntegrationAdvisory) + ref := worksource.Ref{Repo: "o/r", Number: 1} + hints := []string{"semantic"} + server.rememberIntegrationDispatch(ref, "v5", config.IntegrationAdvisory, integration.Decision{Allowed: true, Hints: hints}) + hints[0] = "mutated" + snapshot := server.integrationDispatchAdvice() + if len(snapshot) != 1 || snapshot[0].Decision.Hints[0] != "semantic" { + t.Fatal("input advice aliases cache") + } + snapshot[0].Decision.Hints[0] = "mutated" + if server.integrationDispatchAdvice()[0].Decision.Hints[0] != "semantic" { + t.Fatal("output advice aliases cache") + } + for n := 2; n <= integrationDispatchObservationLimit+2; n++ { + server.rememberIntegrationDispatch(worksource.Ref{Repo: "o/r", Number: n}, "v5", config.IntegrationAdvisory, integration.Decision{Allowed: true}) + } + if len(server.integrationDispatchAdvice()) != integrationDispatchObservationLimit { + t.Fatal("unbounded dispatch diagnostics") + } +} diff --git a/src/pkg/dashboard/integration_merge_receipts.go b/src/pkg/dashboard/integration_merge_receipts.go new file mode 100644 index 0000000000..ff3fe2e20f --- /dev/null +++ b/src/pkg/dashboard/integration_merge_receipts.go @@ -0,0 +1,60 @@ +package dashboard + +import ( + "context" + "fmt" + "strings" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/convergence/integration" +) + +func (s *Server) reconcileIntegrationMerges(ctx context.Context, scope config.IntegrationScope, from, through time.Time, coverage *integration.Coverage) error { + parts := strings.SplitN(scope.Repo, "/", 2) + list, response, err := s.deps.GHClient.GoGitHub().PullRequests.List(ctx, parts[0], parts[1], &gh.PullRequestListOptions{ + State: "closed", Base: scope.Base, Sort: "updated", Direction: "desc", ListOptions: gh.ListOptions{PerPage: integrationReconcilePageSize}}) + if err != nil { + coverage.MergedComplete, coverage.MergeReason = false, "merged PR receipt listing unavailable" + s.webhookLogger().Warn("integration merged-PR accounting unavailable", "repo", scope.Repo, "error", err) + return nil + } + coverage.MergedComplete = response == nil || response.NextPage == 0 + for _, pr := range list { + if pr == nil { + coverage.MergedComplete = false + coverage.MergeReason = "missing closed PR observation" + continue + } + if !pr.GetUpdatedAt().Time.IsZero() && pr.GetUpdatedAt().Time.Before(from) { + coverage.MergedComplete = true + break + } + if pr.GetMergedAt().Time.IsZero() || pr.GetMergedAt().Time.Before(from) || pr.GetMergedAt().Time.After(through) { + continue + } + if pr.GetBase().GetRef() != scope.Base { + coverage.MergedComplete = false + coverage.MergeReason = "merged PR base attribution unavailable" + continue + } + receipt := integration.IntegrationReceipt{Repo: scope.Repo, Base: scope.Base, PR: pr.GetNumber(), Head: pr.GetHead().GetSHA(), MergeSHA: pr.GetMergeCommitSHA(), + MergedAt: pr.GetMergedAt().Time, Source: "github-merged-pr-rest-v1"} + store, storeErr := s.integrationStore() + if storeErr != nil { + return storeErr + } + id := fmt.Sprintf("merged-rest/%s/%d/%s", scope.Repo, pr.GetNumber(), receipt.MergeSHA) + if store.HasEvent(id) { + continue + } + if err := store.Append(integration.Event{Version: integration.SchemaVersion, ID: id, At: through, Integrated: &receipt}); err != nil { + return err + } + } + if !coverage.MergedComplete && coverage.MergeReason == "" { + coverage.MergeReason = "bounded merged-PR cohort overflow" + } + return nil +} diff --git a/src/pkg/dashboard/integration_native_reconcile_test.go b/src/pkg/dashboard/integration_native_reconcile_test.go new file mode 100644 index 0000000000..67f98ce93a --- /dev/null +++ b/src/pkg/dashboard/integration_native_reconcile_test.go @@ -0,0 +1,79 @@ +package dashboard + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/convergence/integration" + "github.com/hivecommons/hive/pkg/github" +) + +func TestIntegrationReconciliationRecoversMissedGroupAndBoundWorkflow(t *testing.T) { + now := time.Now().UTC() + base, tested, head, tree, blob := strings.Repeat("a", 40), strings.Repeat("b", 40), strings.Repeat("c", 40), strings.Repeat("d", 40), strings.Repeat("e", 40) + server := orchestrationServer(t, config.IntegrationAdvisory) + policy := config.IntegrationPolicy{Repo: "o/r", Base: "v5", Engine: "native", Workflow: ".github/workflows/group.yml", Blob: blob, AppID: 42, Jobs: []string{"policy"}} + server.deps.Config.IntegrationOrchestration.Policies = []config.IntegrationPolicy{policy} + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/repos/o/r/pulls": + _, _ = w.Write([]byte("[]")) + case r.URL.Path == "/repos/o/r/actions/runs": + _, _ = fmt.Fprintf(w, `{"total_count":1,"workflow_runs":[{"id":10,"workflow_id":12,"check_suite_id":20,"run_attempt":1,"event":"merge_group","status":"completed","conclusion":"success","head_sha":%q,"head_branch":"gh-readonly-queue/v5/pr-7","head_repository":{"full_name":"o/r"},"repository":{"full_name":"o/r"},"path":%q,"created_at":%q,"updated_at":%q}]}`, tested, policy.Workflow, now.Add(-15*time.Minute).Format(time.RFC3339), now.Format(time.RFC3339)) + case r.URL.Path == "/graphql": + _, _ = fmt.Fprintf(w, `{"data":{"repository":{"mergeQueue":{"entries":{"pageInfo":{"hasNextPage":false},"nodes":[{"id":"entry-7","position":1,"baseCommit":{"oid":%q},"headCommit":{"oid":%q},"pullRequest":{"number":7,"headRefOid":%q,"baseRefName":"v5"}}]}}}}}`, base, tested, head) + case strings.Contains(r.URL.Path, "/git/commits/"): + _, _ = fmt.Fprintf(w, `{"sha":%q,"tree":{"sha":%q}}`, tested, tree) + case strings.Contains(r.URL.Path, "/git/ref/"): + _, _ = fmt.Fprintf(w, `{"object":{"sha":%q}}`, tested) + case strings.Contains(r.URL.Path, "/contents/"): + _, _ = fmt.Fprintf(w, `{"type":"file","sha":%q}`, blob) + case strings.Contains(r.URL.Path, "/attempts/1/jobs"): + _, _ = fmt.Fprintf(w, `{"total_count":1,"jobs":[{"id":30,"run_id":10,"name":"policy","head_sha":%q,"runner_id":50,"status":"completed","conclusion":"success","check_run_url":"https://api.github.com/repos/o/r/check-runs/40","started_at":%q,"completed_at":%q}]}`, tested, now.Add(-10*time.Minute).Format(time.RFC3339), now.Add(-5*time.Minute).Format(time.RFC3339)) + case strings.HasSuffix(r.URL.Path, "/check-runs"): + _, _ = fmt.Fprintf(w, `{"check_runs":[{"id":40,"name":"policy","head_sha":%q,"status":"completed","conclusion":"success","app":{"id":42},"check_suite":{"id":20}}]}`, tested) + default: + t.Errorf("unexpected request %s", r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer api.Close() + server.deps.GHClient = github.NewClient("token", "o", []string{"r"}, nil, api.URL) + if err := server.ReconcileIntegration(context.Background()); err != nil { + t.Fatal(err) + } + store, err := server.integrationStore() + if err != nil { + t.Fatal(err) + } + receipts := store.ForTestedHead("o/r", tested) + if len(receipts) != 1 || receipts[0].State != integration.Passed || receipts[0].Subject.Policy != policy.Fingerprint() || + receipts[0].Subject.TestedTree != tree || len(receipts[0].Subject.Members) != 1 || receipts[0].Subject.Members[0].Head != head { + t.Fatalf("missed group did not recover exact bound evidence: %+v", receipts) + } + snapshot := store.Snapshot(now) + linked := false + if len(snapshot.Jobs) == 1 { + for _, candidate := range snapshot.Jobs[0].Candidates { + if candidate == receipts[0].Subject.ID() { + linked = true + } + } + } + if !linked { + t.Fatalf("physical job uncorrelated: %+v", snapshot.Jobs) + } + if err := store.SetPolicies(nil, now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + if got := store.ForTestedHead("o/r", tested)[0]; got.State != integration.Superseded { + t.Fatal("removed producer policy retained proof") + } +} diff --git a/src/pkg/dashboard/integration_observations.go b/src/pkg/dashboard/integration_observations.go new file mode 100644 index 0000000000..ea3d50e5b4 --- /dev/null +++ b/src/pkg/dashboard/integration_observations.go @@ -0,0 +1,471 @@ +package dashboard + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/convergence/integration" +) + +const ( + integrationReconcileInterval = 3 * time.Minute + integrationReconcileTimeout = 45 * time.Second + integrationReconcileMaxRuns = 8 + integrationReconcilePageSize = 100 + integrationReconcileMaxJobReads = 24 + integrationReconcileMaxPolicyReads = 4 +) + +type integrationWebhookPayload struct { + Action string `json:"action"` + Repository struct { + FullName string `json:"full_name"` + } `json:"repository"` + PullRequest *gh.PullRequest `json:"pull_request"` + WorkflowRun *gh.WorkflowRun `json:"workflow_run"` + MergeGroup struct { + BaseRef string `json:"base_ref"` + BaseSHA string `json:"base_sha"` + HeadRef string `json:"head_ref"` + HeadSHA string `json:"head_sha"` + } `json:"merge_group"` + WorkflowJob struct { + ID int64 `json:"id"` + RunID int64 `json:"run_id"` + RunAttempt int `json:"run_attempt"` + HeadSHA string `json:"head_sha"` + HeadBranch string `json:"head_branch"` + WorkflowName string `json:"workflow_name"` + RunnerID int64 `json:"runner_id"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` + } `json:"workflow_job"` +} + +func integrationBranchBase(branch string, scopes []config.IntegrationScope, repo string) string { + for _, scope := range scopes { + if !strings.EqualFold(scope.Repo, repo) { + continue + } + if branch == scope.Base || strings.HasPrefix(branch, "gh-readonly-queue/"+scope.Base+"/") { + return scope.Base + } + } + return "" +} + +func (s *Server) observeIntegrationWebhook(event, delivery string, body []byte, now time.Time) error { + cfg := s.integrationConfig() + if cfg.Mode == config.IntegrationOff { + return nil + } + if err := cfg.Validate(); err != nil { + return err + } + var p integrationWebhookPayload + if err := json.Unmarshal(body, &p); err != nil { + return err + } + repo := p.Repository.FullName + enrolled := false + for _, scope := range cfg.Scopes { + if strings.EqualFold(scope.Repo, repo) { + enrolled = true + } + } + if !enrolled { + return nil + } + store, err := s.integrationStore() + if err != nil { + return err + } + if event == "workflow_run" && p.WorkflowRun != nil { + if err := s.observeIntegrationRun(p.WorkflowRun, repo, delivery, now); err != nil { + return err + } + if p.WorkflowRun.GetEvent() == "merge_group" { + return s.observeIntegrationPolicyRun(context.Background(), p.WorkflowRun, repo, delivery, now) + } + return s.observeIntegrationDirectPolicyRun(context.Background(), p.WorkflowRun, repo, delivery, now) + } + if store.HasEvent("github/" + delivery) { + return nil + } + if event == "merge_group" { + g := p.MergeGroup + base := strings.TrimPrefix(g.BaseRef, "refs/heads/") + if !cfg.Allows(repo, base) { + return nil + } + if p.Action == "destroyed" { + return store.InvalidateGroup(repo, g.HeadRef, g.HeadSHA, delivery, "native group destroyed", now) + } + ctx, cancel := context.WithTimeout(context.Background(), integrationReconcileTimeout) + defer cancel() + if s.deps == nil || s.deps.GHClient == nil { + return fmt.Errorf("native group observation client unavailable") + } + group, readErr := s.deps.GHClient.ReadNativeIntegrationGroup(ctx, repo, base, g.BaseSHA, g.HeadRef, g.HeadSHA) + subject := integration.Subject{Version: integration.SchemaVersion, Repo: repo, Base: base, Engine: "native", + Authority: g.HeadRef, BaseSHA: group.BaseSHA, TestedSHA: g.HeadSHA, TestedTree: group.Tree, + Policy: "unverified-policy", Producer: "github-merge-group-webhook", Workflow: "merge_group", Attempt: 1, Generation: 1} + for _, m := range group.Members { + subject.Members = append(subject.Members, integration.Member{PR: m.PR, Head: m.Head, Entry: m.Entry, Position: m.Position, BaseCommit: m.BaseCommit, HeadCommit: m.HeadCommit}) + } + state := integration.Waiting + reason := "membership and tested tree observed; required policy/workflow/producer proof remains unknown" + if readErr != nil { + state = integration.Unknown + reason = "authoritative group observation unavailable: " + readErr.Error() + } + evidence := integration.Evidence{Subject: subject, State: state, ObservedAt: now, Source: "github-merge-group-v1", + Reason: reason} + return store.Append(integration.Event{Version: integration.SchemaVersion, ID: "github/" + delivery, At: now, Evidence: &evidence}) + } + + if event == "pull_request" || event == "pull_request_review" { + if pr := p.PullRequest; pr != nil { + reason := "member eligibility or review changed" + head := "" + if p.Action == "synchronize" { + head = pr.GetHead().GetSHA() + reason = "member head changed" + } + if event == "pull_request" && (p.Action == "opened" || p.Action == "reopened") { + return nil + } + // A merged predecessor is NOT member removal: GitHub can retain a + // documented prefix. Only an unmerged close invalidates it here. + if pr.GetMerged() { + receipt := integration.IntegrationReceipt{Repo: repo, PR: pr.GetNumber(), Head: pr.GetHead().GetSHA(), MergeSHA: pr.GetMergeCommitSHA(), + Base: pr.GetBase().GetRef(), MergedAt: pr.GetMergedAt().Time, Source: "github-signed-merged-webhook-v1"} + return store.Append(integration.Event{Version: integration.SchemaVersion, ID: "github/" + delivery, At: now, Integrated: &receipt}) + } + return store.InvalidateMember(repo, pr.GetNumber(), head, reason, delivery, now) + } + } + if event != "workflow_job" { + return nil + } + j := p.WorkflowJob + attempt := j.RunAttempt + // Older GitHub payloads omit run_attempt. Do not silently call it attempt 1: + // the bounded reconciliation resolves the actual attempt. + if attempt < 1 { + return fmt.Errorf("workflow job attempt unavailable; reconciliation required") + } + job := integration.Job{Repo: repo, Base: integrationBranchBase(j.HeadBranch, cfg.Scopes, repo), + ID: j.ID, RunID: j.RunID, Attempt: attempt, Head: j.HeadSHA, Workflow: j.WorkflowName, RunnerID: j.RunnerID, + Status: j.Status, Conclusion: j.Conclusion, StartedAt: j.StartedAt, CompletedAt: j.CompletedAt, ObservedAt: now} + return store.Append(integration.Event{Version: integration.SchemaVersion, ID: "github/" + delivery, At: now, Job: &job}) +} + +func (s *Server) handleIntegrationReconcile(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + if err := s.ReconcileIntegration(r.Context()); err != nil { + jsonError(w, err.Error(), http.StatusServiceUnavailable) + return + } + jsonResponse(w, map[string]string{"status": "observed; completeness is recorded per scope"}) +} + +func (s *Server) observeIntegrationRun(run *gh.WorkflowRun, repo, delivery string, now time.Time) error { + if run.GetEvent() != "pull_request" && run.GetEvent() != "merge_group" { + return nil + } + cfg := s.integrationConfig() + var pr *gh.PullRequest + base, engine := integrationBranchBase(run.GetHeadBranch(), cfg.Scopes, repo), "native" + if run.GetEvent() == "pull_request" { + if len(run.PullRequests) != 1 || run.PullRequests[0] == nil { + return nil + } + pr, engine = run.PullRequests[0], "direct" + base = pr.GetBase().GetRef() + } + if !cfg.Allows(repo, base) { + return nil + } + store, err := s.integrationStore() + if err != nil { + return err + } + subject := integration.Subject{Version: integration.SchemaVersion, Repo: repo, Base: base, Engine: engine, + Authority: fmt.Sprintf("workflow-run/%d", run.GetID()), + Policy: "unverified-policy", Producer: "github-actions-observation", Workflow: run.GetPath(), WorkflowSourceRepo: run.GetHeadRepository().GetFullName(), Attempt: run.GetRunAttempt(), Generation: 1} + if pr != nil { + if pr.GetNumber() > 0 { + subject.Authority = fmt.Sprintf("pull/%d", pr.GetNumber()) + } + subject.BaseSHA = pr.GetBase().GetSHA() + subject.Members = []integration.Member{{PR: pr.GetNumber(), Head: pr.GetHead().GetSHA(), Entry: fmt.Sprintf("pull/%d", pr.GetNumber()), Position: 1}} + } + if run.GetID() > 0 { + subject.Generation = uint64(run.GetID()) + } + evidence := integration.Evidence{Subject: subject, State: integration.Unknown, ObservedAt: now, Source: "github-workflow-run-v1", RunID: run.GetID(), SuiteID: run.GetCheckSuiteID(), + Reason: "author workflow observed; actual checkout commit/tree and combined-base policy evidence unavailable"} + if !store.HasEvent("github/" + delivery) { + if err := store.Append(integration.Event{Version: integration.SchemaVersion, ID: "github/" + delivery, At: now, Evidence: &evidence}); err != nil { + return err + } + } + return nil +} + +func (s *Server) observeIntegrationPolicyRun(parent context.Context, run *gh.WorkflowRun, repo, delivery string, now time.Time) error { + cfg := s.integrationConfig() + if run == nil || run.GetEvent() != "merge_group" { + return nil + } + if run.GetID() < 1 || run.GetRunAttempt() < 1 { + return fmt.Errorf("native workflow generation unavailable") + } + store, err := s.integrationStore() + if err != nil { + return err + } + if s.deps == nil || s.deps.GHClient == nil { + return fmt.Errorf("policy observation client unavailable") + } + ctx, cancel := context.WithTimeout(parent, integrationReconcileTimeout) + defer cancel() + for _, policy := range cfg.Policies { + if !strings.EqualFold(policy.Repo, repo) || policy.Engine != "native" || strings.Split(run.GetPath(), "@")[0] != policy.Workflow || + integrationBranchBase(run.GetHeadBranch(), cfg.Scopes, repo) != policy.Base { + continue + } + eventID := "policy/" + delivery + "/" + policy.Workflow + if store.HasEvent(eventID) { + continue + } + var subject *integration.Subject + for _, evidence := range store.ForTestedHead(repo, run.GetHeadSHA()) { + candidate := evidence.Subject + if candidate.Repo == repo && candidate.Base == policy.Base && candidate.Engine == "native" && + candidate.TestedSHA == run.GetHeadSHA() && evidence.State != integration.Superseded && len(candidate.Members) > 0 { + if subject != nil && (subject.Authority != candidate.Authority || subject.BaseSHA != candidate.BaseSHA) { + return fmt.Errorf("ambiguous native tested subject") + } + subject = &candidate + } + } + if subject == nil { + authority := "refs/heads/" + strings.TrimPrefix(run.GetHeadBranch(), "refs/heads/") + group, groupErr := s.deps.GHClient.ReadNativeIntegrationGroup(ctx, repo, policy.Base, "", authority, run.GetHeadSHA()) + if groupErr != nil { + return groupErr + } + recovered := integration.Subject{Version: integration.SchemaVersion, Repo: repo, Base: policy.Base, Engine: "native", Authority: authority, + BaseSHA: group.BaseSHA, TestedSHA: group.HeadSHA, TestedTree: group.Tree, Policy: "unverified-policy", Producer: "github-queue-read", Workflow: "merge_group", Attempt: 1, Generation: 1} + for _, member := range group.Members { + recovered.Members = append(recovered.Members, integration.Member{PR: member.PR, Head: member.Head, Entry: member.Entry, Position: member.Position, BaseCommit: member.BaseCommit, HeadCommit: member.HeadCommit}) + } + subject = &recovered + } + group, groupErr := s.deps.GHClient.ReadNativeIntegrationGroup(ctx, repo, policy.Base, "", subject.Authority, subject.TestedSHA) + if groupErr != nil { + return groupErr + } + if !integrationNativeMembershipCompatible(*subject, group) { + return fmt.Errorf("native tested membership/tree changed") + } + proof, proofErr := s.deps.GHClient.ReadIntegrationWorkflowProof(ctx, policy, run) + subject.Policy, subject.Producer, subject.Workflow = policy.Fingerprint(), fmt.Sprintf("github-app/%d", policy.AppID), fmt.Sprintf("%s@%d", policy.Workflow, run.GetWorkflowID()) + subject.WorkflowSourceRepo = run.GetHeadRepository().GetFullName() + subject.Attempt, subject.Generation = run.GetRunAttempt(), uint64(run.GetID()) + observedAt := run.GetUpdatedAt().Time + if observedAt.IsZero() { + observedAt = now + } + observation := integration.Evidence{Subject: *subject, State: integration.Unknown, Source: "github-actions-bound-v1", ObservedAt: observedAt, + RunID: proof.RunID, SuiteID: proof.SuiteID, CheckIDs: proof.CheckIDs, Reason: proof.Reason} + if proofErr != nil { + observation.Reason = proofErr.Error() + } else if proof.Passed { + observation.State = integration.Passed + } else if proof.Rejected { + observation.State = integration.Rejected + } else { + observation.State = integration.Running + } + if err := store.Append(integration.Event{Version: integration.SchemaVersion, ID: eventID, At: now, Evidence: &observation}); err != nil { + return err + } + if proofErr != nil { + return proofErr + } + } + return nil +} + +// ReconcileIntegration repairs missed job events with one bounded batch per +// interval, not a per-run poller. Overflow and ambiguous base attribution stay +// incomplete. It never approves/cancels/reruns jobs or modifies native entries. +func (s *Server) ReconcileIntegration(ctx context.Context) error { + if s == nil { + return nil + } + cfg := s.integrationConfig() + if cfg.Mode == config.IntegrationOff { + return nil + } + if err := cfg.Validate(); err != nil { + return err + } + s.integration.mu.Lock() + now := time.Now() + if now.Sub(s.integration.lastReconcile) < integrationReconcileInterval { + s.integration.mu.Unlock() + return nil + } + s.integration.lastReconcile = now + start := s.integration.reconcileScopeOffset % len(cfg.Scopes) + s.integration.reconcileScopeOffset = (start + 1) % len(cfg.Scopes) + s.integration.mu.Unlock() + if s.deps == nil || s.deps.GHClient == nil || s.deps.GHClient.GoGitHub() == nil { + return fmt.Errorf("GitHub workflow observations unavailable") + } + store, err := s.integrationStore() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, integrationReconcileTimeout) + defer cancel() + client := s.deps.GHClient.GoGitHub() + jobReads := 0 + policyReads := 0 + scopes := append(append([]config.IntegrationScope(nil), cfg.Scopes[start:]...), cfg.Scopes[:start]...) + for _, scope := range scopes { + parts := strings.SplitN(scope.Repo, "/", 2) + from := now.Add(-cfg.Window) + c := integration.Coverage{Repo: scope.Repo, Base: scope.Base, From: from, Through: now, Source: "github-actions-rest-v1", Complete: true} + // Creation time is not execution time: older completed runs can overlap + // this window. Inspect the bounded unfiltered cohort, or remain unknown. + runs, resp, readErr := client.Actions.ListRepositoryWorkflowRuns(ctx, parts[0], parts[1], &gh.ListWorkflowRunsOptions{ + ListOptions: gh.ListOptions{PerPage: integrationReconcilePageSize}}) + if readErr != nil || runs == nil { + c.Complete, c.Reason = false, "workflow listing unavailable" + } else { + if resp != nil && resp.NextPage != 0 { + c.Complete, c.Reason = false, "bounded reconciliation overflow" + } + observedRuns := 0 + for _, run := range runs.WorkflowRuns { + if run == nil { + c.Complete = false + continue + } + if run.GetStatus() == "completed" && !run.GetUpdatedAt().Time.IsZero() && run.GetUpdatedAt().Time.Before(from) { + continue + } + if observedRuns >= integrationReconcileMaxRuns || jobReads >= integrationReconcileMaxJobReads { + c.Complete, c.Reason = false, "global bounded job reconciliation budget exhausted" + continue + } + base := integrationBranchBase(run.GetHeadBranch(), cfg.Scopes, scope.Repo) + for _, pr := range run.PullRequests { + if pr != nil && pr.GetBase().GetRef() == scope.Base { + base = scope.Base + } + } + if base == "" { + c.Complete, c.Reason = false, "workflow base attribution unknown" + continue + } + if base != scope.Base { + continue + } + observedRuns++ + delivery := fmt.Sprintf("reconcile-run/%s/%d/%d/%d", scope.Repo, run.GetID(), run.GetRunAttempt(), run.GetUpdatedAt().UnixNano()) + if err := s.observeIntegrationRun(run, scope.Repo, delivery, now); err != nil { + return err + } + for _, policy := range cfg.Policies { + engine := "direct" + if run.GetEvent() == "merge_group" { + engine = "native" + } + if !strings.EqualFold(policy.Repo, scope.Repo) || policy.Base != scope.Base || policy.Engine != engine || strings.Split(run.GetPath(), "@")[0] != policy.Workflow { + continue + } + if policyReads >= integrationReconcileMaxPolicyReads { + s.webhookLogger().Warn("integration policy reconciliation budget exhausted; missing receipts remain unknown", "repo", scope.Repo, "run", run.GetID()) + break + } + policyReads++ + var policyErr error + if engine == "native" { + policyErr = s.observeIntegrationPolicyRun(ctx, run, scope.Repo, delivery, now) + } else { + policyErr = s.observeIntegrationDirectPolicyRun(ctx, run, scope.Repo, delivery, now) + } + if policyErr != nil { + s.webhookLogger().Warn("integration policy reconciliation unavailable", "repo", scope.Repo, "run", run.GetID(), "error", policyErr) + } + } + attempts := run.GetRunAttempt() + if attempts < 1 { + c.Complete, c.Reason = false, "run attempt unknown" + continue + } + for attempt := 1; attempt <= attempts; attempt++ { + if attempt > cfg.MaxAttempts || jobReads >= integrationReconcileMaxJobReads { + c.Complete, c.Reason = false, "attempt cohort exceeds bounded reconciliation" + break + } + jobReads++ + request, requestErr := client.NewRequest("GET", fmt.Sprintf("repos/%s/%s/actions/runs/%d/attempts/%d/jobs?per_page=%d", parts[0], parts[1], run.GetID(), attempt, integrationReconcilePageSize), nil) + if requestErr != nil { + return requestErr + } + jobs := new(gh.Jobs) + jobsResp, jobsErr := client.Do(ctx, request, jobs) + if jobsErr != nil || jobs == nil { + c.Complete, c.Reason = false, "workflow jobs unavailable" + continue + } + if jobsResp != nil && jobsResp.NextPage != 0 { + c.Complete, c.Reason = false, "job fanout exceeds reconciliation bound" + } + for _, j := range jobs.Jobs { + if j == nil { + c.Complete = false + continue + } + job := integration.Job{Repo: scope.Repo, Base: base, ID: j.GetID(), RunID: run.GetID(), Attempt: attempt, + Head: run.GetHeadSHA(), Workflow: run.GetPath(), RunnerID: j.GetRunnerID(), Status: j.GetStatus(), Conclusion: j.GetConclusion(), + StartedAt: j.GetStartedAt().Time, CompletedAt: j.GetCompletedAt().Time, ObservedAt: now} + e := integration.Event{Version: integration.SchemaVersion, ID: fmt.Sprintf("reconcile/%s/%d", job.Key(), now.UnixNano()), At: now, Job: &job} + if err := store.Append(e); err != nil { + return err + } + } + } + } + } + if readErr == nil { + if err := s.reconcileIntegrationMerges(ctx, scope, from, now, &c); err != nil { + return err + } + } + if err := store.Append(integration.Event{Version: integration.SchemaVersion, ID: fmt.Sprintf("coverage/%s/%d", scope.Repo+"#"+scope.Base, now.UnixNano()), At: now, Coverage: &c}); err != nil { + return err + } + if readErr != nil { + return fmt.Errorf("integration workflow listing: %w", readErr) + } + } + return nil +} diff --git a/src/pkg/dashboard/integration_orchestration.go b/src/pkg/dashboard/integration_orchestration.go new file mode 100644 index 0000000000..805ce6e377 --- /dev/null +++ b/src/pkg/dashboard/integration_orchestration.go @@ -0,0 +1,690 @@ +package dashboard + +import ( + "context" + "fmt" + "net/http" + "net/url" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/beads" + "github.com/hivecommons/hive/pkg/claims" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/convergence/integration" + ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/planning" + "github.com/hivecommons/hive/pkg/worksource" +) + +type integrationRuntime struct { + mu sync.Mutex + once sync.Once + store *integration.Store + err error + path string + lastReconcile time.Time + reconcileScopeOffset int + advice map[string]integrationDispatchObservation +} + +func (s *Server) IntegrationNativeAdmission(_ context.Context, repo string, pr *gh.PullRequest) (string, error) { + cfg := s.integrationConfig() + if pr == nil || !cfg.Allows(repo, pr.GetBase().GetRef()) { + return "", nil + } + store, err := s.integrationStore() + if err != nil { + if cfg.Mode == config.IntegrationEnforce { + return "workflow observations unavailable", err + } + return "", nil + } + key := "" + for _, intent := range store.Intents() { + if strings.EqualFold(intent.Repo, repo) && intent.Base == pr.GetBase().GetRef() && intent.Head == pr.GetHead().GetSHA() { + if key != "" && key != intent.Key { + if cfg.Mode == config.IntegrationEnforce { + return "native candidate has ambiguous canonical work intent", nil + } + return "", nil + } + key = intent.Key + } + } + in := s.integrationInput(key, repo, pr.GetBase().GetRef(), time.Now()) + in.Head = pr.GetHead().GetSHA() + if cfg.Mode == config.IntegrationEnforce && key != "" { + in.Reserve = true + in.Claim, in.ClaimGeneration, in.ClaimExpiresAt = "native-candidate/"+pr.GetNodeID()+"/"+in.Head, 1, in.Now.Add(cfg.LeaseTTL) + if ref, valid := worksource.ParseKey(key); valid { + if ledger := s.claimsLedger(); ledger != nil { + if claim, live := ledger.Lookup(ref.Repo, ref.Number); live { + in.Claim, in.ClaimGeneration, in.ClaimExpiresAt = claim.Holder, uint64(claim.ClaimedAt.UnixNano()), claim.ExpiresAt + } + } + if reservation, exists := store.Reservation(key, in.Now); exists { + in.Claim, in.ClaimGeneration, in.ClaimExpiresAt = reservation.Claim, reservation.ClaimGeneration, reservation.ExpiresAt + } + } + } + d, err := store.Decide(cfg, in) + if err != nil && cfg.Mode == config.IntegrationEnforce { + return "admission evaluation unavailable", err + } + if !d.Allowed { + return d.Reason, nil + } + return "", nil +} + +func (s *Server) IntegrationObserveFiles(repo, head string, files []string) { + cfg := s.integrationConfig() + if cfg.Mode == config.IntegrationOff { + return + } + enrolled := false + for _, scope := range cfg.Scopes { + if strings.EqualFold(scope.Repo, repo) { + enrolled = true + break + } + } + if !enrolled { + return + } + store, err := s.integrationStore() + if err == nil { + err = store.ObserveFiles(repo, head, files, time.Now()) + } + if err != nil { + s.webhookLogger().Warn("integration scope drift unavailable; overlap planner unaffected", "repo", repo, "error", err) + } +} + +func (s *Server) integrationConfig() config.IntegrationOrchestrationConfig { + if s == nil || s.deps == nil || s.deps.Config == nil { + return config.IntegrationOrchestrationConfig{}.Effective() + } + s.integration.mu.Lock() + defer s.integration.mu.Unlock() + return s.deps.Config.IntegrationOrchestration.Effective() +} + +func (s *Server) integrationStore() (*integration.Store, error) { + s.integration.once.Do(func() { + path := s.integration.path + if path == "" { + path = filepath.Join(filepath.Dir(taskLeasesFile), "integration-orchestration.jsonl") + } + s.integration.store, s.integration.err = integration.Open(path) + if s.integration.err == nil && s.integrationConfig().Mode != config.IntegrationOff { + s.integration.err = s.integration.store.SetPolicies(s.integrationConfig().Policies, time.Now()) + } + }) + return s.integration.store, s.integration.err +} + +func integrationBeadWorkKey(b *beads.Bead) string { + if b == nil { + return "" + } + key := b.Meta(planning.MetaIssueRepo) + "#" + b.Meta(planning.MetaIssueNumber) + if _, valid := worksource.ParseKey(key); valid { + return key + } + key = strings.TrimPrefix(b.ExternalRef, "gh-") + if _, valid := worksource.ParseKey(key); valid { + return key + } + return "" +} + +func (s *Server) integrationDispatchIntent(ref worksource.Ref, base string, now time.Time) integration.Intent { + intent := integration.Intent{Version: integration.SchemaVersion, Key: ref.Key(), Repo: ref.Repo, Base: base, Generation: 1, + Plan: "unresolved", PlanNode: ref.Key(), ScopeKnown: false, Confidence: "unknown", Provenance: "dispatch-enumerator", ObservedAt: now} + if s.deps == nil { + return intent + } + for _, store := range s.deps.BeadStores { + if store == nil { + continue + } + bead := store.FindByExternalRef("gh-" + ref.Key()) + if bead == nil { + bead = store.FindByExternalRef(ref.Key()) + } + if bead == nil || integrationBeadWorkKey(bead) != ref.Key() { + continue + } + parent := bead.Meta(planning.MetaParentEpic) + tree, found := planning.FindPlanTree(s.deps.BeadStores, parent) + if !found || !tree.Approved { + continue + } + if intent.Plan != "unresolved" && (intent.Plan != tree.EpicID || intent.PlanNode != bead.ID) { + intent.Plan, intent.PlanNode, intent.Confidence = "unresolved", ref.Key(), "ambiguous-approved-plan" + intent.Prerequisites = nil + return intent + } + intent.Plan, intent.PlanNode, intent.PlanRevision = tree.EpicID, bead.ID, tree.DesignRevision + intent.Confidence = "approved-dag-linked-scope-unknown" + for _, node := range tree.Children { + if node.ID != bead.ID { + continue + } + for _, dependencyID := range node.DependsOn { + dependency, err := store.Get(dependencyID) + if err != nil { + continue + } + if key := integrationBeadWorkKey(dependency); key != "" { + intent.Prerequisites = append(intent.Prerequisites, integration.Prerequisite{Key: key, PlanNode: dependencyID, Condition: "integrated"}) + } + } + } + } + return intent +} + +// integrationPlan verifies the existing approved DAG on every use. Neither a +// text similarity hint nor an intent's provenance can approve its own edges. +func (s *Server) integrationPlan(i integration.Intent) (bool, map[string]integration.Dependency) { + deps := map[string]integration.Dependency{} + if s.deps == nil { + return false, deps + } + tree, ok := planning.FindPlanTree(s.deps.BeadStores, i.Plan) + if !ok || !tree.Approved || tree.DesignRevision != i.PlanRevision { + return false, deps + } + var node *planning.PlanChild + nodes := map[string]planning.PlanChild{} + for n := range tree.Children { + child := tree.Children[n] + nodes[child.ID] = child + if child.ID == i.PlanNode { + node = &child + } + } + if node == nil { + return false, deps + } + // Published child beads retain canonical issue metadata. A missing binding + // is unknown, not permission to attach someone else's approved plan. + bound := false + for _, store := range s.deps.BeadStores { + if store == nil { + continue + } + b, err := store.Get(i.PlanNode) + if err == nil && integrationBeadWorkKey(b) == i.Key { + bound = true + break + } + } + if !bound { + return false, deps + } + edges := map[string]bool{} + for _, id := range node.DependsOn { + edges[id] = true + } + if len(edges) != len(i.Prerequisites) { + return false, deps + } + for _, p := range i.Prerequisites { + child, found := nodes[p.PlanNode] + if !found || !edges[p.PlanNode] { + continue + } + keyBound := false + for _, store := range s.deps.BeadStores { + if store == nil { + continue + } + b, err := store.Get(p.PlanNode) + if err == nil && integrationBeadWorkKey(b) == p.Key { + keyBound = true + break + } + } + if keyBound { + dependency := integration.Dependency{} + switch p.Condition { + case "developed": + dependency = integration.Dependency{Known: true, Satisfied: child.Status == beads.StatusDone || child.Status == beads.StatusClosed} + case "", "integrated": + link, err := url.Parse(child.PRURL) + if err == nil && link.Host == "github.com" { + parts := strings.Split(strings.Trim(link.Path, "/"), "/") + if len(parts) == 4 && parts[2] == "pull" { + number, parseErr := strconv.Atoi(parts[3]) + ledger, ledgerErr := s.integrationStore() + if parseErr == nil && ledgerErr == nil { + _, merged := ledger.Integrated(parts[0]+"/"+parts[1], number) + dependency = integration.Dependency{Known: merged, Satisfied: merged} + } + } + } + case "deployed": // No deployment producer is configured: closure is not deployment. + } + deps[p.Key] = dependency + } + } + return true, deps +} + +func (s *Server) integrationInput(key, repo, base string, now time.Time) integration.Input { + in := integration.Input{Key: key, Repo: repo, Base: base, Now: now} + store, err := s.integrationStore() + if err != nil { + return in + } + if i, exists := store.Intent(key); exists { + in.IntentGeneration = i.Generation + in.PlanApproved, in.Dependencies = s.integrationPlan(i) + in.PlanRevision = i.PlanRevision + if ref, valid := worksource.ParseKey(key); valid { + actual := s.integrationDispatchIntent(ref, base, now) + in.PlanRequired = actual.Plan != "unresolved" + if in.PlanRequired && (actual.Plan != i.Plan || actual.PlanNode != i.PlanNode || actual.PlanRevision != i.PlanRevision) { + in.PlanApproved = false + } + } + } + return in +} + +// integrationDispatchDecision is shared by the internal and contributor lanes. +// The scope's base is explicit because issue enumerations do not carry a base. +// An ambiguous multi-base issue is not silently assigned to either cohort. +func (s *Server) integrationDispatchDecision(ref worksource.Ref) (out integration.Decision) { + cfg := s.integrationConfig() + out = integration.Decision{Allowed: true} + if cfg.Mode == config.IntegrationOff { + return out + } + bases := []string{} + for _, scope := range cfg.Scopes { + if !strings.EqualFold(scope.Repo, ref.Repo) { + continue + } + bases = append(bases, scope.Base) + } + if len(bases) == 0 { + return out + } + base := "" + if len(bases) == 1 { + base = bases[0] + } + defer func() { s.rememberIntegrationDispatch(ref, base, cfg.Mode, out) }() + store, err := s.integrationStore() + if err != nil { + out.Reason = "integration ledger unavailable: " + err.Error() + out.WouldBlock, out.Allowed = true, cfg.Mode != config.IntegrationEnforce + return out + } + if len(bases) > 1 { + intent, known := store.Intent(ref.Key()) + if !known || !cfg.Allows(ref.Repo, intent.Base) { + out.Reason = "issue base unknown across multiple enrolled bases" + out.WouldBlock, out.Allowed = true, cfg.Mode != config.IntegrationEnforce + return out + } + base = intent.Base + } + if ref.Key() != "" { + now := time.Now() + intent := s.integrationDispatchIntent(ref, base, now) + if err := store.EnsureIntent(intent); err != nil { + return integration.Decision{Allowed: cfg.Mode != config.IntegrationEnforce, WouldBlock: true, Reason: "intent persistence unavailable: " + err.Error()} + } + } + in := s.integrationInput(ref.Key(), ref.Repo, base, time.Now()) + if ledger := s.claimsLedger(); ledger != nil { + for _, claim := range ledger.List() { + if strings.EqualFold(claim.Repo, ref.Repo) { + in.LiveKeys = append(in.LiveKeys, claim.Key()) + } + } + } + out, err = store.Decide(cfg, in) + if err != nil { + out = integration.Decision{Allowed: cfg.Mode != config.IntegrationEnforce, WouldBlock: true, Reason: err.Error()} + } + return out +} + +func (s *Server) integrationAgingRanks(keys []string) map[string]int { + ranks := map[string]int{} + if s.integrationConfig().Mode != config.IntegrationEnforce { + return ranks + } + store, err := s.integrationStore() + if err != nil { + return ranks + } + for rank, key := range store.Order(keys) { + ranks[key] = rank + } + return ranks +} + +// IntegrationKickAdmission is independent of convergence.mode. OFF returns the +// original pointer; advisory never filters or changes scheduler ordering. +func (s *Server) IntegrationKickAdmission(raw *ghpkg.ActionableResult) *ghpkg.ActionableResult { + if s == nil || raw == nil || s.integrationConfig().Mode == config.IntegrationOff { + return raw + } + out := *raw + items := make([]ghpkg.Issue, 0, len(raw.Issues.Items)) + for _, issue := range raw.Issues.Items { + ref := worksource.Ref{Repo: issue.Repo, Number: issue.Number, SourceType: issue.SourceType, ExternalID: issue.ExternalID} + d := s.integrationDispatchDecision(ref) + if d.Allowed { + items = append(items, issue) + } + if d.WouldBlock && s.logger != nil { + s.logger.Info("integration authoring admission", "lane", "internal", "work", ref.Key(), "enforced", !d.Allowed, "reason", d.Reason) + } + } + if s.integrationConfig().Mode != config.IntegrationEnforce { + return raw + } + keys := make([]string, 0, len(items)) + for _, i := range items { + keys = append(keys, worksource.Ref{Repo: i.Repo, Number: i.Number, SourceType: i.SourceType, ExternalID: i.ExternalID}.Key()) + } + ranks := s.integrationAgingRanks(keys) + buckets := map[string][]ghpkg.Issue{} + for _, i := range items { + buckets[i.Priority] = append(buckets[i.Priority], i) + } + for priority, bucket := range buckets { + sort.SliceStable(bucket, func(a, b int) bool { + ka := worksource.Ref{Repo: bucket[a].Repo, Number: bucket[a].Number, SourceType: bucket[a].SourceType, ExternalID: bucket[a].ExternalID}.Key() + kb := worksource.Ref{Repo: bucket[b].Repo, Number: bucket[b].Number, SourceType: bucket[b].SourceType, ExternalID: bucket[b].ExternalID}.Key() + return ranks[ka] < ranks[kb] + }) + buckets[priority] = bucket + } + for n, i := range items { + items[n] = buckets[i.Priority][0] + buckets[i.Priority] = buckets[i.Priority][1:] + } + out.Issues = ghpkg.IssueResultFromItems(items) + return &out +} + +// IntegrationClaimBlockReason runs in the existing claims admission callback, +// before claiming. Existing ownership remains the sole dedup authority. +func (s *Server) IntegrationClaimBlockReason(repo string, number int) string { + d := s.integrationDispatchDecision(worksource.Ref{Repo: repo, Number: number}) + if !d.Allowed { + return d.Reason + } + return "" +} + +func (s *Server) integrationClaimObserved(c claims.Claim) { + if c.Kind == claims.KindContributor { + return + } // contributor reservations bind the server-issued lease below + cfg := s.integrationConfig() + if cfg.Mode != config.IntegrationEnforce { + return + } + store, err := s.integrationStore() + if err != nil { + s.webhookLogger().Error("integration claim observation failed", "error", err) + return + } + for _, scope := range cfg.Scopes { + if !strings.EqualFold(scope.Repo, c.Repo) { + continue + } + if intent, known := store.Intent(c.Key()); known && intent.Base != scope.Base { + continue + } + in := s.integrationInput(c.Key(), c.Repo, scope.Base, time.Now()) + in.Claim, in.ClaimGeneration, in.ClaimExpiresAt, in.Reserve = c.Holder, uint64(c.ClaimedAt.UnixNano()), c.ExpiresAt, true + in.RenewOnly = true + d, err := store.Decide(cfg, in) + if err != nil || !d.Allowed { + s.webhookLogger().Warn("integration reservation not acquired; existing claim retained", "key", c.Key(), "reason", d.Reason, "error", err) + } + } +} + +func (s *Server) integrationClaimCommit(c claims.Claim) (string, error) { + cfg := s.integrationConfig() + if cfg.Mode != config.IntegrationEnforce || c.Kind == claims.KindContributor { + return "", nil + } + for _, scope := range cfg.Scopes { + if !strings.EqualFold(scope.Repo, c.Repo) { + continue + } + store, err := s.integrationStore() + if err != nil { + return "integration ledger unavailable", err + } + if intent, known := store.Intent(c.Key()); known && intent.Base != scope.Base { + continue + } + in := s.integrationInput(c.Key(), c.Repo, scope.Base, time.Now()) + in.Claim, in.ClaimGeneration = c.Holder, uint64(c.ClaimedAt.UnixNano()) + in.ClaimExpiresAt, in.Reserve = c.ExpiresAt, true + d, err := store.Decide(cfg, in) + if err != nil || !d.Allowed { + return d.Reason, err + } + } + return "", nil +} + +func (s *Server) integrationLease(key, repo, identity string, gen uint64, expires, now time.Time, renewal bool) error { + cfg := s.integrationConfig() + if cfg.Mode != config.IntegrationEnforce { + return nil + } + for _, scope := range cfg.Scopes { + if !strings.EqualFold(scope.Repo, repo) { + continue + } + store, err := s.integrationStore() + if err != nil { + return err + } + if intent, known := store.Intent(key); known && intent.Base != scope.Base { + continue + } + in := s.integrationInput(key, repo, scope.Base, now) + in.Reserve, in.Claim, in.ClaimGeneration, in.ClaimExpiresAt = true, "contributor/"+identity, gen, expires + in.RenewOnly = renewal + d, err := store.Decide(cfg, in) + if err != nil { + return err + } + if !d.Allowed { + return fmt.Errorf("integration admission: %s", d.Reason) + } + } + return nil +} + +func (s *Server) integrationRelease(key, claim string, generation uint64, reason string) { + if s.integrationConfig().Mode == config.IntegrationOff { + return + } + store, err := s.integrationStore() + if err == nil { + err = store.Append(integration.Event{Version: integration.SchemaVersion, ID: fmt.Sprintf("release/%s/%d", key, time.Now().UnixNano()), At: time.Now(), Release: key, ReleaseClaim: claim, ReleaseGeneration: generation, Reason: reason}) + } + if err != nil { + s.webhookLogger().Error("integration reservation release failed", "work", key, "error", err) + } +} + +func (s *Server) registerIntegrationRoutes() { + s.mux.HandleFunc("GET /api/integration-orchestration", s.handleIntegrationGet) + s.mux.HandleFunc("PUT /api/config/integration-orchestration", s.handleIntegrationConfigPut) + s.mux.HandleFunc("POST /api/integration-orchestration/intents", s.handleIntegrationIntent) + s.mux.HandleFunc("POST /api/integration-orchestration/reconcile", s.handleIntegrationReconcile) +} + +func (s *Server) handleIntegrationGet(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + cfg := s.integrationConfig() + resp := map[string]any{"config": cfg, "schema_version": integration.SchemaVersion, "runtime_commit": versionHash, + "source": "github-observations-v1", "source_version": 1, "enabled": cfg.Mode != config.IntegrationOff, + "effective_writer": "unknown in this observer; the existing live preparation/native authority guard remains authoritative", + "native_group_evidence": "unknown until complete authoritative group, tested tree and producer observations", + "author_review_evidence": "separate; never substituted by integration evidence", + "direct_producer_contract": "hive-integration-subject-v1-/subject.json from the pinned workflow; no producer is installed or activated by this feature", + "proof_semantics": "historical one-workflow receipts only; never aggregate merge eligibility", + "preparation_source": "/api/stage-tracker.base_preparation; ownership/backlog, not ready train or native queue seats", + "reconciliation": map[string]any{"interval_seconds": integrationReconcileInterval.Seconds(), "max_runs_per_scope": integrationReconcileMaxRuns, "max_job_reads": integrationReconcileMaxJobReads, "max_policy_reads": integrationReconcileMaxPolicyReads, "cohort": "bounded repository listing; overflow is unknown, not zero"}, + "controls": "Hive authoring and new Hive-owned enqueue admission only; no native queue jumping or GitHub build-window control"} + scopes := make([]map[string]any, 0, len(cfg.Scopes)) + for _, scope := range cfg.Scopes { + status := map[string]any{"repo": scope.Repo, "base": scope.Base, "live_authority_state": "unknown; configured is not effective"} + if s.deps != nil && s.deps.Config != nil { + status["preparation_configured"] = s.deps.Config.AutoMerge.RequireCurrentBaseFor(scope.Repo) + status["native_configured"] = s.deps.Config.AutoMerge.NativeMergeQueue.EnabledFor(scope.Repo) + } + scopes = append(scopes, status) + } + resp["configured_authorities"] = scopes + bindings := make([]map[string]any, 0, len(cfg.Policies)) + for _, policy := range cfg.Policies { + bindings = append(bindings, map[string]any{"policy": policy, "fingerprint": policy.Fingerprint()}) + } + resp["policy_bindings"] = bindings + resp["dispatch_advice"] = s.integrationDispatchAdvice() + resp["dispatch_advice_contract"] = "last actual internal/contributor evaluations; bounded boot-local diagnostics, not durable merge proof" + resp["aging_policy"] = "oldest intent within existing priority/own-work bands; per-repository/base active and budget bounds; no native reordering" + store, err := s.integrationStore() + if err != nil { + resp["state"] = "unknown" + resp["error"] = err.Error() + } else { + now := time.Now() + snapshot := store.Snapshot(now) + resp["ledger"] = snapshot + reports := make([]map[string]any, 0, len(cfg.Scopes)) + for _, scope := range cfg.Scopes { + through := time.Time{} + for _, coverage := range snapshot.Coverage { + if strings.EqualFold(coverage.Repo, scope.Repo) && coverage.Base == scope.Base { + through = coverage.Through + } + } + stale := through.IsZero() || through.After(now) || now.Sub(through) > cfg.MaxObservationAge + if through.IsZero() { + through = now + } + reports = append(reports, map[string]any{"stale": stale, "report": store.CostReport(scope.Repo, scope.Base, through.Add(-cfg.Window), through)}) + } + resp["ci_accounting"] = reports + if source := s.hotspotSourceRef(); source != nil { + resp["existing_hotspots"] = source.HotspotsJSON("") + } + } + jsonResponse(w, resp) +} + +func (s *Server) handleIntegrationConfigPut(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + var next config.IntegrationOrchestrationConfig + if err := decodeBody(r, &next); err != nil { + jsonError(w, "invalid configuration", http.StatusBadRequest) + return + } + if err := next.Validate(); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + if s.deps == nil || s.deps.Config == nil { + jsonError(w, "configuration unavailable", http.StatusServiceUnavailable) + return + } + s.integration.mu.Lock() + defer s.integration.mu.Unlock() + previous := s.deps.Config.IntegrationOrchestration + // Reset only our reservations on every configuration transition, including + // allowlist/bounds drift. Disabling must remain possible with a failed disk. + store, ledgerErr := s.integrationStore() + if ledgerErr == nil { + ledgerErr = store.Append(integration.Event{Version: integration.SchemaVersion, ID: fmt.Sprintf("rollback/%d", time.Now().UnixNano()), At: time.Now(), Release: "*", Reason: "operator changed orchestration configuration"}) + } + if ledgerErr == nil { + ledgerErr = store.SetPolicies(next.Policies, time.Now()) + } + if ledgerErr != nil && next.Effective().Mode == config.IntegrationEnforce { + jsonError(w, ledgerErr.Error(), http.StatusServiceUnavailable) + return + } + s.deps.Config.IntegrationOrchestration = next + if err := s.saveConfig(); err != nil { + s.deps.Config.IntegrationOrchestration = previous + if ledgerErr == nil { + if restoreErr := store.SetPolicies(previous.Policies, time.Now()); restoreErr != nil { + s.webhookLogger().Error("integration producer policy rollback unavailable", "error", restoreErr) + } + } + jsonError(w, "integration configuration persistence failed", http.StatusInternalServerError) + return + } + s.auditFromRequest(r, "config_integration_orchestration", auditDetail("mode", next.Effective().Mode), "") + if ledgerErr != nil { + s.webhookLogger().Error("integration disabled; reservation cleanup unavailable, re-enablement fenced", "error", ledgerErr) + } + jsonResponse(w, next.Effective()) +} + +func (s *Server) handleIntegrationIntent(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + var i integration.Intent + if err := decodeBody(r, &i); err != nil { + jsonError(w, "invalid intent", http.StatusBadRequest) + return + } + if err := i.Validate(); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + if !s.integrationConfig().Allows(i.Repo, i.Base) { + jsonError(w, "intent outside enrolled scope", http.StatusBadRequest) + return + } + if len(i.Prerequisites) > 0 { + approved, deps := s.integrationPlan(i) + if !approved || len(deps) != len(i.Prerequisites) { + jsonError(w, "prerequisites must match bound nodes in the current approved plan DAG", http.StatusConflict) + return + } + } + store, err := s.integrationStore() + if err == nil { + err = store.Append(integration.Event{Version: integration.SchemaVersion, ID: fmt.Sprintf("intent/%s/%d", i.Key, i.Generation), At: i.ObservedAt, Intent: &i}) + } + if err != nil { + jsonError(w, err.Error(), http.StatusConflict) + return + } + jsonResponse(w, i) +} diff --git a/src/pkg/dashboard/integration_orchestration_test.go b/src/pkg/dashboard/integration_orchestration_test.go new file mode 100644 index 0000000000..8547f9a72c --- /dev/null +++ b/src/pkg/dashboard/integration_orchestration_test.go @@ -0,0 +1,237 @@ +package dashboard + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/claims" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/convergence/integration" + ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/worksource" +) + +func orchestrationServer(t *testing.T, mode string) *Server { + t.Helper() + s := newTestServer() + s.deps = &Dependencies{Config: &config.Config{IntegrationOrchestration: config.IntegrationOrchestrationConfig{ + Mode: mode, Scopes: []config.IntegrationScope{{Repo: "o/r", Base: "v5"}}}}} + s.integration.path = filepath.Join(t.TempDir(), "integration.jsonl") + return s +} + +func TestIntegrationBothDispatchLanesAndOffIdentity(t *testing.T) { + raw := &ghpkg.ActionableResult{Issues: ghpkg.IssueResultFromItems([]ghpkg.Issue{{Repo: "o/r", Number: 1}, {Repo: "elsewhere/r", Number: 2}})} + for _, mode := range []string{config.IntegrationOff, config.IntegrationAdvisory, config.IntegrationEnforce} { + t.Run(mode, func(t *testing.T) { + s := orchestrationServer(t, mode) + projected := s.IntegrationKickAdmission(raw) + if mode == config.IntegrationOff && projected != raw { + t.Fatal("OFF changed existing pointer/behavior") + } + want := 2 + if mode == config.IntegrationEnforce { + want = 1 + } + if projected.Issues.Count != want || len(raw.Issues.Items) != 2 { + t.Fatalf("internal dispatch mode=%s got=%+v", mode, projected.Issues) + } + h := &ContributeWSHub{server: s} + ref := worksource.Ref{SourceType: "linear", Repo: "o/r", ExternalID: "ENG-1"} + decision := h.evaluateContributorNeutralAdmission(nil, contributorAdmissionCandidate{repoFull: "o/r", ref: ref}) + if decision.admitted != (mode != config.IntegrationEnforce) { + t.Fatalf("contributor mode=%s got=%+v", mode, decision) + } + if mode != config.IntegrationOff { + store, err := s.integrationStore() + if err != nil { + t.Fatal(err) + } + if len(store.Snapshot(time.Now()).Intents) != 2 { + t.Fatal("actual dispatch did not persist canonical intents") + } + } + }) + } +} + +func TestIntegrationAPIRollbackAndUnknownTelemetry(t *testing.T) { + s := orchestrationServer(t, config.IntegrationAdvisory) + read := httptest.NewRecorder() + s.handleIntegrationGet(read, convergenceGetReq(true)) + if read.Code != http.StatusOK || !strings.Contains(read.Body.String(), "unknown") || !strings.Contains(read.Body.String(), "runtime_commit") { + t.Fatalf("%d %s", read.Code, read.Body) + } + req := httptest.NewRequest(http.MethodPut, "/api/config/integration-orchestration", strings.NewReader(`{"mode":"enforce"}`)) + req.Header.Set("X-Hive-Role", "owner") + req.Header.Set(ownerRoleVerifiedHeader, "true") + w := httptest.NewRecorder() + s.handleIntegrationConfigPut(w, req) + if w.Code != http.StatusBadRequest || s.integrationConfig().Mode != config.IntegrationAdvisory { + t.Fatal("invalid configuration mutated active mode") + } + body, _ := json.Marshal(config.IntegrationOrchestrationConfig{Mode: config.IntegrationOff}) + req = httptest.NewRequest(http.MethodPut, "/api/config/integration-orchestration", bytes.NewReader(body)) + req.Header.Set("X-Hive-Role", "owner") + req.Header.Set(ownerRoleVerifiedHeader, "true") + w = httptest.NewRecorder() + s.handleIntegrationConfigPut(w, req) + if w.Code != http.StatusOK || s.integrationConfig().Mode != config.IntegrationOff { + t.Fatalf("%d %s", w.Code, w.Body) + } +} + +func TestIntegrationWebhookReplayJobCostAndMissingAttempt(t *testing.T) { + s := orchestrationServer(t, config.IntegrationAdvisory) + now := time.Now().UTC() + body := []byte(`{"repository":{"full_name":"o/r"},"workflow_job":{"id":10,"run_id":20,"run_attempt":1,"head_sha":"` + strings.Repeat("a", 40) + `","head_branch":"v5","runner_id":30,"status":"completed","conclusion":"cancelled","started_at":"2026-10-30T20:00:00Z","completed_at":"2026-10-30T20:10:00Z"}}`) + if err := s.observeIntegrationWebhook("workflow_job", "delivery", body, now); err != nil { + t.Fatal(err) + } + if err := s.observeIntegrationWebhook("workflow_job", "delivery", body, now.Add(time.Hour)); err != nil { + t.Fatal("delivery replay changed observation", err) + } + store, err := s.integrationStore() + if err != nil { + t.Fatal(err) + } + snapshot := store.Snapshot(now) + if len(snapshot.Jobs) != 1 || snapshot.Jobs[0].Conclusion != "cancelled" { + t.Fatalf("%+v", snapshot) + } + bad := bytes.Replace(body, []byte(`"run_attempt":1`), []byte(`"run_attempt":0`), 1) + if s.observeIntegrationWebhook("workflow_job", "missing-attempt", bad, now) == nil { + t.Fatal("missing attempt silently attributed") + } +} + +func TestIntegrationLeaseAtomicAdmissionAndExpiry(t *testing.T) { + s := orchestrationServer(t, config.IntegrationEnforce) + now := time.Now().UTC() + store, err := s.integrationStore() + if err != nil { + t.Fatal(err) + } + i := integration.Intent{Version: 1, Key: "o/r#1", Repo: "o/r", Base: "v5", Generation: 1, Plan: "unresolved", PlanNode: "o/r#1", ScopeKnown: true, Provenance: "dispatch-enumerator", ObservedAt: now} + if err := store.EnsureIntent(i); err != nil { + t.Fatal(err) + } + c := integration.Coverage{Repo: "o/r", Base: "v5", From: now.Add(-2 * time.Hour), Through: now, Complete: true, Source: "github-actions"} + if err := store.Append(integration.Event{Version: 1, ID: "coverage", At: now, Coverage: &c}); err != nil { + t.Fatal(err) + } + if err := s.integrationLease("o/r#1", "o/r", "alice", 1, now.Add(time.Hour), now, false); err != nil { + t.Fatal(err) + } + if err := s.integrationLease("o/r#1", "o/r", "bob", 1, now.Add(time.Hour), now, false); err == nil { + t.Fatal("competing lease acquired active reservation") + } + if len(store.Snapshot(now.Add(16*time.Minute)).Reservations) != 0 { + t.Fatal("orphaned reservation did not expire") + } + s.integrationRelease("o/r#1", "contributor/alice", 1, "test") + if len(store.Snapshot(now).Reservations) != 0 { + t.Fatal("release retained reservation") + } +} + +func TestIntegrationScopeAmbiguityAndUnavailableLedger(t *testing.T) { + s := orchestrationServer(t, config.IntegrationEnforce) + s.deps.Config.IntegrationOrchestration.Scopes = append(s.deps.Config.IntegrationOrchestration.Scopes, config.IntegrationScope{Repo: "o/r", Base: "other"}) + d := s.integrationDispatchDecision(worksource.Ref{Repo: "o/r", Number: 1}) + if d.Allowed || !strings.Contains(d.Reason, "base unknown") { + t.Fatalf("%+v", d) + } +} + +func TestIntegrationRealClaimCommitSerializesCapacity(t *testing.T) { + _, server, ledger := claimsHub(t) + server.deps.Config.IntegrationOrchestration = config.IntegrationOrchestrationConfig{Mode: config.IntegrationEnforce, MaxActive: 1, Scopes: []config.IntegrationScope{{Repo: "myorg/repo1", Base: "v5"}}} + server.integration.path = filepath.Join(t.TempDir(), "journal") + store, err := server.integrationStore() + if err != nil { + t.Fatal(err) + } + now := time.Now() + for _, number := range []int{1, 2} { + intent := integration.Intent{Version: 1, Key: fmt.Sprintf("myorg/repo1#%d", number), Repo: "myorg/repo1", Base: "v5", Generation: 1, Plan: "unresolved", PlanNode: "owner-enrolled", ScopeKnown: true, Provenance: "owner", ObservedAt: now} + if err := store.EnsureIntent(intent); err != nil { + t.Fatal(err) + } + } + coverage := integration.Coverage{Repo: "myorg/repo1", Base: "v5", From: now.Add(-2 * time.Hour), Through: now, Complete: true, Source: "github-actions"} + if err := store.Append(integration.Event{Version: 1, ID: "coverage", At: now, Coverage: &coverage}); err != nil { + t.Fatal(err) + } + var workers sync.WaitGroup + changed := make(chan bool, 2) + for _, number := range []int{1, 2} { + workers.Add(1) + go func(number int) { + defer workers.Done() + result, err := ledger.Claim(claims.Request{Repo: "myorg/repo1", Issue: number, Holder: "agent", Kind: claims.KindAgent}) + if err != nil { + t.Error(err) + } + changed <- result.Outcome == claims.OutcomeClaimed + }(number) + } + workers.Wait() + close(changed) + count := 0 + for granted := range changed { + if granted { + count++ + } + } + if count != 1 || len(store.Snapshot(now).Reservations) != 1 { + t.Fatalf("claims=%d reservations=%d", count, len(store.Snapshot(now).Reservations)) + } +} + +func TestIntegrationActualContributorSelectionModes(t *testing.T) { + for _, mode := range []string{config.IntegrationOff, config.IntegrationAdvisory, config.IntegrationEnforce} { + t.Run(mode, func(t *testing.T) { + hub, server, _ := claimsHub(t) + server.deps.Config.IntegrationOrchestration = config.IntegrationOrchestrationConfig{Mode: mode, Scopes: []config.IntegrationScope{{Repo: "myorg/repo1", Base: "v5"}}} + server.integration.path = filepath.Join(t.TempDir(), "journal") + setStatusIssues(server, intgIssue(1, "independent work", "someone", nil)) + connection := &ContributorConnection{profile: &ContributorProfile{GitHubUsername: "worker", ContributorID: "c-worker", TrustTier: "contributor"}, lastPong: time.Now()} + hub.mu.Lock() + hub.connections["conn-worker"] = connection + hub.mu.Unlock() + message := hub.selectTask(connection) + admitted := message != nil && message.Type == "task_assign" + if admitted != (mode != config.IntegrationEnforce) { + t.Fatalf("mode=%s message=%+v", mode, message) + } + }) + } +} + +func TestIntegrationFailedLedgerCannotPreventDisabling(t *testing.T) { + server := orchestrationServer(t, config.IntegrationEnforce) + if err := os.WriteFile(server.integration.path, []byte("corrupt\n"), 0600); err != nil { + t.Fatal(err) + } + if _, err := server.integrationStore(); err == nil { + t.Fatal("invalid journal accepted") + } + request := httptest.NewRequest(http.MethodPut, "/api/config/integration-orchestration", strings.NewReader(`{"mode":"off"}`)) + request.Header.Set("X-Hive-Role", "owner") + request.Header.Set(ownerRoleVerifiedHeader, "true") + response := httptest.NewRecorder() + server.handleIntegrationConfigPut(response, request) + if response.Code != http.StatusOK || server.integrationConfig().Mode != config.IntegrationOff { + t.Fatalf("%d %s", response.Code, response.Body) + } +} diff --git a/src/pkg/dashboard/integration_plan_test.go b/src/pkg/dashboard/integration_plan_test.go new file mode 100644 index 0000000000..71af763ae1 --- /dev/null +++ b/src/pkg/dashboard/integration_plan_test.go @@ -0,0 +1,83 @@ +package dashboard + +import ( + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/beads" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/convergence/integration" + "github.com/hivecommons/hive/pkg/planning" + "github.com/hivecommons/hive/pkg/worksource" +) + +func TestIntegrationDispatchUsesApprovedCanonicalDAGAndMergedPrerequisite(t *testing.T) { + server := orchestrationServer(t, config.IntegrationAdvisory) + store, err := beads.NewStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + epic, err := store.Create("approved plan", beads.TypeEpic, beads.PriorityMedium, "architect", "") + if err != nil { + t.Fatal(err) + } + for key, value := range map[string]string{planning.MetaPlanStatus: planning.PlanStatusApproved, planning.MetaDesignRevision: "2"} { + if err := store.SetMetadata(epic.ID, key, value); err != nil { + t.Fatal(err) + } + } + var children []*beads.Bead + for _, ref := range []string{"gh-o/r#1", "gh-o/r#2", "gh-o/r#3"} { + child, err := store.Create(ref, beads.TypeTask, beads.PriorityMedium, "architect", ref) + if err != nil { + t.Fatal(err) + } + if err := store.SetMetadata(child.ID, planning.MetaParentEpic, epic.ID); err != nil { + t.Fatal(err) + } + children = append(children, child) + } + if err := store.AddDependency(children[1].ID, children[0].ID); err != nil { + t.Fatal(err) + } + if err := store.SetMetadata(children[0].ID, planning.MetaPRURL, "https://github.com/o/r/pull/9"); err != nil { + t.Fatal(err) + } + server.deps.BeadStores = map[string]*beads.Store{"architect": store} + intent := server.integrationDispatchIntent(worksource.Ref{Repo: "o/r", Number: 2}, "v5", time.Now()) + if intent.Plan != epic.ID || intent.PlanRevision != 2 || intent.ScopeKnown || len(intent.Prerequisites) != 1 || intent.Prerequisites[0].Key != "o/r#1" { + t.Fatalf("approved DAG not attached to canonical dispatch: %+v", intent) + } + approved, dependencies := server.integrationPlan(intent) + if !approved || dependencies["o/r#1"].Known { + t.Fatalf("unmerged closure inferred readiness: %+v", dependencies) + } + ledger, err := server.integrationStore() + if err != nil { + t.Fatal(err) + } + receipt := integration.IntegrationReceipt{Repo: "o/r", PR: 9, Head: strings.Repeat("a", 40), MergeSHA: strings.Repeat("b", 40), MergedAt: time.Now(), Source: "github-signed-merged-webhook-v1"} + if err := ledger.Append(integration.Event{Version: 1, ID: "merged-prerequisite", At: receipt.MergedAt, Integrated: &receipt}); err != nil { + t.Fatal(err) + } + approved, dependencies = server.integrationPlan(intent) + if !approved || !dependencies["o/r#1"].Known || !dependencies["o/r#1"].Satisfied { + t.Fatalf("%v %+v", approved, dependencies) + } + omitted := intent + omitted.Prerequisites = nil + if approved, _ := server.integrationPlan(omitted); approved { + t.Fatal("omitting approved dependency bypassed DAG") + } + unrelated := server.integrationDispatchIntent(worksource.Ref{Repo: "o/r", Number: 3}, "v5", time.Now()) + if approved, deps := server.integrationPlan(unrelated); !approved || len(deps) != 0 { + t.Fatal("independent approved node inherited another consumer's dependency") + } + if err := store.SetMetadata(epic.ID, planning.MetaDesignRevision, "3"); err != nil { + t.Fatal(err) + } + if approved, _ := server.integrationPlan(intent); approved { + t.Fatal("changed plan generation reused dependency proof") + } +} diff --git a/src/pkg/dashboard/integration_policy_observations.go b/src/pkg/dashboard/integration_policy_observations.go new file mode 100644 index 0000000000..7346af2bb1 --- /dev/null +++ b/src/pkg/dashboard/integration_policy_observations.go @@ -0,0 +1,88 @@ +package dashboard + +import ( + "context" + "fmt" + "strings" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/convergence/integration" + "github.com/hivecommons/hive/pkg/github" +) + +func integrationNativeMembershipCompatible(previous integration.Subject, current github.NativeIntegrationGroup) bool { + if previous.TestedTree != current.Tree || previous.TestedSHA != current.HeadSHA || previous.Authority != current.HeadRef || + !strings.EqualFold(previous.Repo, current.Repo) || previous.Base != current.Base || len(current.Members) == 0 || len(current.Members) > len(previous.Members) { + return false + } + offset := len(previous.Members) - len(current.Members) + if offset == 0 && previous.BaseSHA != current.BaseSHA || offset > 0 && previous.Members[offset-1].HeadCommit != current.BaseSHA { + return false + } + for index, member := range current.Members { + old := previous.Members[offset+index] + if old.PR != member.PR || old.Head != member.Head || old.Entry != member.Entry || member.Position != index+1 || + old.BaseCommit != member.BaseCommit || old.HeadCommit != member.HeadCommit { + return false + } + } + // A documented exact prefix promotion retains its original subject and + // lineage. It does not copy an old receipt onto a member-removed subject. + return true +} + +func (s *Server) observeIntegrationDirectPolicyRun(parent context.Context, run *gh.WorkflowRun, repo, delivery string, now time.Time) error { + if s.deps == nil || s.deps.GHClient == nil { + return nil + } + cfg := s.integrationConfig() + store, err := s.integrationStore() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(parent, integrationReconcileTimeout) + defer cancel() + for _, policy := range cfg.Policies { + if policy.Engine != "direct" || !strings.EqualFold(policy.Repo, repo) || len(run.PullRequests) != 1 || + run.PullRequests[0].GetBase().GetRef() != policy.Base || strings.Split(run.GetPath(), "@")[0] != policy.Workflow { + continue + } + eventID := "policy/" + delivery + "/" + policy.Workflow + if store.HasEvent(eventID) { + continue + } + proof, proofErr := s.deps.GHClient.ReadIntegrationWorkflowProof(ctx, policy, run) + if proofErr != nil { + return proofErr + } + attestation, attestationErr := s.deps.GHClient.ReadDirectIntegrationAttestation(ctx, policy, run) + if attestationErr != nil { + return attestationErr + } + subject := integration.Subject{Version: integration.SchemaVersion, Repo: repo, Base: policy.Base, Engine: "direct", Authority: fmt.Sprintf("pull/%d", run.PullRequests[0].GetNumber()), + BaseSHA: attestation.BaseSHA, TestedSHA: attestation.TestedSHA, TestedTree: attestation.TestedTree, + Policy: policy.Fingerprint(), Producer: fmt.Sprintf("github-app/%d", policy.AppID), Workflow: fmt.Sprintf("%s@%d", policy.Workflow, run.GetWorkflowID()), + WorkflowSourceRepo: run.GetHeadRepository().GetFullName(), + Attempt: run.GetRunAttempt(), Generation: uint64(run.GetID())} + for _, member := range attestation.Members { + subject.Members = append(subject.Members, integration.Member{PR: member.PR, Head: member.Head, Entry: fmt.Sprintf("pull/%d", member.PR), Position: 1}) + } + state := integration.Running + if proof.Passed { + state = integration.Passed + } else if proof.Rejected { + state = integration.Rejected + } + observedAt := run.GetUpdatedAt().Time + if observedAt.IsZero() { + observedAt = now + } + receipt := integration.Evidence{Subject: subject, State: state, ObservedAt: observedAt, Source: "github-actions-attested-v1", + RunID: proof.RunID, SuiteID: proof.SuiteID, CheckIDs: proof.CheckIDs, Reason: proof.Reason} + if err := store.Append(integration.Event{Version: integration.SchemaVersion, ID: eventID, At: now, Evidence: &receipt}); err != nil { + return err + } + } + return nil +} diff --git a/src/pkg/dashboard/integration_policy_observations_test.go b/src/pkg/dashboard/integration_policy_observations_test.go new file mode 100644 index 0000000000..45f8d9aed6 --- /dev/null +++ b/src/pkg/dashboard/integration_policy_observations_test.go @@ -0,0 +1,141 @@ +package dashboard + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/convergence/integration" + "github.com/hivecommons/hive/pkg/github" +) + +func TestIntegrationNativePrefixPromotionRetainsOriginalIdentity(t *testing.T) { + base, a, b, tree, head1, head2 := strings.Repeat("a", 40), strings.Repeat("b", 40), strings.Repeat("c", 40), strings.Repeat("d", 40), strings.Repeat("e", 40), strings.Repeat("f", 40) + original := integration.Subject{Repo: "o/r", Base: "v5", BaseSHA: base, Authority: "group", TestedSHA: b, TestedTree: tree, + Members: []integration.Member{{PR: 1, Head: head1, Entry: "A", Position: 1, BaseCommit: base, HeadCommit: a}, {PR: 2, Head: head2, Entry: "B", Position: 2, BaseCommit: a, HeadCommit: b}}} + promoted := github.NativeIntegrationGroup{Repo: "o/r", Base: "v5", BaseSHA: a, HeadRef: "group", HeadSHA: b, Tree: tree, + Members: []github.NativeIntegrationMember{{PR: 2, Head: head2, Entry: "B", Position: 1, BaseCommit: a, HeadCommit: b}}} + if !integrationNativeMembershipCompatible(original, promoted) || len(original.Members) != 2 { + t.Fatal("valid documented prefix discarded or rewritten") + } + for _, change := range []func(*github.NativeIntegrationGroup){ + func(g *github.NativeIntegrationGroup) { g.BaseSHA = base }, + func(g *github.NativeIntegrationGroup) { g.Tree = base }, + func(g *github.NativeIntegrationGroup) { g.Members[0].Head = head1 }, + func(g *github.NativeIntegrationGroup) { g.Members[0].Entry = "other" }, + func(g *github.NativeIntegrationGroup) { g.HeadSHA = a }, + } { + bad := promoted + bad.Members = append([]github.NativeIntegrationMember(nil), promoted.Members...) + change(&bad) + if integrationNativeMembershipCompatible(original, bad) { + t.Fatalf("transferred evidence to changed candidate: %+v", bad) + } + } +} + +func TestIntegrationRepairUsesExactLiveFrontWithoutBlameOrAuthority(t *testing.T) { + for _, scenario := range []string{"valid", "direct-bound", "unbound-author-head", "advisory", "superseded", "unknown", "stale", "future", "new-head", "closed", "retargeted", "read-error"} { + t.Run(scenario, func(t *testing.T) { + now := time.Now().UTC() + head, base, tested, tree := strings.Repeat("a", 40), strings.Repeat("b", 40), strings.Repeat("c", 40), strings.Repeat("d", 40) + server := orchestrationServer(t, config.IntegrationEnforce) + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/repos/o/r/pulls/7" { + t.Errorf("unexpected request %s", r.URL) + } + if scenario == "read-error" { + w.WriteHeader(http.StatusBadGateway) + return + } + actualHead, state, target := head, "open", "v5" + if scenario == "new-head" { + actualHead = strings.Repeat("f", 40) + } + if scenario == "closed" { + state = "closed" + } + if scenario == "retargeted" { + target = "other" + } + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"number":7,"state":%q,"title":"front","html_url":"https://github.com/o/r/pull/7","user":{"login":"service-owner"},"head":{"sha":%q},"base":{"ref":%q}}`, state, actualHead, target) + })) + defer api.Close() + server.deps.GHClient = github.NewClient("token", "o", []string{"r"}, nil, api.URL) + if scenario == "advisory" { + server.deps.Config.IntegrationOrchestration.Mode = config.IntegrationAdvisory + } + store, err := server.integrationStore() + if err != nil { + t.Fatal(err) + } + receipt := integration.Evidence{Subject: integration.Subject{Version: 1, Repo: "o/r", Base: "v5", Engine: "native", Authority: "group", BaseSHA: base, TestedSHA: tested, TestedTree: tree, + Policy: "unverified-policy", Producer: "github-merge-group-webhook", Workflow: "merge_group", Attempt: 1, Generation: 1, + Members: []integration.Member{{PR: 7, Head: head, Entry: "entry", Position: 1}}}, State: integration.Waiting, ObservedAt: now, Source: "github-merge-group-v1"} + switch scenario { + case "superseded": + receipt.State = integration.Superseded + case "unknown": + receipt.State = integration.Unknown + case "stale": + receipt.ObservedAt = now.Add(-time.Hour) + case "future": + receipt.ObservedAt = now.Add(time.Hour) + } + if err := store.Append(integration.Event{Version: 1, ID: "receipt", At: now, Evidence: &receipt}); err != nil { + t.Fatal(err) + } + failure := &github.WebhookCIFailure{HeadSHA: tested} + if scenario == "direct-bound" || scenario == "unbound-author-head" { + receipt.RunID, receipt.SuiteID, receipt.CheckIDs = 10, 20, []int64{30} + receipt.Subject.Authority, receipt.Subject.Engine = "pull/7", "direct" + receipt.ObservedAt = now.Add(time.Nanosecond) + if err := store.Append(integration.Event{Version: 1, ID: "direct-receipt", At: now, Evidence: &receipt}); err != nil { + t.Fatal(err) + } + failure.HeadSHA = head + if scenario == "direct-bound" { + failure.SuiteID, failure.CheckRunID = 20, 30 + } + } + owner, context, exact := server.integrationFailureOwner("o/r", failure) + if scenario == "valid" || scenario == "direct-bound" { + if owner == nil || owner.Number != 7 || context == nil || context.TestedTree != tree || exact != head || len(context.Members) != 1 { + t.Fatalf("%+v %+v %s", owner, context, exact) + } + + } else if owner != nil || context != nil || exact != "" { + t.Fatalf("stale/unknown repair owner selected: %s %+v", scenario, owner) + } + }) + } +} + +func TestIntegrationFailureIdentityOnlyAddsSignedPayloadIDsInEnforce(t *testing.T) { + for _, mode := range []string{config.IntegrationOff, config.IntegrationAdvisory, config.IntegrationEnforce} { + server := orchestrationServer(t, mode) + failure := &github.WebhookCIFailure{} + body := []byte(`{"check_run":{"id":30,"check_suite":{"id":20}},"check_suite":{"id":40}}`) + if err := server.integrationFailureIdentity("check_run", body, failure); err != nil { + t.Fatal(err) + } + if mode == config.IntegrationEnforce { + if failure.SuiteID != 20 || failure.CheckRunID != 30 { + t.Fatalf("lost signed check identity: %+v", failure) + } + if err := server.integrationFailureIdentity("check_suite", body, failure); err != nil || failure.SuiteID != 40 { + t.Fatal("lost signed suite") + } + if err := server.integrationFailureIdentity("check_run", []byte("{"), failure); err == nil { + t.Fatal("malformed identity accepted") + } + } else if failure.SuiteID != 0 || failure.CheckRunID != 0 { + t.Fatal("disabled/advisory changed legacy failure") + } + } +} diff --git a/src/pkg/dashboard/integration_reconcile_test.go b/src/pkg/dashboard/integration_reconcile_test.go new file mode 100644 index 0000000000..5bf00aa0c8 --- /dev/null +++ b/src/pkg/dashboard/integration_reconcile_test.go @@ -0,0 +1,140 @@ +package dashboard + +import ( + "context" + "fmt" + "math" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/convergence/integration" + ghpkg "github.com/hivecommons/hive/pkg/github" +) + +func TestIntegrationReconcilesOldCompletedRunsAndEveryAttempt(t *testing.T) { + for _, scenario := range []string{"complete", "overflow", "missing-attempt", "job-fanout", "unknown-base", "unavailable"} { + t.Run(scenario, func(t *testing.T) { + now := time.Now().UTC() + head := strings.Repeat("a", 40) + attemptReads := 0 + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/repos/o/r/pulls": + _, _ = w.Write([]byte("[]")) + case r.URL.Path == "/repos/o/r/actions/runs": + if r.URL.Query().Get("created") != "" { + t.Error("creation window can omit old completed execution") + } + if scenario == "unavailable" { + w.WriteHeader(http.StatusBadGateway) + return + } + if scenario == "overflow" { + w.Header().Set("Link", `; rel="next"`) + } + attempt, branch := 2, "v5" + if scenario == "missing-attempt" { + attempt = 0 + } + if scenario == "unknown-base" { + branch = "unidentified-feature" + } + _, _ = fmt.Fprintf(w, `{"total_count":1,"workflow_runs":[{"id":10,"run_attempt":%d,"status":"completed","head_sha":%q,"head_branch":%q,"path":".github/workflows/test.yml","created_at":%q,"updated_at":%q}]}`, attempt, head, branch, now.Add(-2*time.Hour).Format(time.RFC3339), now.Add(-20*time.Minute).Format(time.RFC3339)) + case strings.Contains(r.URL.Path, "/attempts/"): + attemptReads++ + attempt, start, end, conclusion := 1, now.Add(-70*time.Minute), now.Add(-50*time.Minute), "cancelled" + if strings.Contains(r.URL.Path, "/attempts/2/") { + attempt, start, end, conclusion = 2, now.Add(-40*time.Minute), now.Add(-20*time.Minute), "success" + } + if scenario == "job-fanout" { + w.Header().Set("Link", `; rel="next"`) + } + _, _ = fmt.Fprintf(w, `{"total_count":1,"jobs":[{"id":%d,"run_id":10,"head_sha":%q,"runner_id":30,"status":"completed","conclusion":%q,"started_at":%q,"completed_at":%q}]}`, 100+attempt, head, conclusion, start.Format(time.RFC3339), end.Format(time.RFC3339)) + default: + t.Errorf("unexpected request %s", r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer api.Close() + server := orchestrationServer(t, config.IntegrationAdvisory) + server.deps.GHClient = ghpkg.NewClient("token", "o", []string{"r"}, nil, api.URL) + err := server.ReconcileIntegration(context.Background()) + if (err != nil) != (scenario == "unavailable") { + t.Fatalf("%v", err) + } + store, storeErr := server.integrationStore() + if storeErr != nil { + t.Fatal(storeErr) + } + snapshot := store.Snapshot(now) + if len(snapshot.Coverage) != 1 || snapshot.Coverage[0].Complete != (scenario == "complete") { + t.Fatalf("%+v", snapshot.Coverage) + } + if scenario == "complete" { + cost := store.Cost("O/R", "v5", snapshot.Coverage[0].From, snapshot.Coverage[0].Through) + if cost.RunnerMinutes == nil || math.Abs(*cost.RunnerMinutes-30) > 0.05 || cost.Jobs != 2 || attemptReads != 2 { + t.Fatalf("cost=%+v reads=%d", cost, attemptReads) + } + if len(snapshot.Jobs) != 2 || snapshot.Jobs[0].Attempt == snapshot.Jobs[1].Attempt { + t.Fatalf("%+v", snapshot.Jobs) + } + } + reads := attemptReads + if err := server.ReconcileIntegration(context.Background()); err != nil { + t.Fatal(err) + } + if attemptReads != reads { + t.Fatal("minimum batch interval ignored") + } + }) + } +} + +func TestIntegrationObserverOffAndUnavailableRemainExplicit(t *testing.T) { + server := orchestrationServer(t, config.IntegrationOff) + if err := server.ReconcileIntegration(context.Background()); err != nil { + t.Fatal(err) + } + if err := server.observeIntegrationWebhook("workflow_run", "unused", []byte("malformed"), time.Now()); err != nil { + t.Fatal("OFF parsed payload", err) + } + if server.integration.store != nil { + t.Fatal("OFF opened observation store") + } + server.deps.Config.IntegrationOrchestration.Mode = config.IntegrationEnforce + if server.ReconcileIntegration(context.Background()) == nil { + t.Fatal("missing telemetry client silently succeeded") + } + var nilServer *Server + if err := nilServer.ReconcileIntegration(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestIntegrationMergedWebhookHasSeparateLifecycleReceipt(t *testing.T) { + server := orchestrationServer(t, config.IntegrationAdvisory) + body := []byte(`{"action":"closed","repository":{"full_name":"o/r"},"pull_request":{"number":7,"merged":true,"merged_at":"2026-10-30T20:46:06Z","merge_commit_sha":"` + strings.Repeat("a", 40) + `","head":{"sha":"` + strings.Repeat("b", 40) + `"}}}`) + if err := server.observeIntegrationWebhook("pull_request", "merged", body, time.Now()); err != nil { + t.Fatal(err) + } + store, err := server.integrationStore() + if err != nil { + t.Fatal(err) + } + if receipt, exists := store.Integrated("o/r", 7); !exists || receipt.Source != "github-signed-merged-webhook-v1" { + t.Fatalf("%+v %v", receipt, exists) + } + if len(store.Snapshot(time.Now()).Evidence) != 0 { + t.Fatal("merge receipt invented passing integration CI proof") + } + unknown := integration.Subject{Version: 1, Repo: "o/r", Authority: "run/1"} + receipt := integration.Evidence{Subject: unknown, State: integration.Unknown, ObservedAt: time.Now(), Source: "github"} + if err := store.Append(integration.Event{Version: 1, ID: "unknown", At: time.Now(), Evidence: &receipt}); err != nil { + t.Fatal(err) + } +} diff --git a/src/pkg/dashboard/integration_repair.go b/src/pkg/dashboard/integration_repair.go new file mode 100644 index 0000000000..c800c459fd --- /dev/null +++ b/src/pkg/dashboard/integration_repair.go @@ -0,0 +1,99 @@ +package dashboard + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/convergence/integration" + "github.com/hivecommons/hive/pkg/github" +) + +// integrationFailureOwner enriches the existing owner-first repair route. +// The front is a service owner, not an assertion that it caused the failure. +func (s *Server) integrationFailureOwner(repo string, failure *github.WebhookCIFailure) (*github.CIFailurePR, *github.IntegrationFailureContext, string) { + cfg := s.integrationConfig() + if cfg.Mode != config.IntegrationEnforce || failure == nil || s.deps == nil || s.deps.GHClient == nil { + return nil, nil, "" + } + enrolled := false + for _, scope := range cfg.Scopes { + if strings.EqualFold(scope.Repo, repo) { + enrolled = true + break + } + } + if !enrolled { + return nil, nil, "" + } + store, err := s.integrationStore() + if err != nil { + return nil, nil, "" + } + var selected *integration.Evidence + now := time.Now() + for _, receipt := range store.ForFailure(repo, failure.HeadSHA, failure.SuiteID, failure.CheckRunID) { + if !strings.EqualFold(receipt.Subject.Repo, repo) || !cfg.Allows(repo, receipt.Subject.Base) || + receipt.State == integration.Superseded || receipt.State == integration.Unknown || receipt.Subject.Validate() != nil || + receipt.ObservedAt.After(now) || now.Sub(receipt.ObservedAt) > cfg.MaxObservationAge { + continue + } + if selected == nil || receipt.ObservedAt.After(selected.ObservedAt) { + copyReceipt := receipt + selected = ©Receipt + } + } + if selected == nil { + return nil, nil, "" + } + subject := selected.Subject + member := subject.Members[0] + parts := strings.SplitN(repo, "/", 2) + if len(parts) != 2 { + return nil, nil, "" + } + ctx, cancel := context.WithTimeout(context.Background(), integrationReconcileTimeout) + defer cancel() + pr, _, err := s.deps.GHClient.GoGitHub().PullRequests.Get(ctx, parts[0], parts[1], member.PR) + if err != nil || pr == nil || pr.GetState() != "open" || pr.GetHead().GetSHA() != member.Head || pr.GetBase().GetRef() != subject.Base { + return nil, nil, "" + } + ref := &github.CIFailurePR{Number: member.PR, Author: pr.GetUser().GetLogin(), Title: pr.GetTitle(), URL: pr.GetHTMLURL(), BaseRef: subject.Base, + HiveAuthored: s.deps.GHClient.IsHiveAuthorLogin(pr.GetUser().GetLogin())} + context := &github.IntegrationFailureContext{CandidateID: subject.ID(), TestedSHA: subject.TestedSHA, TestedTree: subject.TestedTree, + BaseSHA: subject.BaseSHA, Policy: subject.Policy, Producer: subject.Producer, Attempt: subject.Attempt, RunID: selected.RunID, SuiteID: selected.SuiteID, Source: selected.Source} + for _, m := range subject.Members { + context.Members = append(context.Members, fmt.Sprintf("%d:%s", m.PR, m.Head)) + } + return ref, context, member.Head +} + +func (s *Server) integrationFailureIdentity(event string, body []byte, failure *github.WebhookCIFailure) error { + if failure == nil || s.integrationConfig().Mode != config.IntegrationEnforce || event != "check_run" && event != "check_suite" { + return nil + } + var payload struct { + CheckRun struct { + ID int64 `json:"id"` + CheckSuite struct { + ID int64 `json:"id"` + } `json:"check_suite"` + } `json:"check_run"` + CheckSuite struct { + ID int64 `json:"id"` + } `json:"check_suite"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return err + } + if event == "check_run" { + failure.CheckRunID, failure.SuiteID = payload.CheckRun.ID, payload.CheckRun.CheckSuite.ID + } + if event == "check_suite" { + failure.SuiteID = payload.CheckSuite.ID + } + return nil +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 99f286748d..c3d791e3d5 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -100,6 +100,7 @@ var proxyProofRequired = true const staleDropStarvationThreshold = 5 type Server struct { + integration integrationRuntime webhookNoSecretOnce sync.Once // warn once per process about a missing webhook secret (#13666) feedbackRefreshBackoff feedbackBackoff diff --git a/src/pkg/dashboard/static-src/js/config-features.js b/src/pkg/dashboard/static-src/js/config-features.js index 57154b8833..05fdb86da4 100644 --- a/src/pkg/dashboard/static-src/js/config-features.js +++ b/src/pkg/dashboard/static-src/js/config-features.js @@ -143,6 +143,7 @@ am = am || {}; rv = rv || {}; const cv = (_configState.data || {}).convergence || {}; + const io = (_configState.data || {}).integration_orchestration || { mode: 'off', scopes: [] }; const planOn = f.planFromLabel === true; const formalAvailable = f.formalAvailable === true; const formalMinACMMLevel = f.formalMinACMMLevel || 5; @@ -1143,6 +1144,48 @@
Checkpoint, triage and recheck settings are applied live; no restart is needed.
+
+
Integration orchestration
+
Default OFF. Coordinates authoring admission above existing preparation and native queues; never replaces CI, review, holds or current-base guards. Advisory cannot block. Enforce requires explicit repository/base scopes and complete runner telemetry. No native queue jumping or GitHub build-window control. 5,000/day is a capacity target, not a measured result.
+
+ +
+
+ +
+
+ ${[ + ['max_active','Active admissions',2,'integration_orchestration.max_active'], + ['max_jobs','Executed jobs per window',20,'integration_orchestration.max_jobs'], + ['max_runner_minutes','Runner minutes per window',60,'integration_orchestration.max_runner_minutes'], + ['max_attempts','Attempt bound',2,'integration_orchestration.max_attempts'], + ['admission_jobs','Job allocation per admission',1,'integration_orchestration.admission_jobs'], + ['admission_runner_minutes','Minute allocation per admission',10,'integration_orchestration.admission_runner_minutes'] + ].map(([key,label,def,path]) => + `
`).join('')} +
+
+ ${[ + ['lease_ttl','Reservation TTL (minutes)',5,'integration_orchestration.lease_ttl'], + ['window','Accounting window (minutes)',60,'integration_orchestration.window'], + ['max_observation_age','Maximum observation age (minutes)',30,'integration_orchestration.max_observation_age'] + ].map(([key,label,def,path]) => + `
`).join('')} +
+
+ +
+
${[ + 'integration_orchestration.policies[].repo', 'integration_orchestration.policies[].base', + 'integration_orchestration.policies[].engine', 'integration_orchestration.policies[].workflow', + 'integration_orchestration.policies[].blob', 'integration_orchestration.policies[].app_id', + 'integration_orchestration.policies[].jobs[]' + ].map(path => yamlHint(path)).join(' ')} Disabling releases only orchestration reservations; existing claims and native entries remain. Missing group membership, policy or producer proof stays unknown.
+ +

+        
Convergence Rollout ${settingHelpMark('settings', 'features', 'Convergence Rollout', `Convergence checks whether work Hive is about to kick is actually admissible right now — blocker still open, dependency unmerged, evidence stale — so agents stop burning tokens rediscovering "not ready yet". It gates candidates only; it never plans or schedules.`)}
@@ -1159,4 +1202,3 @@
`)}`; } - diff --git a/src/pkg/dashboard/static-src/js/config-helpers.js b/src/pkg/dashboard/static-src/js/config-helpers.js index c1bc594720..407f9531c6 100644 --- a/src/pkg/dashboard/static-src/js/config-helpers.js +++ b/src/pkg/dashboard/static-src/js/config-helpers.js @@ -476,6 +476,7 @@ : 'No soak telemetry recorded yet — it accumulates one row per eval cycle, so give it a few cycles.'; return; } + const groups = {}; for (const e of entries) { const k = `${e.commit || '?'} / ${e.mode || '?'} (gen ${e.generation || 0})`; @@ -511,6 +512,32 @@ } } + async function loadIntegrationOrchestration() { + const out = document.getElementById('integration-orchestration-summary'); + if (!out) return; + out.textContent = 'Loading authoritative observations…'; + try { + const res = await fetch('/api/integration-orchestration', { headers: _inceptionAuthHeaders() }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const data = await res.json(); + const preparation = await fetch('/api/stage-tracker', { headers: _inceptionAuthHeaders() }); + if (preparation.ok) { + const tracker = await preparation.json(); + data.preparation = { + source: '/api/stage-tracker.base_preparation', + state: Array.isArray(tracker.base_preparation) ? 'observed' : 'unavailable', + queues: Array.isArray(tracker.base_preparation) ? tracker.base_preparation : null, + meaning: 'Preparation ownership/backlog only; not green or approved train seats, and not native queue membership.' + }; + } else { + data.preparation = { state: 'unavailable', reason: `Tracker HTTP ${preparation.status}` }; + } + out.textContent = JSON.stringify(data, null, 2); + } catch (e) { + out.textContent = `Integration observations unknown: ${e.message}`; + } + } + // ── Upstream watch editor + divergence view (#9969, #11211) ───────────── // loadUpstreamWatch paints the editor (on/off, poll interval, add fork, // per-fork settings) and the per-repo divergence table from @@ -674,4 +701,3 @@ if (!await hiveConfirm({ title: `Stop watching ${repo}?`, message: 'The watch stops polling its upstream. Recorded state and issues already filed are kept.', confirmLabel: 'Stop watching', danger: true })) return; putUpstreamWatch({ repos: { [repo]: null } }, `Stopped watching ${repo}`); } - diff --git a/src/pkg/dashboard/static-src/js/linear-repos-save.js b/src/pkg/dashboard/static-src/js/linear-repos-save.js index c26f7270b3..8aea52a3d3 100644 --- a/src/pkg/dashboard/static-src/js/linear-repos-save.js +++ b/src/pkg/dashboard/static-src/js/linear-repos-save.js @@ -768,6 +768,33 @@ }); if (!slaRes.ok) throw new Error(await saveErrorMessage(slaRes)); _configState.data.sla = await slaRes.json().catch(() => _configState.data.sla); + } else if (_configState.type === 'governor' && section === 'integration_orchestration') { + const body = Object.assign({}, (_configState.data || {}).integration_orchestration || {}, values); + const integrationMinuteNanoseconds = 60 * 1e9; + for (const key of ['lease_ttl', 'window', 'max_observation_age']) { + if (Object.prototype.hasOwnProperty.call(body, key + '_minutes')) { + body[key] = Number(body[key + '_minutes']) * integrationMinuteNanoseconds; + delete body[key + '_minutes']; + } + } + if (Object.prototype.hasOwnProperty.call(body, 'policy_json')) { + body.policies = JSON.parse(body.policy_json); + if (!Array.isArray(body.policies)) throw new Error('Integration workflow contracts must be a JSON array'); + delete body.policy_json; + } + if (Object.prototype.hasOwnProperty.call(body, 'scope_lines')) { + body.scopes = String(body.scope_lines).split('\n').map(s => s.trim()).filter(Boolean).map(s => { + const at = s.indexOf('#'); + if (at < 1 || at === s.length - 1) throw new Error('Each integration scope must be owner/repo#base'); + return { repo: s.slice(0, at), base: s.slice(at + 1) }; + }); + delete body.scope_lines; + } + const ioRes = await fetch('/api/config/integration-orchestration', { + method: 'PUT', headers: _inceptionAuthHeaders(), body: JSON.stringify(body) + }); + if (!ioRes.ok) throw new Error(await saveErrorMessage(ioRes)); + _configState.data.integration_orchestration = await ioRes.json(); } else if (_configState.type === 'governor' && section === 'convergence') { // convergence is a TOP-LEVEL config block, not under governor — // it saves through its own owner-only /api/config/convergence @@ -931,4 +958,3 @@ }) .catch(() => {}); } - diff --git a/src/pkg/dashboard/static-src/js/tooltips-dialogs.js b/src/pkg/dashboard/static-src/js/tooltips-dialogs.js index d861466a1f..d28aeb13d5 100644 --- a/src/pkg/dashboard/static-src/js/tooltips-dialogs.js +++ b/src/pkg/dashboard/static-src/js/tooltips-dialogs.js @@ -272,6 +272,7 @@ case 'wsWavefrontSetEnabled': wsWavefrontSetEnabled(el); break; case 'toggleAgentEnabled': setAgentEnabled(el.dataset.agent, el.dataset.enabled !== 'true'); break; case 'loadConvergenceSoak': loadConvergenceSoak(); break; + case 'loadIntegrationOrchestration': loadIntegrationOrchestration(); break; case 'loadReviewOutcomes': loadReviewOutcomes(); break; case 'loadLinearAgentStatus': loadLinearAgentStatus(); break; case 'loadUpstreamWatch': loadUpstreamWatch(); break; @@ -1120,4 +1121,3 @@ el.disabled = false; } } - diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 808ecc9374..a273c7a9f5 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -43849,6 +43849,7 @@

GitHub Projects sync

case 'wsWavefrontSetEnabled': wsWavefrontSetEnabled(el); break; case 'toggleAgentEnabled': setAgentEnabled(el.dataset.agent, el.dataset.enabled !== 'true'); break; case 'loadConvergenceSoak': loadConvergenceSoak(); break; + case 'loadIntegrationOrchestration': loadIntegrationOrchestration(); break; case 'loadReviewOutcomes': loadReviewOutcomes(); break; case 'loadLinearAgentStatus': loadLinearAgentStatus(); break; case 'loadUpstreamWatch': loadUpstreamWatch(); break; @@ -44697,7 +44698,6 @@

GitHub Projects sync

el.disabled = false; } } - // ── Plan review view (#7537) ───────────────────────────────────────── // The governor PLANNING tile links here: openPlanList() enumerates every // plan (GET /api/plans, drafts first) and openPlanReview(epicID) shows one @@ -57769,6 +57769,7 @@

${esc(s.title)}

am = am || {}; rv = rv || {}; const cv = (_configState.data || {}).convergence || {}; + const io = (_configState.data || {}).integration_orchestration || { mode: 'off', scopes: [] }; const planOn = f.planFromLabel === true; const formalAvailable = f.formalAvailable === true; const formalMinACMMLevel = f.formalMinACMMLevel || 5; @@ -58769,6 +58770,48 @@

${esc(s.title)}

Checkpoint, triage and recheck settings are applied live; no restart is needed.
+
+
Integration orchestration
+
Default OFF. Coordinates authoring admission above existing preparation and native queues; never replaces CI, review, holds or current-base guards. Advisory cannot block. Enforce requires explicit repository/base scopes and complete runner telemetry. No native queue jumping or GitHub build-window control. 5,000/day is a capacity target, not a measured result.
+
+ +
+
+ +
+
+ ${[ + ['max_active','Active admissions',2,'integration_orchestration.max_active'], + ['max_jobs','Executed jobs per window',20,'integration_orchestration.max_jobs'], + ['max_runner_minutes','Runner minutes per window',60,'integration_orchestration.max_runner_minutes'], + ['max_attempts','Attempt bound',2,'integration_orchestration.max_attempts'], + ['admission_jobs','Job allocation per admission',1,'integration_orchestration.admission_jobs'], + ['admission_runner_minutes','Minute allocation per admission',10,'integration_orchestration.admission_runner_minutes'] + ].map(([key,label,def,path]) => + `
`).join('')} +
+
+ ${[ + ['lease_ttl','Reservation TTL (minutes)',5,'integration_orchestration.lease_ttl'], + ['window','Accounting window (minutes)',60,'integration_orchestration.window'], + ['max_observation_age','Maximum observation age (minutes)',30,'integration_orchestration.max_observation_age'] + ].map(([key,label,def,path]) => + `
`).join('')} +
+
+ +
+
${[ + 'integration_orchestration.policies[].repo', 'integration_orchestration.policies[].base', + 'integration_orchestration.policies[].engine', 'integration_orchestration.policies[].workflow', + 'integration_orchestration.policies[].blob', 'integration_orchestration.policies[].app_id', + 'integration_orchestration.policies[].jobs[]' + ].map(path => yamlHint(path)).join(' ')} Disabling releases only orchestration reservations; existing claims and native entries remain. Missing group membership, policy or producer proof stays unknown.
+ +

+        
Convergence Rollout ${settingHelpMark('settings', 'features', 'Convergence Rollout', `Convergence checks whether work Hive is about to kick is actually admissible right now — blocker still open, dependency unmerged, evidence stale — so agents stop burning tokens rediscovering "not ready yet". It gates candidates only; it never plans or schedules.`)}
@@ -58785,7 +58828,6 @@

${esc(s.title)}

`)}`; } - function markReleaseSentinelList(key, value) { markDirty('release-sentinel', key, splitConfigLines(value)); } @@ -60678,6 +60720,7 @@

${esc(s.title)}

: 'No soak telemetry recorded yet — it accumulates one row per eval cycle, so give it a few cycles.'; return; } + const groups = {}; for (const e of entries) { const k = `${e.commit || '?'} / ${e.mode || '?'} (gen ${e.generation || 0})`; @@ -60713,6 +60756,32 @@

${esc(s.title)}

} } + async function loadIntegrationOrchestration() { + const out = document.getElementById('integration-orchestration-summary'); + if (!out) return; + out.textContent = 'Loading authoritative observations…'; + try { + const res = await fetch('/api/integration-orchestration', { headers: _inceptionAuthHeaders() }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const data = await res.json(); + const preparation = await fetch('/api/stage-tracker', { headers: _inceptionAuthHeaders() }); + if (preparation.ok) { + const tracker = await preparation.json(); + data.preparation = { + source: '/api/stage-tracker.base_preparation', + state: Array.isArray(tracker.base_preparation) ? 'observed' : 'unavailable', + queues: Array.isArray(tracker.base_preparation) ? tracker.base_preparation : null, + meaning: 'Preparation ownership/backlog only; not green or approved train seats, and not native queue membership.' + }; + } else { + data.preparation = { state: 'unavailable', reason: `Tracker HTTP ${preparation.status}` }; + } + out.textContent = JSON.stringify(data, null, 2); + } catch (e) { + out.textContent = `Integration observations unknown: ${e.message}`; + } + } + // ── Upstream watch editor + divergence view (#9969, #11211) ───────────── // loadUpstreamWatch paints the editor (on/off, poll interval, add fork, // per-fork settings) and the per-repo divergence table from @@ -60876,7 +60945,6 @@

${esc(s.title)}

if (!await hiveConfirm({ title: `Stop watching ${repo}?`, message: 'The watch stops polling its upstream. Recorded state and issues already filed are kept.', confirmLabel: 'Stop watching', danger: true })) return; putUpstreamWatch({ repos: { [repo]: null } }, `Stopped watching ${repo}`); } - // ── Linear agent (RFC #4492 Part 2) ───────────────────────────────────── // loadLinearAgentStatus paints install/workspace/session state from the // owner-only status endpoint into the Features tab's Linear Agent panel. @@ -61647,6 +61715,33 @@

${esc(s.title)}

}); if (!slaRes.ok) throw new Error(await saveErrorMessage(slaRes)); _configState.data.sla = await slaRes.json().catch(() => _configState.data.sla); + } else if (_configState.type === 'governor' && section === 'integration_orchestration') { + const body = Object.assign({}, (_configState.data || {}).integration_orchestration || {}, values); + const integrationMinuteNanoseconds = 60 * 1e9; + for (const key of ['lease_ttl', 'window', 'max_observation_age']) { + if (Object.prototype.hasOwnProperty.call(body, key + '_minutes')) { + body[key] = Number(body[key + '_minutes']) * integrationMinuteNanoseconds; + delete body[key + '_minutes']; + } + } + if (Object.prototype.hasOwnProperty.call(body, 'policy_json')) { + body.policies = JSON.parse(body.policy_json); + if (!Array.isArray(body.policies)) throw new Error('Integration workflow contracts must be a JSON array'); + delete body.policy_json; + } + if (Object.prototype.hasOwnProperty.call(body, 'scope_lines')) { + body.scopes = String(body.scope_lines).split('\n').map(s => s.trim()).filter(Boolean).map(s => { + const at = s.indexOf('#'); + if (at < 1 || at === s.length - 1) throw new Error('Each integration scope must be owner/repo#base'); + return { repo: s.slice(0, at), base: s.slice(at + 1) }; + }); + delete body.scope_lines; + } + const ioRes = await fetch('/api/config/integration-orchestration', { + method: 'PUT', headers: _inceptionAuthHeaders(), body: JSON.stringify(body) + }); + if (!ioRes.ok) throw new Error(await saveErrorMessage(ioRes)); + _configState.data.integration_orchestration = await ioRes.json(); } else if (_configState.type === 'governor' && section === 'convergence') { // convergence is a TOP-LEVEL config block, not under governor — // it saves through its own owner-only /api/config/convergence @@ -61810,7 +61905,6 @@

${esc(s.title)}

}) .catch(() => {}); } - // --- Chat rich formatting --- function sanitizeChatText(raw) { return String(raw || '') diff --git a/src/pkg/github/automerge/automerge_merge_train.go b/src/pkg/github/automerge/automerge_merge_train.go index 7b54f1ba22..161648eb29 100644 --- a/src/pkg/github/automerge/automerge_merge_train.go +++ b/src/pkg/github/automerge/automerge_merge_train.go @@ -568,6 +568,9 @@ func (c *Engine) planMergeTrains(ctx context.Context, lane, displayRepo, owner, if c.hotspotRecorder != nil { c.hotspotRecorder.RecordPRFiles(owner+"/"+repo, pr.GetNumber(), files, pr.GetCreatedAt().Time) } + if c.integrationFilesObserved != nil { + c.integrationFilesObserved(owner+"/"+repo, pr.GetHead().GetSHA(), append([]string(nil), files...)) + } members = append(members, mergeTrainMember{ Number: pr.GetNumber(), Base: base, diff --git a/src/pkg/github/automerge/automerge_native_queue.go b/src/pkg/github/automerge/automerge_native_queue.go index b781064595..a762e9bbcf 100644 --- a/src/pkg/github/automerge/automerge_native_queue.go +++ b/src/pkg/github/automerge/automerge_native_queue.go @@ -259,6 +259,12 @@ func (c *Engine) tryNativeQueue(ctx context.Context, pr *gh.PullRequest, claim e } } if !state.IsInMergeQueue { + if c.integrationAdmission != nil { + reason, err := c.integrationAdmission(ctx, claim.Repo, pr) + if reason != "" || err != nil { + return "integration-admission: " + reason, err + } + } // Each removal starts a new queue attempt, even at the same head. // The previous applied journal entry must never suppress re-enqueue. cycle := "" diff --git a/src/pkg/github/automerge/automerge_sweep.go b/src/pkg/github/automerge/automerge_sweep.go index 887f5bfde7..c7cacc073e 100644 --- a/src/pkg/github/automerge/automerge_sweep.go +++ b/src/pkg/github/automerge/automerge_sweep.go @@ -77,6 +77,9 @@ const autoMergeReasonLabelNeedsRebase = "label:needs-rebase" // Options carries policy dependencies owned by the caller. type Options struct { + // IntegrationAdmission may withhold a NEW native enqueue for a scoped CI + // budget. It never replaces any gate, cancels work, or reorders the queue. + IntegrationAdmission func(context.Context, string, *gh.PullRequest) (string, error) // BasePreparation persists front-only final preparation across all lanes. // nil uses the shared durable store. BasePreparation *BasePreparationStore @@ -246,7 +249,8 @@ type Options struct { // conflicts it asks the resolver to repair, so the merge-hotspot // advisor (pkg/hotspot) derives its evidence without polling GitHub // again; nil records nothing. - HotspotRecorder HotspotRecorder + HotspotRecorder HotspotRecorder + IntegrationFilesObserved func(repo, head string, files []string) } // Engine owns the automerge sweep policy state. @@ -267,6 +271,7 @@ type Engine struct { claimants map[string][]*gh.PullRequest nativeMergeQueuePolicy func() NativeMergeQueuePolicy + integrationAdmission func(context.Context, string, *gh.PullRequest) (string, error) nativeQueueProbeMu sync.Mutex nativeQueueProbes map[string]nativeQueueProbe @@ -394,11 +399,12 @@ type Engine struct { // sweepClock records the lanes' passes for the dashboard (sweep_clock.go). sweepClock *SweepClock - mergeTrainPolicy func() MergeTrainPolicy - rebaseResolver RebaseResolver - rebaseEpisode func(repo string, number int) (RebaseEpisode, bool) - fixDispatch func(repo string, number int, head string) (review.FixDispatchStatus, bool) - hotspotRecorder HotspotRecorder + mergeTrainPolicy func() MergeTrainPolicy + rebaseResolver RebaseResolver + rebaseEpisode func(repo string, number int) (RebaseEpisode, bool) + fixDispatch func(repo string, number int, head string) (review.FixDispatchStatus, bool) + hotspotRecorder HotspotRecorder + integrationFilesObserved func(repo, head string, files []string) mergeTrainState } @@ -456,10 +462,12 @@ func New(transport Transport, opts Options) *Engine { sweepTrigger: opts.SweepTrigger, mergeTrainPolicy: mergeTrainPolicyOption(opts), nativeMergeQueuePolicy: opts.NativeMergeQueue, + integrationAdmission: opts.IntegrationAdmission, rebaseResolver: opts.RebaseResolver, rebaseEpisode: opts.RebaseEpisode, fixDispatch: opts.FixDispatchStatus, hotspotRecorder: opts.HotspotRecorder, + integrationFilesObserved: opts.IntegrationFilesObserved, evaluatedHeads: make(map[string]string), requiredChecksFallbackWarned: make(map[string]bool), } diff --git a/src/pkg/github/automerge/integration_admission_test.go b/src/pkg/github/automerge/integration_admission_test.go new file mode 100644 index 0000000000..ecbf372acb --- /dev/null +++ b/src/pkg/github/automerge/integration_admission_test.go @@ -0,0 +1,56 @@ +package automerge + +import ( + "context" + "errors" + "strings" + "testing" + + gh "github.com/google/go-github/v72/github" +) + +func TestIntegrationAdmissionOnlyBeforeNewJournaledEnqueue(t *testing.T) { + for _, mode := range []string{"deny", "unavailable", "admit", "already-queued", "native-off"} { + t.Run(mode, func(t *testing.T) { + engine, transport, pr, claim := nativeQueueEngine() + boundary := &recordingBoundary{} + engine.mutation = boundary + calls := 0 + engine.integrationAdmission = func(_ context.Context, repo string, observed *gh.PullRequest) (string, error) { + calls++ + if repo != claim.Repo || observed != pr { + t.Fatal("wrong live candidate") + } + if mode == "deny" { + return "runner budget exhausted", nil + } + if mode == "unavailable" { + return "unknown telemetry", errors.New("unavailable") + } + return "", nil + } + if mode == "already-queued" { + transport.state.IsInMergeQueue = true + transport.state.MergeQueueEntry = queueEntry(testHiveAppBotLogin) + } + if mode == "native-off" { + engine.nativeMergeQueuePolicy = nil + } + reason, err := engine.tryNativeQueue(context.Background(), pr, claim) + if mode == "deny" || mode == "unavailable" { + if transport.enqueues != 0 || len(boundary.claims) != 0 || !strings.Contains(reason, "integration") { + t.Fatalf("%s %v enqueues=%d", reason, err, transport.enqueues) + } + } else if mode == "admit" && (err != nil || transport.enqueues != 1 || len(boundary.claims) != 1) { + t.Fatalf("existing journal writer not used: %s %v %+v", reason, err, boundary.claims) + } + wantCalls := 1 + if mode == "already-queued" || mode == "native-off" { + wantCalls = 0 + } + if calls != wantCalls { + t.Fatalf("calls=%d want %d", calls, wantCalls) + } + }) + } +} diff --git a/src/pkg/github/ci_failure_signal.go b/src/pkg/github/ci_failure_signal.go index 718b7dfb3e..52fe9d2792 100644 --- a/src/pkg/github/ci_failure_signal.go +++ b/src/pkg/github/ci_failure_signal.go @@ -26,6 +26,8 @@ type CIFailurePR struct { // check_suite / status delivery, carried on WebhookEventResult.CIFailure so // the receiver can react now rather than on the next sweep. type WebhookCIFailure struct { + SuiteID int64 `json:"suite_id,omitempty"` + CheckRunID int64 `json:"check_run_id,omitempty"` HeadSHA string `json:"head_sha"` Check string `json:"check"` Conclusion string `json:"conclusion"` @@ -37,17 +39,32 @@ type WebhookCIFailure struct { // observation. Agent is the hive agent the PR is attributed to (empty for a // contributor PR or an unattributed one). type CIFailureSignal struct { - Repo string `json:"repo"` - Number int `json:"number"` - HeadSHA string `json:"head_sha"` - Check string `json:"check"` - Conclusion string `json:"conclusion,omitempty"` - DetailsURL string `json:"details_url,omitempty"` - Title string `json:"title,omitempty"` - Author string `json:"author,omitempty"` - HiveAuthored bool `json:"hive_authored"` - Agent string `json:"agent,omitempty"` - At time.Time `json:"at"` + Repo string `json:"repo"` + Number int `json:"number"` + HeadSHA string `json:"head_sha"` + Check string `json:"check"` + Conclusion string `json:"conclusion,omitempty"` + DetailsURL string `json:"details_url,omitempty"` + Title string `json:"title,omitempty"` + Author string `json:"author,omitempty"` + HiveAuthored bool `json:"hive_authored"` + Agent string `json:"agent,omitempty"` + At time.Time `json:"at"` + Integration *IntegrationFailureContext `json:"integration,omitempty"` +} + +type IntegrationFailureContext struct { + CandidateID string `json:"candidate_id"` + TestedSHA string `json:"tested_sha"` + TestedTree string `json:"tested_tree"` + BaseSHA string `json:"base_sha"` + Members []string `json:"ordered_members"` + Policy string `json:"policy"` + Producer string `json:"producer"` + Attempt int `json:"attempt"` + RunID int64 `json:"run_id,omitempty"` + SuiteID int64 `json:"suite_id,omitempty"` + Source string `json:"source,omitempty"` } // CIFailureSignalsStatus is the /api/status view of the store. diff --git a/src/pkg/github/integration_attestation.go b/src/pkg/github/integration_attestation.go new file mode 100644 index 0000000000..b93cc76c94 --- /dev/null +++ b/src/pkg/github/integration_attestation.go @@ -0,0 +1,181 @@ +package github + +import ( + "archive/zip" + "bytes" + "context" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/config" +) + +const ( + integrationSubjectArtifactPrefix = "hive-integration-subject-v1-" + integrationSubjectFile = "subject.json" + integrationArtifactMaxBytes = 2 << 20 + integrationSubjectMaxBytes = 64 << 10 + integrationArtifactTimeout = 15 * time.Second + integrationOIDBytes = 20 +) + +func integrationOID(value string) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == integrationOIDBytes +} + +// DirectIntegrationAttestation is supplied by the pinned workflow itself. +// A run's head SHA alone does not establish its actual checkout or tested tree. +type DirectIntegrationAttestation struct { + Version int `json:"version"` + Repo string `json:"repo"` + Base string `json:"base"` + BaseSHA string `json:"base_sha"` + TestedSHA string `json:"tested_sha"` + TestedTree string `json:"tested_tree"` + RunID int64 `json:"run_id"` + Attempt int `json:"attempt"` + Policy string `json:"policy"` + Members []DirectIntegrationMember `json:"members"` +} + +type DirectIntegrationMember struct { + PR int `json:"pr"` + Head string `json:"head"` +} + +// ReadDirectIntegrationAttestation never produces a candidate or executes Git. +// The external pinned producer must publish the actual checkout identity. +func (c *Client) ReadDirectIntegrationAttestation(ctx context.Context, policy config.IntegrationPolicy, run *gh.WorkflowRun) (DirectIntegrationAttestation, error) { + var out DirectIntegrationAttestation + if run == nil || run.GetEvent() != "pull_request" || run.GetID() <= 0 || run.GetRunAttempt() < 1 || len(run.PullRequests) != 1 || + run.PullRequests[0] == nil || run.GetStatus() != "completed" || policy.Engine != "direct" { + return out, fmt.Errorf("direct tested-subject producer unavailable") + } + request, err := c.client.NewRequest("GET", fmt.Sprintf("repos/%s/actions/runs/%d/artifacts?per_page=%d", policy.Repo, run.GetID(), integrationGroupMaxEntries), nil) + if err != nil { + return out, err + } + var listing struct { + Artifacts []struct { + ID int64 `json:"id"` + Name string `json:"name"` + Expired bool `json:"expired"` + Size int64 `json:"size_in_bytes"` + WorkflowRun struct { + ID int64 `json:"id"` + HeadSHA string `json:"head_sha"` + } `json:"workflow_run"` + } `json:"artifacts"` + } + response, err := c.client.Do(ctx, request, &listing) + if err != nil { + return out, err + } + if response != nil && response.NextPage != 0 { + return out, fmt.Errorf("direct attestation artifact cohort truncated") + } + var artifactID int64 + name := fmt.Sprintf("%s%d", integrationSubjectArtifactPrefix, run.GetRunAttempt()) + for _, artifact := range listing.Artifacts { + if artifact.Name != name { + continue + } + if artifactID != 0 || artifact.Expired || artifact.ID <= 0 || artifact.Size > integrationArtifactMaxBytes || + artifact.WorkflowRun.ID != run.GetID() || artifact.WorkflowRun.HeadSHA != run.GetHeadSHA() { + return out, fmt.Errorf("direct attestation artifact expired, ambiguous or unbound") + } + artifactID = artifact.ID + } + if artifactID == 0 { + return out, fmt.Errorf("pinned workflow did not publish %s; actual tested subject remains unknown", name) + } + parts := strings.SplitN(policy.Repo, "/", 2) + if len(parts) != 2 { + return out, fmt.Errorf("invalid direct repository") + } + location, _, err := c.client.Actions.DownloadArtifact(ctx, parts[0], parts[1], artifactID, 0) + if err != nil { + return out, err + } + if location == nil || location.Scheme != "https" && !(c.client.BaseURL.Scheme == "http" && location.Scheme == "http" && location.Host == c.client.BaseURL.Host) { + return out, fmt.Errorf("direct attestation download requires HTTPS") + } + download, err := http.NewRequestWithContext(ctx, http.MethodGet, location.String(), nil) + if err != nil { + return out, err + } + // Deliberately do not use the authenticated GitHub transport for an + // expiring artifact-storage URL. + client := &http.Client{Timeout: integrationArtifactTimeout, CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse }} + payload, err := client.Do(download) + if err != nil { + return out, err + } + defer payload.Body.Close() + if payload.StatusCode != http.StatusOK { + return out, fmt.Errorf("direct artifact download HTTP %d", payload.StatusCode) + } + compressed, err := io.ReadAll(io.LimitReader(payload.Body, integrationArtifactMaxBytes+1)) + if err != nil { + return out, err + } + if len(compressed) > integrationArtifactMaxBytes { + return out, fmt.Errorf("direct artifact exceeds compressed bound") + } + out, err = decodeDirectIntegrationAttestation(compressed) + if err != nil { + return out, err + } + pr := run.PullRequests[0] + if out.Version != 1 || !strings.EqualFold(out.Repo, policy.Repo) || out.Base != policy.Base || out.Base != pr.GetBase().GetRef() || + out.BaseSHA != pr.GetBase().GetSHA() || out.RunID != run.GetID() || out.Attempt != run.GetRunAttempt() || out.Policy != policy.Fingerprint() || + !integrationOID(out.BaseSHA) || !integrationOID(out.TestedSHA) || !integrationOID(out.TestedTree) || + len(out.Members) != 1 || out.Members[0].PR != pr.GetNumber() || out.Members[0].Head != pr.GetHead().GetSHA() || !integrationOID(out.Members[0].Head) { + return DirectIntegrationAttestation{}, fmt.Errorf("direct subject is not bound to exact policy, workflow attempt and pull-request event") + } + commit, _, err := c.client.Git.GetCommit(ctx, parts[0], parts[1], out.TestedSHA) + if err != nil { + return DirectIntegrationAttestation{}, err + } + if commit == nil || commit.GetSHA() != out.TestedSHA || commit.GetTree().GetSHA() != out.TestedTree || len(commit.Parents) != 2 { + return DirectIntegrationAttestation{}, fmt.Errorf("direct tested commit/tree not verified") + } + a, b := commit.Parents[0].GetSHA(), commit.Parents[1].GetSHA() + if out.BaseSHA == out.Members[0].Head || !(a == out.BaseSHA && b == out.Members[0].Head || b == out.BaseSHA && a == out.Members[0].Head) { + return DirectIntegrationAttestation{}, fmt.Errorf("direct tested merge parents do not bind current event base and author head") + } + return out, nil +} + +func decodeDirectIntegrationAttestation(compressed []byte) (DirectIntegrationAttestation, error) { + var out DirectIntegrationAttestation + archive, err := zip.NewReader(bytes.NewReader(compressed), int64(len(compressed))) + if err != nil { + return out, err + } + if len(archive.File) != 1 || archive.File[0].Name != integrationSubjectFile || archive.File[0].UncompressedSize64 > integrationSubjectMaxBytes { + return out, fmt.Errorf("direct attestation must contain one bounded subject.json") + } + file, err := archive.File[0].Open() + if err != nil { + return out, err + } + defer file.Close() + decoder := json.NewDecoder(io.LimitReader(file, integrationSubjectMaxBytes+1)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&out); err != nil { + return out, err + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + return out, fmt.Errorf("direct attestation has trailing or oversized content") + } + return out, nil +} diff --git a/src/pkg/github/integration_attestation_test.go b/src/pkg/github/integration_attestation_test.go new file mode 100644 index 0000000000..6f68a2377d --- /dev/null +++ b/src/pkg/github/integration_attestation_test.go @@ -0,0 +1,135 @@ +package github + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/config" +) + +func directSubjectZIP(t *testing.T, files map[string][]byte) []byte { + t.Helper() + var out bytes.Buffer + writer := zip.NewWriter(&out) + for name, content := range files { + file, err := writer.Create(name) + if err != nil { + t.Fatal(err) + } + if _, err := file.Write(content); err != nil { + t.Fatal(err) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return out.Bytes() +} + +func TestDirectIntegrationAttestationRequiresExactProducerAndCheckout(t *testing.T) { + for _, scenario := range []string{"valid", "missing", "expired", "duplicate", "wrong-run", "wrong-attempt", "wrong-policy", "wrong-member", "wrong-tree", "wrong-parents", "truncated", "bad-zip", "bad-json", "unknown-field", "extra-file", "oversized", "download-error", "insecure-location"} { + t.Run(scenario, func(t *testing.T) { + base, head, tested, tree := strings.Repeat("a", 40), strings.Repeat("b", 40), strings.Repeat("c", 40), strings.Repeat("d", 40) + policy := config.IntegrationPolicy{Repo: "o/r", Base: "v5", Engine: "direct", Workflow: ".github/workflows/test.yml", Blob: strings.Repeat("e", 40), AppID: 42, Jobs: []string{"test"}} + run := &gh.WorkflowRun{ID: gh.Ptr(int64(10)), RunAttempt: gh.Ptr(1), Event: gh.Ptr("pull_request"), Status: gh.Ptr("completed"), HeadSHA: gh.Ptr(head), + PullRequests: []*gh.PullRequest{{Number: gh.Ptr(7), Base: &gh.PullRequestBranch{Ref: gh.Ptr("v5"), SHA: gh.Ptr(base)}, Head: &gh.PullRequestBranch{SHA: gh.Ptr(head)}}}} + subject := DirectIntegrationAttestation{Version: 1, Repo: "o/r", Base: "v5", BaseSHA: base, TestedSHA: tested, TestedTree: tree, RunID: 10, Attempt: 1, Policy: policy.Fingerprint(), Members: []DirectIntegrationMember{{PR: 7, Head: head}}} + switch scenario { + case "wrong-attempt": + subject.Attempt = 2 + case "wrong-policy": + subject.Policy = "other-policy" + case "wrong-member": + subject.Members[0].Head = strings.Repeat("f", 40) + } + data, err := json.Marshal(subject) + if err != nil { + t.Fatal(err) + } + switch scenario { + case "bad-json": + data = []byte("malformed") + case "unknown-field": + data = append(data[:len(data)-1], []byte(`,"unreviewed":true}`)...) + case "oversized": + data = bytes.Repeat([]byte(" "), integrationSubjectMaxBytes+1) + } + files := map[string][]byte{integrationSubjectFile: data} + if scenario == "extra-file" { + files["other.json"] = data + } + archive := directSubjectZIP(t, files) + if scenario == "bad-zip" { + archive = []byte("malformed") + } + var api *httptest.Server + api = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case strings.HasSuffix(r.URL.Path, "/artifacts"): + if scenario == "truncated" { + w.Header().Set("Link", `; rel="next"`) + } + if scenario == "missing" { + _, _ = w.Write([]byte(`{"artifacts":[]}`)) + return + } + runID := 10 + if scenario == "wrong-run" { + runID = 11 + } + artifact := fmt.Sprintf(`{"id":20,"name":"%s1","expired":%t,"size_in_bytes":%d,"workflow_run":{"id":%d,"head_sha":%q}}`, integrationSubjectArtifactPrefix, scenario == "expired", len(archive), runID, head) + if scenario == "duplicate" { + artifact = artifact + "," + artifact + } + _, _ = fmt.Fprintf(w, `{"artifacts":[%s]}`, artifact) + case strings.HasSuffix(r.URL.Path, "/artifacts/20/zip"): + location := api.URL + "/archive" + if scenario == "insecure-location" { + location = "http://example.invalid/archive" + } + http.Redirect(w, r, location, http.StatusFound) + case r.URL.Path == "/archive": + if r.Header.Get("Authorization") != "" { + t.Error("GitHub token was sent to artifact storage") + } + if scenario == "download-error" { + w.WriteHeader(http.StatusBadGateway) + return + } + _, _ = w.Write(archive) + case strings.Contains(r.URL.Path, "/git/commits/"): + actualTree, parent := tree, head + if scenario == "wrong-tree" { + actualTree = strings.Repeat("f", 40) + } + if scenario == "wrong-parents" { + parent = strings.Repeat("f", 40) + } + _, _ = fmt.Fprintf(w, `{"sha":%q,"tree":{"sha":%q},"parents":[{"sha":%q},{"sha":%q}]}`, tested, actualTree, base, parent) + default: + t.Errorf("unexpected request %s", r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer api.Close() + client := NewClient("token", "o", []string{"r"}, nil, api.URL) + got, err := client.ReadDirectIntegrationAttestation(context.Background(), policy, run) + if scenario == "valid" { + if err != nil || got.TestedSHA != tested || got.TestedTree != tree { + t.Fatalf("%+v %v", got, err) + } + } else if err == nil { + t.Fatalf("unbound attestation accepted: %s %+v", scenario, got) + } + }) + } +} diff --git a/src/pkg/github/integration_group.go b/src/pkg/github/integration_group.go new file mode 100644 index 0000000000..60b1cbb804 --- /dev/null +++ b/src/pkg/github/integration_group.go @@ -0,0 +1,115 @@ +package github + +import ( + "context" + "fmt" + "strings" +) + +const integrationGroupMaxEntries = 100 + +// NativeIntegrationGroup is an observation of GitHub's documented queue prefix, +// not a queue-entry ID repurposed as a group ID and not a merge authorization. +type NativeIntegrationGroup struct { + Repo string + Base string + BaseSHA string + HeadRef string + HeadSHA string + Tree string + Members []NativeIntegrationMember +} + +type NativeIntegrationMember struct { + Entry string + Position int + PR int + Head string + BaseCommit string + HeadCommit string +} + +type nativeIntegrationQueue struct { + Repository struct { + MergeQueue *struct { + Entries struct { + PageInfo struct{ HasNextPage bool } + Nodes []struct { + ID string + Position int + BaseCommit struct{ OID string } + HeadCommit struct{ OID string } + PullRequest struct { + Number int + HeadRefOID string + BaseRefName string + } + } + } + } + } +} + +func (c *Client) ReadNativeIntegrationGroup(ctx context.Context, repo, base, baseSHA, headRef, headSHA string) (NativeIntegrationGroup, error) { + out := NativeIntegrationGroup{Repo: repo, Base: base, BaseSHA: baseSHA, HeadRef: headRef, HeadSHA: headSHA} + parts := strings.SplitN(repo, "/", 2) + if len(parts) != 2 || base == "" || !integrationOID(headSHA) || baseSHA != "" && !integrationOID(baseSHA) || !strings.HasPrefix(headRef, "refs/heads/gh-readonly-queue/"+base+"/") { + return out, fmt.Errorf("native integration group identity incomplete") + } + var q nativeIntegrationQueue + err := c.graphQL(ctx, `query($owner:String!,$name:String!,$branch:String!,$limit:Int!) { + repository(owner:$owner,name:$name) { mergeQueue(branch:$branch) { + entries(first:$limit) { pageInfo { hasNextPage } nodes { + id position baseCommit { oid } headCommit { oid } + pullRequest { number headRefOid baseRefName } + } } } } }`, map[string]any{"owner": parts[0], "name": parts[1], "branch": base, "limit": integrationGroupMaxEntries}, &q) + if err != nil { + return out, err + } + queue := q.Repository.MergeQueue + if queue == nil || queue.Entries.PageInfo.HasNextPage { + return out, fmt.Errorf("native queue membership unavailable or exceeds bounded page") + } + // Bounded missed-event reconciliation can recover the root from the + // authoritative queue chain, never from a run's author head or tree guess. + if baseSHA == "" && len(queue.Entries.Nodes) > 0 { + baseSHA = queue.Entries.Nodes[0].BaseCommit.OID + out.BaseSHA = baseSHA + } + if !integrationOID(baseSHA) { + return out, fmt.Errorf("native root SHA unavailable") + } + parent := baseSHA + found := false + for index, n := range queue.Entries.Nodes { + if n.Position != index+1 || n.BaseCommit.OID != parent || n.PullRequest.BaseRefName != base || + n.ID == "" || n.PullRequest.Number < 1 || !integrationOID(n.PullRequest.HeadRefOID) || !integrationOID(n.HeadCommit.OID) { + return out, fmt.Errorf("native queue prefix chain is incomplete or changed") + } + out.Members = append(out.Members, NativeIntegrationMember{Entry: n.ID, Position: n.Position, PR: n.PullRequest.Number, Head: n.PullRequest.HeadRefOID, BaseCommit: n.BaseCommit.OID, HeadCommit: n.HeadCommit.OID}) + parent = n.HeadCommit.OID + if parent == headSHA { + found = true + break + } + } + if !found { + return out, fmt.Errorf("tested group is not an authoritative contiguous queue prefix") + } + commit, _, err := c.client.Git.GetCommit(ctx, parts[0], parts[1], headSHA) + if err != nil { + return out, err + } + out.Tree = commit.GetTree().GetSHA() + if commit.GetSHA() != headSHA || !integrationOID(out.Tree) { + return out, fmt.Errorf("native tested commit/tree unavailable") + } + ref, _, err := c.client.Git.GetRef(ctx, parts[0], parts[1], strings.TrimPrefix(headRef, "refs/")) + if err != nil { + return out, err + } + if ref.GetObject().GetSHA() != headSHA { + return out, fmt.Errorf("native group synthetic ref changed during observation") + } + return out, nil +} diff --git a/src/pkg/github/integration_group_test.go b/src/pkg/github/integration_group_test.go new file mode 100644 index 0000000000..9a8297e50c --- /dev/null +++ b/src/pkg/github/integration_group_test.go @@ -0,0 +1,61 @@ +package github + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestNativeIntegrationGroupExactPrefix(t *testing.T) { + for _, mode := range []string{"valid", "gap", "truncated", "moved-ref", "missing-group", "read-error"} { + t.Run(mode, func(t *testing.T) { + base, head, tree, member := strings.Repeat("a", 40), strings.Repeat("b", 40), strings.Repeat("c", 40), strings.Repeat("d", 40) + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/graphql": + if mode == "read-error" { + w.WriteHeader(http.StatusBadGateway) + return + } + if mode == "missing-group" { + _, _ = w.Write([]byte(`{"data":{"repository":{"mergeQueue":null}}}`)) + return + } + position := 1 + if mode == "gap" { + position = 2 + } + _, _ = fmt.Fprintf(w, `{"data":{"repository":{"mergeQueue":{"entries":{"pageInfo":{"hasNextPage":%t},"nodes":[{"id":"entry","position":%d,"baseCommit":{"oid":%q},"headCommit":{"oid":%q},"pullRequest":{"number":7,"headRefOid":%q,"baseRefName":"v5"}}]}}}}}`, mode == "truncated", position, base, head, member) + case strings.Contains(r.URL.Path, "/git/commits/"): + _, _ = fmt.Fprintf(w, `{"sha":%q,"tree":{"sha":%q}}`, head, tree) + case strings.Contains(r.URL.Path, "/git/ref/"): + if mode == "moved-ref" { + head = strings.Repeat("f", 40) + } + _, _ = fmt.Fprintf(w, `{"object":{"sha":%q}}`, head) + default: + t.Errorf("unexpected request %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer api.Close() + c := NewClient("token", "o", []string{"r"}, nil, api.URL) + got, err := c.ReadNativeIntegrationGroup(context.Background(), "o/r", "v5", base, "refs/heads/gh-readonly-queue/v5/pr-7", head) + if mode == "valid" { + if err != nil || got.Tree != tree || len(got.Members) != 1 || got.Members[0].Head != member { + t.Fatalf("%+v %v", got, err) + } + } else if err == nil { + t.Fatal("incomplete or changed group accepted") + } + }) + } + c := NewClient("token", "o", []string{"r"}, nil, "https://example.invalid") + if _, err := c.ReadNativeIntegrationGroup(context.Background(), "bad", "", "", "", ""); err == nil { + t.Fatal("incomplete identity accepted") + } +} diff --git a/src/pkg/github/integration_proof.go b/src/pkg/github/integration_proof.go new file mode 100644 index 0000000000..5e50a8970e --- /dev/null +++ b/src/pkg/github/integration_proof.go @@ -0,0 +1,130 @@ +package github + +import ( + "context" + "fmt" + "strconv" + "strings" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/config" +) + +// IntegrationWorkflowProof is a receipt for ONE pinned workflow, not an +// assertion that all CI/review/trust/merge gates have passed. +type IntegrationWorkflowProof struct { + Passed bool + Rejected bool + Reason string + RunID int64 + SuiteID int64 + CheckIDs []int64 +} + +func (c *Client) ReadIntegrationWorkflowProof(ctx context.Context, policy config.IntegrationPolicy, run *gh.WorkflowRun) (IntegrationWorkflowProof, error) { + out := IntegrationWorkflowProof{Reason: "workflow evidence incomplete"} + parts := strings.SplitN(policy.Repo, "/", 2) + if len(parts) != 2 || run == nil || run.GetRunAttempt() < 1 || run.GetID() <= 0 || run.GetCheckSuiteID() <= 0 || run.GetWorkflowID() <= 0 { + return out, fmt.Errorf("workflow observation identity incomplete") + } + out.RunID, out.SuiteID = run.GetID(), run.GetCheckSuiteID() + event := "pull_request" + if policy.Engine == "native" { + event = "merge_group" + } + if run.GetEvent() != event || strings.Split(run.GetPath(), "@")[0] != policy.Workflow || + !strings.EqualFold(run.GetRepository().GetFullName(), policy.Repo) || run.GetHeadRepository().GetFullName() == "" || + policy.Engine == "native" && !strings.EqualFold(run.GetHeadRepository().GetFullName(), policy.Repo) { + return out, fmt.Errorf("workflow event, path or source repository not bound to policy") + } + content, _, _, err := c.client.Repositories.GetContents(ctx, parts[0], parts[1], policy.Workflow, &gh.RepositoryContentGetOptions{Ref: run.GetHeadSHA()}) + if err != nil { + return out, err + } + if content == nil || content.GetSHA() != policy.Blob { + return out, fmt.Errorf("tested workflow does not match pinned policy blob") + } + trusted, _, _, err := c.client.Repositories.GetContents(ctx, parts[0], parts[1], policy.Workflow, &gh.RepositoryContentGetOptions{Ref: policy.Base}) + if err != nil { + return out, err + } + if trusted == nil || trusted.GetSHA() != policy.Blob { + return out, fmt.Errorf("pinned workflow is not present on the trusted target base") + } + request, err := c.client.NewRequest("GET", fmt.Sprintf("repos/%s/actions/runs/%d/attempts/%d/jobs?per_page=%d", policy.Repo, run.GetID(), run.GetRunAttempt(), integrationGroupMaxEntries), nil) + if err != nil { + return out, err + } + jobs := new(gh.Jobs) + response, err := c.client.Do(ctx, request, jobs) + if err != nil { + return out, err + } + if response != nil && response.NextPage != 0 { + return out, fmt.Errorf("workflow job proof exceeds bounded page") + } + checks, response, err := c.client.Checks.ListCheckRunsForRef(ctx, parts[0], parts[1], run.GetHeadSHA(), &gh.ListCheckRunsOptions{ListOptions: gh.ListOptions{PerPage: integrationGroupMaxEntries}}) + if err != nil { + return out, err + } + if checks == nil || response != nil && response.NextPage != 0 { + return out, fmt.Errorf("check proof unavailable or truncated") + } + if err := bindIntegrationWorkflowJobs(policy, run, jobs.Jobs, checks.CheckRuns, &out); err != nil { + return out, err + } + return out, nil +} + +func bindIntegrationWorkflowJobs(policy config.IntegrationPolicy, run *gh.WorkflowRun, jobs []*gh.WorkflowJob, checks []*gh.CheckRun, out *IntegrationWorkflowProof) error { + if run.GetStatus() != "completed" { + out.Reason = "pinned integration workflow still running" + return nil + } + for _, name := range policy.Jobs { + var job *gh.WorkflowJob + for _, j := range jobs { + if j.GetName() == name { + if job != nil { + return fmt.Errorf("ambiguous required integration job %q", name) + } + job = j + } + } + if job == nil || job.GetRunID() != run.GetID() || job.GetHeadSHA() != run.GetHeadSHA() { + return fmt.Errorf("required integration job %q missing exact run/head binding", name) + } + url := strings.Split(strings.TrimRight(job.GetCheckRunURL(), "/"), "/") + id, err := strconv.ParseInt(url[len(url)-1], 10, 64) + if err != nil { + return fmt.Errorf("integration job missing check identity") + } + var check *gh.CheckRun + for _, candidate := range checks { + if candidate.GetID() == id { + check = candidate + } + } + if check == nil || check.GetHeadSHA() != run.GetHeadSHA() || check.GetCheckSuite().GetID() != run.GetCheckSuiteID() || + check.GetApp().GetID() != policy.AppID || check.GetName() != name { + return fmt.Errorf("integration check app/suite/job/head binding unavailable") + } + out.CheckIDs = append(out.CheckIDs, id) + if job.GetStatus() != "completed" || check.GetStatus() != "completed" { + out.Reason = "integration job still running" + return nil + } + if job.GetConclusion() != "success" || check.GetConclusion() != "success" { + out.Rejected = true + out.Reason = "pinned integration workflow job rejected" + return nil + } + } + if run.GetConclusion() != "success" { + out.Rejected = true + out.Reason = "pinned integration workflow rejected" + return nil + } + out.Passed, out.Reason = true, "exact tested workflow blob, producer, suite and required job membership verified; other gates remain separate" + return nil +} diff --git a/src/pkg/github/integration_proof_test.go b/src/pkg/github/integration_proof_test.go new file mode 100644 index 0000000000..99b57a8a6c --- /dev/null +++ b/src/pkg/github/integration_proof_test.go @@ -0,0 +1,109 @@ +package github + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/config" +) + +func TestIntegrationWorkflowProducerBinding(t *testing.T) { + for _, mode := range []string{"passed", "rejected", "running", "wrong-app", "wrong-suite", "wrong-head", "wrong-blob", "untrusted-base", "missing-job", "ambiguous-job", "wrong-event", "wrong-repo", "wrong-target", "wrong-path", "read-error", "truncated"} { + t.Run(mode, func(t *testing.T) { + sha, blob := strings.Repeat("a", 40), strings.Repeat("b", 40) + policy := config.IntegrationPolicy{Repo: "o/r", Base: "v5", Engine: "native", Workflow: ".github/workflows/group.yml", Blob: blob, AppID: 42, Jobs: []string{"policy"}} + run := &gh.WorkflowRun{ID: gh.Ptr(int64(1)), WorkflowID: gh.Ptr(int64(12)), CheckSuiteID: gh.Ptr(int64(2)), RunAttempt: gh.Ptr(1), Event: gh.Ptr("merge_group"), + Path: gh.Ptr(policy.Workflow), HeadSHA: gh.Ptr(sha), HeadRepository: &gh.Repository{FullName: gh.Ptr("o/r")}, Repository: &gh.Repository{FullName: gh.Ptr("o/r")}, Status: gh.Ptr("completed"), Conclusion: gh.Ptr("success")} + if mode == "wrong-event" { + run.Event = gh.Ptr("pull_request_target") + } + if mode == "wrong-repo" { + run.HeadRepository.FullName = gh.Ptr("attacker/r") + } + if mode == "wrong-target" { + run.Repository.FullName = gh.Ptr("attacker/r") + } + if mode == "wrong-path" { + run.Path = gh.Ptr(".github/workflows/other.yml") + } + if mode == "running" { + run.Status = gh.Ptr("in_progress") + } + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if mode == "read-error" { + w.WriteHeader(http.StatusBadGateway) + return + } + if mode == "truncated" { + w.Header().Set("Link", `; rel="next"`) + } + switch { + case strings.Contains(r.URL.Path, "/contents/"): + if mode == "wrong-blob" || mode == "untrusted-base" && r.URL.Query().Get("ref") == policy.Base { + blob = strings.Repeat("f", 40) + } + _, _ = fmt.Fprintf(w, `{"type":"file","sha":%q}`, blob) + case strings.Contains(r.URL.Path, "/jobs"): + job := fmt.Sprintf(`{"id":3,"run_id":1,"name":"policy","head_sha":%q,"check_run_url":"https://api.github.com/repos/o/r/check-runs/4","status":"completed","conclusion":"success"}`, sha) + if mode == "missing-job" { + _, _ = w.Write([]byte(`{"jobs":[]}`)) + return + } + if mode == "ambiguous-job" { + job = job + "," + job + } + _, _ = fmt.Fprintf(w, `{"jobs":[%s]}`, job) + case strings.Contains(r.URL.Path, "/check-runs"): + app, suite, conclusion := 42, 2, "success" + if mode == "wrong-app" { + app = 99 + } + if mode == "wrong-suite" { + suite = 9 + } + if mode == "wrong-head" { + sha = strings.Repeat("f", 40) + } + if mode == "rejected" { + conclusion = "failure" + } + _, _ = fmt.Fprintf(w, `{"check_runs":[{"id":4,"name":"policy","head_sha":%q,"app":{"id":%d},"check_suite":{"id":%d},"status":"completed","conclusion":%q}]}`, sha, app, suite, conclusion) + default: + t.Errorf("unexpected request %s", r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer api.Close() + client := NewClient("token", "o", []string{"r"}, nil, api.URL) + proof, err := client.ReadIntegrationWorkflowProof(context.Background(), policy, run) + switch mode { + case "passed": + if err != nil || !proof.Passed || proof.RunID != 1 || proof.SuiteID != 2 || len(proof.CheckIDs) != 1 { + t.Fatalf("%+v %v", proof, err) + } + case "rejected": + if err != nil || !proof.Rejected || proof.Passed { + t.Fatalf("%+v %v", proof, err) + } + case "running": + if err != nil || proof.Passed || proof.Rejected { + t.Fatalf("%+v %v", proof, err) + } + default: + if err == nil || proof.Passed { + t.Fatalf("unbound proof accepted: %+v %v", proof, err) + } + } + }) + } + client := NewClient("token", "o", []string{"r"}, nil, "https://example.invalid") + if _, err := client.ReadIntegrationWorkflowProof(context.Background(), config.IntegrationPolicy{}, nil); err == nil { + t.Fatal("missing identity accepted") + } +} diff --git a/src/pkg/scheduler/ci_failure_signal_test.go b/src/pkg/scheduler/ci_failure_signal_test.go index c1e36063d1..d8b7a672ec 100644 --- a/src/pkg/scheduler/ci_failure_signal_test.go +++ b/src/pkg/scheduler/ci_failure_signal_test.go @@ -96,6 +96,13 @@ func TestCIFailureKickMessage(t *testing.T) { if got := s.CIFailureKickMessage("advisor", sig); got != "" { t.Errorf("advisory agent must not get a CI kick, got:\n%s", got) } + sig.Integration = &github.IntegrationFailureContext{CandidateID: "exact-candidate", TestedSHA: "tested", TestedTree: "tree", BaseSHA: "base", Policy: "pinned-policy", Producer: "bound-app", Attempt: 2, Members: []string{"77:author-head", "78:other-head"}} + got = s.CIFailureKickMessage("scanner", sig) + for _, want := range []string{"exact-candidate", "tree", "pinned-policy", "77:author-head, 78:other-head", "NOT attribution of blame", "FIX-BEFORE-NEW", "Do not copy green statuses", "cancel shared jobs"} { + if !strings.Contains(got, want) { + t.Errorf("exact candidate kick omitted %q: %s", want, got) + } + } // The regular kick path carries the same signal until ci-failing.json // catches up, so nothing is lost between the webhook and the sweep. if msg := s.addRedPRFixFirst("scanner", "[agent:scanner]\nWORK\n"); !strings.Contains(msg, "#77 test-org/console") { diff --git a/src/pkg/scheduler/policy_overlays.go b/src/pkg/scheduler/policy_overlays.go index 33d2f54fe6..d551ac96aa 100644 --- a/src/pkg/scheduler/policy_overlays.go +++ b/src/pkg/scheduler/policy_overlays.go @@ -223,6 +223,16 @@ func (s *Scheduler) CIFailureKickMessage(agentName string, sig github.CIFailureS } header := fmt.Sprintf("CI failed on %s#%d (%s) — repair it now. Detection was out-of-band (webhook); do not watch or poll CI yourself.\n", sig.Repo, sig.Number, sig.Check) + if context := sig.Integration; context != nil && context.CandidateID != "" { + header = fmt.Sprintf("Integration CI failed on candidate %s, tested commit %s tree %s, base %s, workflow policy %s, producer %s, attempt %d. Ordered PR:head members: %s. Existing ownership routes %s#%d at exact head %s to you first; this is NOT attribution of blame. Inspect the failed job %s (%s), coordinate shared prerequisite repair once, and retain every normal CI/review/trust/hold/current-base gate. Do not copy green statuses, cancel shared jobs, poll CI, or modify other members without their owner.\n", + context.CandidateID, context.TestedSHA, context.TestedTree, context.BaseSHA, context.Policy, context.Producer, context.Attempt, + strings.Join(context.Members, ", "), sig.Repo, sig.Number, sig.HeadSHA, sig.Check, sig.DetailsURL) + if context.RunID > 0 && context.SuiteID > 0 { + header += fmt.Sprintf("Evidence source %s; workflow run %d, suite %d.\n", context.Source, context.RunID, context.SuiteID) + } else { + header += "Workflow run/suite binding unavailable; membership observation is not passing policy proof.\n" + } + } return header + section }