diff --git a/changelog.d/fixed-ci-reporting-reconciliation.md b/changelog.d/fixed-ci-reporting-reconciliation.md new file mode 100644 index 0000000000..b72fa986e1 --- /dev/null +++ b/changelog.d/fixed-ci-reporting-reconciliation.md @@ -0,0 +1 @@ +- Detect verified current-head GitHub Actions workflow/check reporting stalls after bounded consistency observations, expose exact run/attempt provenance and owner action without changing CI gates or automatically rerunning workflows. diff --git a/src/docs/hive-merge.md b/src/docs/hive-merge.md index bc08ec7db8..ff453008f7 100644 --- a/src/docs/hive-merge.md +++ b/src/docs/hive-merge.md @@ -443,6 +443,42 @@ cannot stop external writers and does not relax strict current-base checks. Deploying the changed hive binary activates the policy; no agent, memory, review, CI, hold or authorization setting needs changing. +### CI reporting reconciliation + +An unfinished required check is not green merely because its Actions workflow +ended. PR CI evaluation now reports `ci-reporting-stalled` only after two fresh, +consistent observations separated by a propagation grace. It verifies the exact +current PR head, Actions App/check/suite provenance, owning workflow execution, +latest attempt's job-to-check URL, and a final head/base/attempt consistency read. +A newer execution, changed head, unsupported provider or unavailable API cannot +be used to declare a stall. Ordinary queued/running workflows remain ordinary +pending CI, regardless of elapsed time. Workflow success does not fabricate a +successful check; workflow failure and pending job/check reporting remain +separately visible. + +The structured `CIReportingObservation` and existing reason tooltip identify +the check, run, latest attempt, job, conclusion and observation time. API refusal +is `ci-reporting-unverified`, not green. The stage remains CI-pending-compatible, +and all current-head review, authorization, hold and merge gates remain intact. +Commit-only/base observations retain their existing behavior. No rendering API +polling is added: evaluation uses an identity/PR/head/check-set-scoped, bounded +single-flight cache. `HIVE_CI_REPORTING_RECHECK` and `HIVE_CI_REPORTING_GRACE` +default to five minutes; verified check/workflow events request a refresh bounded +to once per minute. Duplicate deliveries cannot shorten grace; cold restart +requires fresh consistency observations again. + +**Automatic recovery is not implemented or enabled.** A confirmed stall exposes +owner action: inspect the exact workflow's current attempt and logs before one +affected-workflow rerun. Do not rerun all CI or rebase to hide reporting failure. +The existing filesystem mutation ledger only serializes users sharing that +filesystem; shared-CI post-effect comment markers are not atomic cross-hive +claims. Future recovery must prove a pre-mutation episode claim and enforced +designated executor (other hives observe only), with durable ambiguous intent +blocking blind retries. Actions rerun RPCs accept no fencing/idempotency token: +designated routing or a forge CAS alone is not failover fencing or exactly-once +execution, and ambiguous intent must not be automatically stolen after a TTL. +App permission changes, live configuration and experimental integration +activation are not prerequisites for detection and are not made by this change. ### Merge train: ready PRs lead Every lane groups its candidates on one base whose changed files overlap diff --git a/src/pkg/cistatus/cistatus.go b/src/pkg/cistatus/cistatus.go index b1f652ccec..365c89827e 100644 --- a/src/pkg/cistatus/cistatus.go +++ b/src/pkg/cistatus/cistatus.go @@ -12,7 +12,9 @@ import "strings" // ReasonIsPending reports whether a commit-CI skip reason means the checks // are still running or not yet reported, as opposed to failing. func ReasonIsPending(reason string) bool { - return strings.HasSuffix(reason, "-pending") || + return strings.HasPrefix(reason, "ci-reporting-stalled:") || + strings.HasPrefix(reason, "ci-reporting-unverified:") || + strings.HasSuffix(reason, "-pending") || strings.HasPrefix(reason, "pending: ") || strings.HasPrefix(reason, "required-check-missing:") } diff --git a/src/pkg/dashboard/stage_tracker_caption.go b/src/pkg/dashboard/stage_tracker_caption.go index 19e24c2017..36f094faec 100644 --- a/src/pkg/dashboard/stage_tracker_caption.go +++ b/src/pkg/dashboard/stage_tracker_caption.go @@ -306,6 +306,12 @@ func humanizeStageReason(reason string) string { if r == "" { return "" } + if rest, ok := strings.CutPrefix(r, "ci-reporting-stalled:"); ok { + return "CI reporting stalled — " + rest + } + if rest, ok := strings.CutPrefix(r, "ci-reporting-unverified:"); ok { + return "CI reporting unverified — " + rest + } if base, approver := stagelabels.HumanApprovedReason(r); approver != "" { clause := humanizeStageReason(base) if clause == "" { diff --git a/src/pkg/dashboard/stage_tracker_ci_reporting_test.go b/src/pkg/dashboard/stage_tracker_ci_reporting_test.go new file mode 100644 index 0000000000..aabf16868c --- /dev/null +++ b/src/pkg/dashboard/stage_tracker_ci_reporting_test.go @@ -0,0 +1,17 @@ +package dashboard + +import ( + "strings" + "testing" +) + +func TestCIReportingUsesExistingReasonWithoutPromotingStage(t *testing.T) { + for _, prefix := range []string{"stalled", "unverified"} { + reason := "ci-reporting-" + prefix + ":check=\"build-and-test\" run=21 attempt=2 owner-action-required" + got := humanizeStageReason(reason) + if !strings.HasPrefix(got, "CI reporting "+prefix+" — ") || + !strings.Contains(got, "run=21 attempt=2") || !strings.Contains(got, "owner-action-required") { + t.Fatalf("reporting provenance disappeared from existing tooltip reason: %s", got) + } + } +} diff --git a/src/pkg/github/automerge/automerge_sweep_queued.go b/src/pkg/github/automerge/automerge_sweep_queued.go index f23e0d8453..69880f0f85 100644 --- a/src/pkg/github/automerge/automerge_sweep_queued.go +++ b/src/pkg/github/automerge/automerge_sweep_queued.go @@ -350,6 +350,7 @@ func (c *Engine) commitGreenForPRSkipping(ctx context.Context, owner, repo, bran c.warnRequiredChecksFallback(owner, repo, branch) } st, err := hgithub.EvaluateCommitCI(ctx, c.gh, owner, repo, sha, hgithub.CommitCIOptions{ + PRNumber: prNumber, Required: required, RequiredKnown: requiredKnown, RequiredKnownFromConfig: fromConfig, @@ -364,6 +365,10 @@ func (c *Engine) commitGreenForPRSkipping(ctx context.Context, owner, repo, bran if err != nil { return false, st.Reason, err } + if len(st.Reporting) > 0 { + c.info("automerge CI reporting observation", "repo", owner+"/"+repo, "pr", prNumber, + "head", sha, "observations", st.Reporting) + } if prNumber > 0 && len(st.IgnoredMetaContexts) > 0 { c.info("automerge ignoring meta contexts on unprotected base", "pr", prNumber, "contexts", st.IgnoredMetaContexts) } diff --git a/src/pkg/github/ci_reporting.go b/src/pkg/github/ci_reporting.go new file mode 100644 index 0000000000..65253737dd --- /dev/null +++ b/src/pkg/github/ci_reporting.go @@ -0,0 +1,420 @@ +package github + +import ( + "context" + "errors" + "fmt" + "net/url" + "sort" + "strconv" + "strings" + "sync" + "time" + + gh "github.com/google/go-github/v72/github" +) + +const ( + defaultCIReportingRecheck = 5 * time.Minute + defaultCIReportingGrace = 5 * time.Minute + ciReportingEventRecheck = time.Minute + ciReportingMaxEntries = 1024 + ciReportingMaxRunPages = 3 + ciReportingMaxJobPages = 3 + ciReportingMaxChecks = 64 + ciReportingMaxRuns = 12 +) + +// CIReportingObservation describes reporting, never a replacement CI verdict. +// Even a successful owning workflow cannot satisfy an unfinished required check. +type CIReportingObservation struct { + Repo string `json:"repo"` + PRNumber int `json:"pr_number"` + State string `json:"state"` + HeadSHA string `json:"head_sha"` + Check string `json:"check"` + CheckRunID int64 `json:"check_run_id,omitempty"` + CheckStatus string `json:"check_status"` + RunID int64 `json:"run_id,omitempty"` + Attempt int `json:"attempt,omitempty"` + JobID int64 `json:"job_id,omitempty"` + JobStatus string `json:"job_status,omitempty"` + JobConclusion string `json:"job_conclusion,omitempty"` + RunConclusion string `json:"run_conclusion,omitempty"` + RunUpdatedAt time.Time `json:"run_updated_at,omitzero"` + RunURL string `json:"run_url,omitempty"` + ObservedAt time.Time `json:"observed_at"` + Detail string `json:"detail,omitempty"` + Recovery string `json:"recovery"` +} + +type ciReportingEntry struct { + repo, head string + checkedAt time.Time + busy, dirty bool + observations []CIReportingObservation + suspicions map[string]time.Time +} + +type ciReportingCache struct { + mu sync.Mutex + entries map[string]*ciReportingEntry +} + +func newCIReportingCache() *ciReportingCache { + return &ciReportingCache{entries: make(map[string]*ciReportingEntry)} +} + +var sharedCIReportingCache = newCIReportingCache() + +func ciReportingKey(client *gh.Client, owner, repo, sha string, number int, checks []*gh.CheckRun) string { + var fingerprint strings.Builder + for _, check := range checks { + fmt.Fprintf(&fingerprint, "/%d:%s:%s", check.GetID(), check.GetName(), check.GetStatus()) + } + return fmt.Sprintf("%p:%s:%s/%s#%d@%s%s", client, client.BaseURL, + strings.ToLower(owner), strings.ToLower(repo), number, sha, fingerprint.String()) +} + +func (c *ciReportingCache) invalidate(repo, sha string) { + c.mu.Lock() + defer c.mu.Unlock() + for _, entry := range c.entries { + if entry.repo == strings.ToLower(repo) && entry.head == sha { + entry.dirty = true + } + } +} + +func (c *ciReportingCache) observe(ctx context.Context, client *gh.Client, owner, repo, sha string, opts CommitCIOptions, checks []*gh.CheckRun) []CIReportingObservation { + if opts.PRNumber <= 0 || !opts.RequiredKnown || len(checks) == 0 { + return nil + } + if len(checks) > ciReportingMaxChecks { + return []CIReportingObservation{{State: "unverified", HeadSHA: sha, + Detail: "observation-check-capacity", Recovery: "owner-action-required"}} + } + sort.Slice(checks, func(i, j int) bool { + if checks[i].GetName() != checks[j].GetName() { + return checks[i].GetName() < checks[j].GetName() + } + return checks[i].GetID() < checks[j].GetID() + }) + now := time.Now() + if opts.Now != nil { + now = opts.Now() + } + recheck := durationFromEnv("HIVE_CI_REPORTING_RECHECK", defaultCIReportingRecheck) + grace := durationFromEnv("HIVE_CI_REPORTING_GRACE", defaultCIReportingGrace) + if recheck <= 0 { + recheck = defaultCIReportingRecheck + } + if grace <= 0 { + grace = defaultCIReportingGrace + } + key := ciReportingKey(client, owner, repo, sha, opts.PRNumber, checks) + c.mu.Lock() + entry := c.entries[key] + if entry != nil { + if entry.busy { + out := append([]CIReportingObservation(nil), entry.observations...) + c.mu.Unlock() + return out + } + interval := recheck + if entry.dirty && interval > ciReportingEventRecheck { + interval = ciReportingEventRecheck + } + if !now.Before(entry.checkedAt) && now.Sub(entry.checkedAt) < interval { + out := append([]CIReportingObservation(nil), entry.observations...) + c.mu.Unlock() + return out + } + if now.Before(entry.checkedAt) { + entry.suspicions = nil + } + } else { + if len(c.entries) >= ciReportingMaxEntries { + var oldestKey string + var oldest time.Time + for candidate, value := range c.entries { + if !value.busy && (oldestKey == "" || value.checkedAt.Before(oldest)) { + oldestKey, oldest = candidate, value.checkedAt + } + } + if oldestKey == "" { + c.mu.Unlock() + return []CIReportingObservation{{State: "unverified", HeadSHA: sha, ObservedAt: now, + Detail: "observation-capacity", Recovery: "owner-action-required"}} + } + delete(c.entries, oldestKey) + } + entry = &ciReportingEntry{repo: strings.ToLower(owner + "/" + repo), head: sha} + c.entries[key] = entry + } + entry.busy = true + entry.dirty = false + c.mu.Unlock() + + observations := observeCIReporting(ctx, client, owner, repo, sha, opts.PRNumber, checks, now) + c.mu.Lock() + defer c.mu.Unlock() + next := make(map[string]time.Time) + for i := range observations { + observation := &observations[i] + if observation.State != "contradiction" { + continue + } + signature := fmt.Sprintf("%d:%d:%d:%d:%s:%s:%s:%s:%s", observation.CheckRunID, + observation.RunID, observation.Attempt, observation.JobID, observation.CheckStatus, + observation.JobStatus, observation.JobConclusion, observation.RunConclusion, + observation.RunUpdatedAt.UTC().Format(time.RFC3339Nano)) + first, seen := entry.suspicions[signature] + if !seen || now.Before(first) { + first = now + } + next[signature] = first + if seen && now.Sub(first) >= grace { + observation.State = "stalled" + observation.Recovery = "owner-action-required" + observation.Detail = "terminal workflow still reports an unfinished required check; inspect the latest attempt before one owner-authorized affected-workflow rerun" + } else { + observation.State = "propagation-grace" + observation.Recovery = "awaiting-consistency" + observation.Detail = "terminal workflow/check mismatch awaiting an independent consistency observation" + } + } + entry.checkedAt, entry.busy = now, false + entry.suspicions = next + entry.observations = append([]CIReportingObservation(nil), observations...) + return observations +} + +func observeCIReporting(ctx context.Context, client *gh.Client, owner, repo, sha string, number int, checks []*gh.CheckRun, now time.Time) []CIReportingObservation { + ctx = WithRESTCaller(ctx, "hive:ci_reporting") + out := make([]CIReportingObservation, len(checks)) + actions := false + for i, check := range checks { + out[i] = CIReportingObservation{Repo: owner + "/" + repo, PRNumber: number, + State: "unverified", HeadSHA: sha, Check: check.GetName(), + CheckRunID: check.GetID(), CheckStatus: check.GetStatus(), ObservedAt: now, + Detail: "missing-actions-provenance", Recovery: "owner-inspection-required"} + if check.GetApp().GetSlug() != "github-actions" { + if check.GetApp().GetSlug() != "" { + out[i].State = "unsupported-provider" + out[i].Recovery = "unsupported-provider" + } + continue + } + if check.GetHeadSHA() == sha && check.GetID() > 0 && check.GetCheckSuite().GetID() > 0 { + actions = true + } + } + if !actions { + return out + } + unverified := func(detail string) []CIReportingObservation { + for i := range out { + out[i].State, out[i].Detail = "unverified", detail + out[i].Recovery = "owner-inspection-required" + } + return out + } + pr, _, err := client.PullRequests.Get(ctx, owner, repo, number) + if err != nil { + return unverified("head-read: " + ciReportingAPIError(err)) + } + if pr.GetHead().GetSHA() != sha || pr.GetState() != "open" { + return unverified("head-changed-or-closed") + } + base := pr.GetBase().GetRef() + var runs []*gh.WorkflowRun + options := &gh.ListWorkflowRunsOptions{HeadSHA: sha, ListOptions: gh.ListOptions{PerPage: workflowRunsPerPage}} + for page := 0; page < ciReportingMaxRunPages; page++ { + list, response, err := client.Actions.ListRepositoryWorkflowRuns(ctx, owner, repo, options) + if err != nil { + return unverified("workflow-runs: " + ciReportingAPIError(err)) + } + if list != nil { + runs = append(runs, list.WorkflowRuns...) + } + if response == nil || response.NextPage == 0 { + break + } + if page == ciReportingMaxRunPages-1 { + return unverified("workflow-run-pagination-incomplete") + } + options.Page = response.NextPage + } + jobsByRun := make(map[int64][]*gh.WorkflowJob) + currentByRun := make(map[int64]*gh.WorkflowRun) + contradiction := false + for i, check := range checks { + if check.GetApp().GetSlug() != "github-actions" || check.GetHeadSHA() != sha || check.GetID() <= 0 || check.GetCheckSuite().GetID() <= 0 { + continue + } + var candidate *gh.WorkflowRun + for _, run := range runs { + if run != nil && run.GetHeadSHA() == sha && run.GetCheckSuiteID() == check.GetCheckSuite().GetID() { + candidate = run + break + } + } + if candidate == nil || candidate.GetID() <= 0 || candidate.GetRunAttempt() <= 0 || + candidate.GetWorkflowID() <= 0 || candidate.GetRunNumber() <= 0 || candidate.GetEvent() == "" || + !strings.EqualFold(candidate.GetRepository().GetFullName(), owner+"/"+repo) { + out[i].Detail = "owning-workflow-not-verified" + continue + } + out[i].RunID, out[i].Attempt, out[i].RunURL = candidate.GetID(), candidate.GetRunAttempt(), candidate.GetHTMLURL() + superseded := false + for _, other := range runs { + if other == nil || other.GetHeadSHA() != sha || other.GetWorkflowID() != candidate.GetWorkflowID() || + other.GetEvent() != candidate.GetEvent() { + continue + } + if other.GetRunNumber() > candidate.GetRunNumber() || + (other.GetID() == candidate.GetID() && other.GetRunAttempt() > candidate.GetRunAttempt()) { + superseded = true + } + } + if superseded { + out[i].State, out[i].Detail = "superseded", "newer workflow execution owns current evidence" + out[i].Recovery = "awaiting-current-evidence" + continue + } + if candidate.GetStatus() != "completed" { + switch candidate.GetStatus() { + case "queued", "in_progress", "waiting", "requested", "pending": + out[i].State, out[i].Detail = "active", "owning workflow is queued or running; elapsed time alone is not stale" + out[i].Recovery = "none" + default: + out[i].Detail = "owning-workflow-status-unknown" + } + continue + } + if candidate.GetConclusion() == "" || candidate.GetUpdatedAt().IsZero() { + out[i].Detail = "terminal-workflow-conclusion-unknown" + continue + } + switch candidate.GetConclusion() { + case "success", "failure", "cancelled", "timed_out", "neutral", "skipped", "action_required", "startup_failure": + default: + out[i].Detail = "terminal-workflow-conclusion-unknown" + continue + } + jobs, known := jobsByRun[candidate.GetID()] + if !known { + if len(jobsByRun) >= ciReportingMaxRuns { + out[i].Detail = "observation-workflow-capacity" + continue + } + jobOptions := &gh.ListOptions{PerPage: workflowRunsPerPage} + for page := 0; page < ciReportingMaxJobPages; page++ { + list, response, err := client.Actions.ListWorkflowJobsAttempt(ctx, owner, repo, candidate.GetID(), int64(candidate.GetRunAttempt()), jobOptions) + if err != nil { + return unverified("latest-attempt-jobs: " + ciReportingAPIError(err)) + } + if list != nil { + jobs = append(jobs, list.Jobs...) + } + if response == nil || response.NextPage == 0 { + break + } + if page == ciReportingMaxJobPages-1 { + return unverified("workflow-job-pagination-incomplete") + } + jobOptions.Page = response.NextPage + } + jobsByRun[candidate.GetID()] = jobs + } + var matched *gh.WorkflowJob + for _, job := range jobs { + if job != nil && job.GetID() > 0 && job.GetRunID() == candidate.GetID() && + job.GetRunAttempt() == int64(candidate.GetRunAttempt()) && job.GetHeadSHA() == sha && + ciReportingJobCheck(client, owner, repo, job.GetCheckRunURL(), check.GetID()) { + matched = job + break + } + } + if matched == nil || matched.GetStatus() == "" { + out[i].Detail = "check-not-in-latest-attempt" + continue + } + switch matched.GetStatus() { + case "queued", "in_progress", "completed", "waiting", "requested", "pending": + default: + out[i].Detail = "owning-job-status-unknown" + continue + } + current, known := currentByRun[candidate.GetID()] + if !known { + var err error + current, _, err = client.Actions.GetWorkflowRunByID(ctx, owner, repo, candidate.GetID()) + if err != nil { + return unverified("workflow-consistency-read: " + ciReportingAPIError(err)) + } + currentByRun[candidate.GetID()] = current + } + if current.GetID() != candidate.GetID() || current.GetHeadSHA() != sha || + !strings.EqualFold(current.GetRepository().GetFullName(), owner+"/"+repo) || + current.GetWorkflowID() != candidate.GetWorkflowID() || current.GetEvent() != candidate.GetEvent() || + current.GetRunNumber() != candidate.GetRunNumber() || + current.GetCheckSuiteID() != candidate.GetCheckSuiteID() || + current.GetRunAttempt() != candidate.GetRunAttempt() || current.GetStatus() != "completed" || + current.GetConclusion() != candidate.GetConclusion() || + !current.GetUpdatedAt().Time.Equal(candidate.GetUpdatedAt().Time) { + out[i].Detail = "workflow-attempt-changed-during-observation" + continue + } + out[i].JobID, out[i].JobStatus = matched.GetID(), matched.GetStatus() + out[i].JobConclusion = matched.GetConclusion() + out[i].RunConclusion = current.GetConclusion() + out[i].RunUpdatedAt = current.GetUpdatedAt().Time + out[i].State = "contradiction" + contradiction = true + } + if contradiction { + pr, _, err := client.PullRequests.Get(ctx, owner, repo, number) + if err != nil { + return unverified("final-head-read: " + ciReportingAPIError(err)) + } + if pr.GetHead().GetSHA() != sha || pr.GetState() != "open" || pr.GetBase().GetRef() != base { + return unverified("head-changed-or-closed-during-observation") + } + } + return out +} + +func ciReportingJobCheck(client *gh.Client, owner, repo, raw string, checkID int64) bool { + link, err := url.Parse(raw) + if err != nil || link.User != nil || client.BaseURL == nil || link.Host != client.BaseURL.Host || link.Scheme != client.BaseURL.Scheme { + return false + } + expected := strings.TrimSuffix(client.BaseURL.Path, "/") + "/repos/" + owner + "/" + repo + "/check-runs/" + strconv.FormatInt(checkID, 10) + return link.Path == expected && link.RawQuery == "" && link.Fragment == "" +} + +func ciReportingAPIError(err error) string { + var response *gh.ErrorResponse + if errors.As(err, &response) && response.Response != nil { + return fmt.Sprintf("api-http-%d (permission/rate-limit/unavailable; not a CI verdict)", response.Response.StatusCode) + } + return "api-unavailable (not a CI verdict)" +} + +func ciReportingReason(observations []CIReportingObservation) string { + for _, observation := range observations { + if observation.State == "stalled" { + return fmt.Sprintf("ci-reporting-stalled:check=%q check-id=%d run=%d attempt=%d job=%d check-state=%s job-state=%s job-result=%q workflow=%s owner-action-required", + observation.Check, observation.CheckRunID, observation.RunID, observation.Attempt, + observation.JobID, observation.CheckStatus, observation.JobStatus, observation.JobConclusion, observation.RunConclusion) + } + } + for _, observation := range observations { + if observation.State == "unverified" && observation.Detail != "missing-actions-provenance" { + return fmt.Sprintf("ci-reporting-unverified:check=%q %s owner-action-required", observation.Check, observation.Detail) + } + } + return "" +} diff --git a/src/pkg/github/ci_reporting_test.go b/src/pkg/github/ci_reporting_test.go new file mode 100644 index 0000000000..0463a1c32b --- /dev/null +++ b/src/pkg/github/ci_reporting_test.go @@ -0,0 +1,367 @@ +package github + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/stageengine" +) + +type ciReportingFixture struct { + client *gh.Client + check *gh.CheckRun + mode string + reads atomic.Int64 + heads atomic.Int64 + beforeRuns func() +} + +func newCIReportingFixture(t *testing.T) *ciReportingFixture { + t.Helper() + f := &ciReportingFixture{} + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Errorf("read-only observer attempted %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + f.reads.Add(1) + run := map[string]any{ + "id": 21, "head_sha": "sha", "check_suite_id": 31, "workflow_id": 11, + "run_number": 9, "run_attempt": 1, "event": "pull_request", + "status": "completed", "conclusion": "failure", + "updated_at": "2026-10-31T00:00:00Z", + "repository": map[string]string{"full_name": "acme/widget"}, + "html_url": server.URL + "/acme/widget/actions/runs/21", + } + switch f.mode { + case "active", "old-active": + run["status"], run["conclusion"] = "queued", "" + case "success": + run["conclusion"] = "success" + case "cancelled": + run["conclusion"] = "cancelled" + case "unknown-conclusion": + run["conclusion"] = "unrecognized" + case "unknown-run-status": + run["status"] = "unrecognized" + case "run-head": + run["head_sha"] = "other-head" + case "run-repo": + run["repository"] = map[string]string{"full_name": "other/widget"} + case "run-suite": + run["check_suite_id"] = 99 + case "attempt-two": + run["run_attempt"] = 2 + } + switch r.URL.Path { + case "/repos/acme/widget/pulls/7": + count := f.heads.Add(1) + head, base := "sha", "v5" + if f.mode == "head" || f.mode == "head-during-read" && count%2 == 0 { + head = "new-head" + } + if f.mode == "retarget-during-read" && count%2 == 0 { + base = "other-base" + } + _ = json.NewEncoder(w).Encode(map[string]any{"state": "open", + "head": map[string]string{"sha": head}, "base": map[string]string{"ref": base}}) + case "/repos/acme/widget/actions/runs": + if f.beforeRuns != nil { + f.beforeRuns() + } + if f.mode == "forbidden" || f.mode == "unavailable" || f.mode == "rate-limit" { + status := http.StatusForbidden + if f.mode == "unavailable" { + status = http.StatusInternalServerError + } + if f.mode == "rate-limit" { + status = http.StatusTooManyRequests + } + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"message":"observation unavailable"}`)) + return + } + runs := []any{run} + if f.mode == "newer-active" || f.mode == "newer-success" { + run["conclusion"] = "cancelled" + newer := map[string]any{"id": 22, "head_sha": "sha", "workflow_id": 11, + "run_number": 10, "run_attempt": 1, "event": "pull_request", + "status": "queued", "conclusion": ""} + if f.mode == "newer-success" { + newer["status"], newer["conclusion"] = "completed", "success" + } + runs = append(runs, newer) + } + _ = json.NewEncoder(w).Encode(map[string]any{"workflow_runs": runs}) + case "/repos/acme/widget/actions/runs/21": + if f.mode == "attempt-during-read" { + run["run_attempt"], run["status"] = 2, "queued" + } + if f.mode == "run-id-during-read" { + run["id"] = 99 + } + _ = json.NewEncoder(w).Encode(run) + case "/repos/acme/widget/actions/runs/21/attempts/1/jobs", "/repos/acme/widget/actions/runs/21/attempts/2/jobs": + attempt := 1 + if f.mode == "attempt-two" { + attempt = 2 + } + job := map[string]any{"id": 41, "run_id": 21, "run_attempt": attempt, + "head_sha": "sha", "name": "build-and-test", "status": "queued", + "check_run_url": server.URL + "/repos/acme/widget/check-runs/51"} + switch f.mode { + case "job-head": + job["head_sha"] = "other-head" + case "job-attempt": + job["run_attempt"] = 99 + case "job-run": + job["run_id"] = 99 + case "job-check": + job["check_run_url"] = server.URL + "/repos/acme/widget/check-runs/99" + case "job-provider": + job["check_run_url"] = "https://other.example/repos/acme/widget/check-runs/51" + case "failed-job": + job["status"], job["conclusion"] = "completed", "failure" + case "unknown-job-status": + job["status"] = "unrecognized" + } + _ = json.NewEncoder(w).Encode(map[string]any{"jobs": []any{job}}) + case "/repos/acme/widget/commits/sha/status": + _ = json.NewEncoder(w).Encode(map[string]any{"statuses": []any{}}) + case "/repos/acme/widget/commits/sha/check-runs": + _ = json.NewEncoder(w).Encode(map[string]any{"check_runs": []any{f.check}}) + default: + t.Errorf("unexpected observation endpoint %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + f.client = gh.NewClient(server.Client()) + f.client.BaseURL, _ = f.client.BaseURL.Parse(server.URL + "/") + f.check = &gh.CheckRun{ID: gh.Ptr(int64(51)), Name: gh.Ptr("build-and-test"), + HeadSHA: gh.Ptr("sha"), Status: gh.Ptr("queued"), + App: &gh.App{Slug: gh.Ptr("github-actions")}, CheckSuite: &gh.CheckSuite{ID: gh.Ptr(int64(31))}} + return f +} + +func TestCIReportingExactRunLifecycle(t *testing.T) { + cases := []struct{ mode, state string }{ + {"", "stalled"}, {"success", "stalled"}, {"cancelled", "stalled"}, {"failed-job", "stalled"}, + {"active", "active"}, {"old-active", "active"}, {"head", "unverified"}, + {"head-during-read", "unverified"}, {"retarget-during-read", "unverified"}, + {"run-head", "unverified"}, {"run-repo", "unverified"}, {"run-suite", "unverified"}, + {"job-head", "unverified"}, {"job-attempt", "unverified"}, {"job-run", "unverified"}, + {"job-check", "unverified"}, {"job-provider", "unverified"}, + {"unknown-conclusion", "unverified"}, {"unknown-run-status", "unverified"}, {"unknown-job-status", "unverified"}, {"attempt-during-read", "unverified"}, + {"run-id-during-read", "unverified"}, {"newer-active", "superseded"}, {"newer-success", "superseded"}, + {"forbidden", "unverified"}, {"rate-limit", "unverified"}, {"unavailable", "unverified"}, + } + for _, tc := range cases { + t.Run(tc.mode, func(t *testing.T) { + f := newCIReportingFixture(t) + f.mode = tc.mode + now := time.Date(2026, 10, 31, 2, 0, 0, 0, time.UTC) + opts := CommitCIOptions{PRNumber: 7, RequiredKnown: true, Now: func() time.Time { return now }} + cache := newCIReportingCache() + first := cache.observe(context.Background(), f.client, "acme", "widget", "sha", opts, []*gh.CheckRun{f.check}) + if tc.state == "stalled" && first[0].State != "propagation-grace" { + t.Fatalf("terminal age alone declared a stall: %+v", first) + } + now = now.Add(defaultCIReportingRecheck) + observations := cache.observe(context.Background(), f.client, "acme", "widget", "sha", opts, []*gh.CheckRun{f.check}) + if len(observations) != 1 || observations[0].State != tc.state { + t.Fatalf("observations=%+v want=%s", observations, tc.state) + } + got := observations[0] + if got.State == "stalled" { + if got.Repo != "acme/widget" || got.PRNumber != 7 || got.HeadSHA != "sha" || got.RunID != 21 || got.Attempt != 1 || got.JobID != 41 || + got.CheckRunID != 51 || got.CheckStatus != "queued" || got.Recovery != "owner-action-required" { + t.Fatalf("lost provenance or invented recovery: %+v", got) + } + if got.State == "active" && got.Recovery != "none" { + t.Fatalf("ordinary active CI requested recovery: %+v", got) + } + reason := ciReportingReason(observations) + if !strings.HasPrefix(reason, "ci-reporting-stalled:") || stageengine.ClassifyCIReason(reason) != stageengine.CIReasonPending { + t.Fatalf("reporting mismatch changed stage semantics: %s", reason) + } + } + if strings.Contains(tc.mode, "forbidden") || tc.mode == "rate-limit" || tc.mode == "unavailable" { + if !strings.HasPrefix(ciReportingReason(observations), "ci-reporting-unverified:") { + t.Fatalf("API error disappeared: %+v", observations) + } + } + }) + } +} + +func TestCIReportingCacheEventsRestartAndAttemptReset(t *testing.T) { + f := newCIReportingFixture(t) + now := time.Date(2026, 10, 31, 2, 0, 0, 0, time.UTC) + opts := CommitCIOptions{PRNumber: 7, RequiredKnown: true, Now: func() time.Time { return now }} + cache := newCIReportingCache() + observe := func() CIReportingObservation { + return cache.observe(context.Background(), f.client, "acme", "widget", "sha", opts, []*gh.CheckRun{f.check})[0] + } + if observe().State != "propagation-grace" { + t.Fatal("first observation did not wait") + } + reads := f.reads.Load() + for range 20 { + cache.invalidate("acme/widget", "sha") + if observe().State != "propagation-grace" { + t.Fatal("duplicate event shortened grace") + } + } + if f.reads.Load() != reads { + t.Fatal("duplicate events caused an API storm") + } + now = now.Add(ciReportingEventRecheck) + if observe().State != "propagation-grace" { + t.Fatal("early independent sample shortened grace") + } + now = now.Add(defaultCIReportingGrace - ciReportingEventRecheck) + cache.invalidate("acme/widget", "sha") + if observe().State != "stalled" { + t.Fatal("consistent terminal/reporting contradiction never became actionable") + } + restarted := newCIReportingCache() + if restarted.observe(context.Background(), f.client, "acme", "widget", "sha", opts, []*gh.CheckRun{f.check})[0].State != "propagation-grace" { + t.Fatal("restart assumed a previous observation or authorized a mutation") + } + now = now.Add(defaultCIReportingRecheck) + f.mode = "active" + if observe().State != "active" { + t.Fatal("terminal attempt overrode an active rerun") + } + now = now.Add(defaultCIReportingRecheck) + f.mode = "attempt-two" + got := observe() + if got.State != "propagation-grace" || got.Attempt != 2 { + t.Fatalf("new attempt inherited old recovery episode: %+v", got) + } + now = now.Add(defaultCIReportingRecheck) + f.mode = "forbidden" + if observe().State != "unverified" { + t.Fatal("permission failure looked normal") + } + now = now.Add(defaultCIReportingRecheck) + f.mode = "attempt-two" + if observe().State != "propagation-grace" { + t.Fatal("API gap counted as a consistency sample") + } + now = now.Add(-defaultCIReportingRecheck) + if observe().State != "propagation-grace" { + t.Fatal("clock reversal promoted a stale episode") + } +} + +func TestCIReportingGateNeverFabricatesGreenAndCommitOnlyUnchanged(t *testing.T) { + f := newCIReportingFixture(t) + now := time.Date(2026, 10, 31, 2, 0, 0, 0, time.UTC) + previous := sharedCIReportingCache + sharedCIReportingCache = newCIReportingCache() + t.Cleanup(func() { sharedCIReportingCache = previous }) + opts := CommitCIOptions{PRNumber: 7, RequiredKnown: true, Required: map[string]bool{"build-and-test": true}, + Now: func() time.Time { return now }} + for range 2 { + got, err := EvaluateCommitCI(context.Background(), f.client, "acme", "widget", "sha", opts) + if err != nil || got.Green || got.InFlight != 1 { + t.Fatalf("reporting replaced semantic CI: %+v %v", got, err) + } + now = now.Add(defaultCIReportingRecheck) + } + got, err := EvaluateCommitCI(context.Background(), f.client, "acme", "widget", "sha", opts) + if err != nil || !strings.HasPrefix(got.Reason, "ci-reporting-stalled:") { + t.Fatalf("stall was not exposed through the existing reason: %+v %v", got, err) + } + opts.PRNumber = 0 + got, err = EvaluateCommitCI(context.Background(), f.client, "acme", "widget", "sha", opts) + if err != nil || got.Green || got.Reason != "required-check-pending:build-and-test" || len(got.Reporting) != 0 { + t.Fatalf("base/commit behavior changed: %+v %v", got, err) + } + f.check.Status, f.check.Conclusion, f.check.ID = gh.Ptr("completed"), gh.Ptr("success"), gh.Ptr(int64(52)) + opts.PRNumber = 7 + got, err = EvaluateCommitCI(context.Background(), f.client, "acme", "widget", "sha", opts) + if err != nil || !got.Green || len(got.Reporting) != 0 { + t.Fatalf("old observation overrode actual newer success: %+v %v", got, err) + } +} + +func TestCIReportingUnsupportedAndBoundedCache(t *testing.T) { + f := newCIReportingFixture(t) + f.check.App.Slug = gh.Ptr("external-ci") + cache := newCIReportingCache() + now := time.Date(2026, 10, 31, 2, 0, 0, 0, time.UTC) + for i := range ciReportingMaxEntries { + cache.entries[fmt.Sprint(i)] = &ciReportingEntry{checkedAt: now.Add(-time.Duration(i) * time.Minute)} + } + opts := CommitCIOptions{PRNumber: 7, RequiredKnown: true, Now: func() time.Time { return now }} + got := cache.observe(context.Background(), f.client, "acme", "widget", "sha", opts, []*gh.CheckRun{f.check}) + if len(got) != 1 || got[0].State != "unsupported-provider" || f.reads.Load() != 0 || len(cache.entries) != ciReportingMaxEntries { + t.Fatalf("unsupported provider polled or unbounded cache: %+v reads=%d entries=%d", got, f.reads.Load(), len(cache.entries)) + } +} + +func TestCIReportingConcurrentObserversShareOneRead(t *testing.T) { + f := newCIReportingFixture(t) + entered, release, finished := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var arrivals atomic.Int64 + f.beforeRuns = func() { + if arrivals.Add(1) == 1 { + close(entered) + <-release + } + } + cache := newCIReportingCache() + now := time.Date(2026, 10, 31, 2, 0, 0, 0, time.UTC) + opts := CommitCIOptions{PRNumber: 7, RequiredKnown: true, Now: func() time.Time { return now }} + go func() { + defer close(finished) + cache.observe(context.Background(), f.client, "acme", "widget", "sha", opts, []*gh.CheckRun{f.check}) + }() + select { + case <-entered: + case <-time.After(5 * time.Second): + close(release) + t.Fatal("observer did not reach the owning workflow read") + } + reads := f.reads.Load() + for range 20 { + cache.observe(context.Background(), f.client, "acme", "widget", "sha", opts, []*gh.CheckRun{f.check}) + } + if f.reads.Load() != reads { + t.Error("competing observers duplicated workflow reads") + } + close(release) + <-finished +} + +func TestCIReportingWorkflowWebhookInvalidatesOnlyObservedHead(t *testing.T) { + c, _ := setupWebhookCacheTest(t) + previous := sharedCIReportingCache + sharedCIReportingCache = newCIReportingCache() + t.Cleanup(func() { sharedCIReportingCache = previous }) + current := &ciReportingEntry{repo: "org/repo", head: "sha"} + other := &ciReportingEntry{repo: "org/repo", head: "other"} + sharedCIReportingCache.entries["current"], sharedCIReportingCache.entries["other"] = current, other + for range 20 { + res, err := c.HandleWebhookEvent("workflow_run", []byte(`{"action":"completed","repository":{"full_name":"org/repo"},"workflow_run":{"head_sha":"sha","pull_requests":[{"number":7}]}}`)) + if err != nil || res.HeadSHA != "sha" || len(res.Invalidated) != 1 || res.Invalidated[0] != 7 { + t.Fatalf("workflow delivery was not propagated: %+v %v", res, err) + } + } + if !current.dirty || other.dirty { + t.Fatal("workflow event dirtied unrelated immutable heads") + } +} diff --git a/src/pkg/github/commit_ci.go b/src/pkg/github/commit_ci.go index 4fbb9bbab5..5906b3e5e9 100644 --- a/src/pkg/github/commit_ci.go +++ b/src/pkg/github/commit_ci.go @@ -74,6 +74,8 @@ func (c *requiredChecksForbiddenCache) put(key string) { // this package (the transport, the ignore lists) and the watcher lives here; // a core both can call has to sit below both. type CommitCIState struct { + // Reporting is a read-only consistency observation, not CI success. + Reporting []CIReportingObservation // Green is true when every gating status/check run has succeeded (or is // ignorable). Reason names the first blocker when false: "status-pending", // "check-pending", "status-" or "check-". Zero statuses @@ -126,6 +128,9 @@ type CommitCIState struct { // CommitCIOptions carries policy facts that are known to callers before the // commit-status/check-run walk starts. type CommitCIOptions struct { + // PRNumber enables bounded, exact-current-head reporting reconciliation. + // Commit-only/base observations retain their existing behavior. + PRNumber int Required map[string]bool RequiredKnown bool RequiredKnownFromConfig bool @@ -654,6 +659,18 @@ func EvaluateCommitCI(ctx context.Context, client *gh.Client, owner, repo, sha s } } st.Green = st.Reason == "" + if !st.Green && strings.HasPrefix(st.Reason, "required-check-pending:") && opts.PRNumber > 0 { + var pending []*gh.CheckRun + for _, check := range latestCheckRunsByNameAndApp(allCheckRuns) { + if required[check.GetName()] && check.GetStatus() != "completed" { + pending = append(pending, check) + } + } + st.Reporting = sharedCIReportingCache.observe(ctx, client, owner, repo, sha, opts, pending) + if reason := ciReportingReason(st.Reporting); reason != "" { + st.Reason = reason + } + } return st, nil } diff --git a/src/pkg/github/merge_ci_gate.go b/src/pkg/github/merge_ci_gate.go index 51badd0099..880f2164c9 100644 --- a/src/pkg/github/merge_ci_gate.go +++ b/src/pkg/github/merge_ci_gate.go @@ -196,6 +196,7 @@ func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number i } } st, err := EvaluateCommitCI(ctx, c.client, owner, name, sha, CommitCIOptions{ + PRNumber: number, Required: required, RequiredKnown: requiredKnown, RequiredKnownFromConfig: fromConfig, @@ -210,6 +211,10 @@ func (c *Client) verifyMergeRequestCI(ctx context.Context, repo string, number i if cfgKnown && actualKnown { c.warnRequiredChecksMismatch(owner, name, cfgSet, actualRequired, st.Observed) } + if len(st.Reporting) > 0 { + c.info("merge-request CI reporting observation", "repo", owner+"/"+name, "pr", number, + "head", sha, "observations", st.Reporting) + } if !st.Green && (strings.HasPrefix(st.Reason, "pending: head pushed ") || (strings.HasPrefix(st.Reason, "pending: ") && strings.HasSuffix(st.Reason, " has not started"))) { c.info("merge-request CI gate blocked fresh or incomplete head", "repo", owner+"/"+name, "pr", number, "sha", sha, "reason", st.Reason) } diff --git a/src/pkg/github/webhook_cache.go b/src/pkg/github/webhook_cache.go index 03361e734a..92cace1e19 100644 --- a/src/pkg/github/webhook_cache.go +++ b/src/pkg/github/webhook_cache.go @@ -471,6 +471,10 @@ type webhookEventPayload struct { HTMLURL string `json:"html_url"` PullRequests []webhookPRRef `json:"pull_requests"` } `json:"check_run"` + WorkflowRun *struct { + HeadSHA string `json:"head_sha"` + PullRequests []webhookPRRef `json:"pull_requests"` + } `json:"workflow_run"` SHA string `json:"sha"` State string `json:"state"` Context string `json:"context"` @@ -562,6 +566,9 @@ func (c *Client) HandleWebhookEvent(event string, payload []byte) (WebhookEventR c.RecordReviewEvidenceRevert(merged) } res.HeadSHA = webhookHeadSHA(event, p) + if res.HeadSHA != "" { + sharedCIReportingCache.invalidate(p.Repository.FullName, res.HeadSHA) + } numbers := webhookInvalidationTargets(event, p, aliases) // Capture the failed-check identity and the PRs it resolved to BEFORE // invalidation drops their cache entries: the receiver reacts to a red @@ -712,6 +719,10 @@ func webhookHeadSHA(event string, p webhookEventPayload) string { } case "status": return strings.TrimSpace(p.SHA) + case "workflow_run": + if p.WorkflowRun != nil { + return strings.TrimSpace(p.WorkflowRun.HeadSHA) + } } return "" } @@ -764,6 +775,11 @@ func webhookInvalidationTargets(event string, p webhookEventPayload, aliases []s } case "status": bySHA(p.SHA) + case "workflow_run": + if p.WorkflowRun != nil { + addRefs(p.WorkflowRun.PullRequests) + bySHA(p.WorkflowRun.HeadSHA) + } case "push": if ref, ok := strings.CutPrefix(p.Ref, "refs/heads/"); ok && ref != "" { addMatching(func(_, headRef, baseRef string) bool { return headRef == ref || baseRef == ref })