From 7d08679b5d97ca275b5fe7f4b0594c3dec61f0a9 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Mon, 2 Nov 2026 15:24:55 -0500 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=90=9B=20fix:=20commit=20native=20dec?= =?UTF-8?q?isions=20before=20bounded=20reporting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- .../fixed-journal-decision-throughput.md | 1 + src/pkg/convergence/mutation/boundary.go | 14 +- src/pkg/convergence/mutation/executor.go | 21 +- src/pkg/convergence/mutation/journal.go | 169 +++++++++--- .../mutation/journal_throughput_test.go | 250 ++++++++++++++++++ .../api_repo_item_decide_delegated.go | 113 ++++++-- src/pkg/dashboard/api_repo_item_decision.go | 7 +- .../dashboard/api_repo_item_decision_test.go | 13 + .../delegated_approval_ordering_test.go | 189 +++++++++++++ src/pkg/dashboard/operator_actions.go | 7 +- src/pkg/dashboard/row_action_result.go | 7 + src/pkg/delegatedecision/policy.go | 13 +- src/pkg/github/client.go | 29 +- src/pkg/github/pr_approval_ordering_test.go | 133 ++++++++++ src/pkg/github/pr_decision_operator.go | 128 ++++++--- src/pkg/review/human_approval_commit_test.go | 41 +++ src/pkg/review/human_change_request.go | 35 ++- 17 files changed, 1049 insertions(+), 121 deletions(-) create mode 100644 changelog.d/fixed-journal-decision-throughput.md create mode 100644 src/pkg/convergence/mutation/journal_throughput_test.go create mode 100644 src/pkg/dashboard/delegated_approval_ordering_test.go create mode 100644 src/pkg/github/pr_approval_ordering_test.go create mode 100644 src/pkg/review/human_approval_commit_test.go diff --git a/changelog.d/fixed-journal-decision-throughput.md b/changelog.d/fixed-journal-decision-throughput.md new file mode 100644 index 0000000000..51a6c7dbdc --- /dev/null +++ b/changelog.d/fixed-journal-decision-throughput.md @@ -0,0 +1 @@ +- Record validated operator approvals before optional GitHub reporting, bound reporting waits, and keep mutation-journal readers responsive during ordered durable writes without weakening merge gates. diff --git a/src/pkg/convergence/mutation/boundary.go b/src/pkg/convergence/mutation/boundary.go index cdc5beecac..007ba4d053 100644 --- a/src/pkg/convergence/mutation/boundary.go +++ b/src/pkg/convergence/mutation/boundary.go @@ -60,13 +60,13 @@ func (b *Boundary) ExecuteJournaled(ctx context.Context, claim effects.Claim, ef invoked := false out, err := frozen.Execute(ctx, claim, func(ctx context.Context) (effects.Result, error) { invoked = true - if err := frozen.Executor.Journal.syncPersistent(); err != nil { + if err := frozen.Executor.Journal.syncPersistentContext(ctx); err != nil { return effects.Result{}, effects.NotApplied(fmt.Errorf("syncing mutation intent before effect: %w", err)) } return effect(ctx) }) if invoked || err == nil || out.Provenance != "" { - if syncErr := frozen.Executor.Journal.syncPersistent(); syncErr != nil { + if syncErr := frozen.Executor.Journal.syncPersistentContext(ctx); syncErr != nil { return out, fmt.Errorf("mutation journal acknowledgement not synced: %w (execution: %v)", syncErr, err) } } @@ -126,7 +126,7 @@ func (b *Boundary) Execute(ctx context.Context, claim effects.Claim, effect func } mutEffect := boundaryEffect(claim) var out effects.Result - op, execErr := executor.Execute(mutEffect, entry.Epoch, holder, func() (string, error) { + op, execErr := executor.ExecuteContext(ctx, mutEffect, entry.Epoch, holder, func() (string, error) { res, err := effect(ctx) out = res return res.Provenance, err @@ -205,7 +205,7 @@ func (b *Boundary) noteResolved(logicalID string) { // observation of external state (effects.Reconciler): Applied records the // found effect without repeating it, !Applied authorizes the same logical // operation to retry. Off mode journals nothing, so there is nothing to do. -func (b *Boundary) Reconcile(_ context.Context, claim effects.Claim, state effects.ExternalState) error { +func (b *Boundary) Reconcile(ctx context.Context, claim effects.Claim, state effects.ExternalState) error { if b == nil { return nil } @@ -222,7 +222,7 @@ func (b *Boundary) Reconcile(_ context.Context, claim effects.Claim, state effec if executor.Journal == nil { return fmt.Errorf("mutation boundary requires a journal when mode is enabled") } - op, err := executor.Journal.Reconcile(boundaryEffect(claim).LogicalID(), ExternalState{ + op, err := executor.Journal.ReconcileContext(ctx, boundaryEffect(claim).LogicalID(), ExternalState{ Known: true, Applied: state.Applied, Result: state.Provenance, @@ -239,7 +239,7 @@ func (b *Boundary) Reconcile(_ context.Context, claim effects.Claim, state effec // be attempted again. A Known external observation (Reconcile) is always // preferred; this is for the case where none can be obtained and the entry // would otherwise fence every retry forever. Off mode journals nothing. -func (b *Boundary) ExpireUnresolved(_ context.Context, claim effects.Claim, ttl time.Duration) (bool, error) { +func (b *Boundary) ExpireUnresolved(ctx context.Context, claim effects.Claim, ttl time.Duration) (bool, error) { if b == nil || ttl <= 0 { return false, nil } @@ -265,7 +265,7 @@ func (b *Boundary) ExpireUnresolved(_ context.Context, claim effects.Claim, ttl if now.Sub(op.UpdatedAt) < ttl { return false, nil } - if _, err := executor.Journal.Reconcile(id, ExternalState{Known: true, Applied: false}, now); err != nil { + if _, err := executor.Journal.ReconcileContext(ctx, id, ExternalState{Known: true, Applied: false}, now); err != nil { return false, err } if b.Logger != nil { diff --git a/src/pkg/convergence/mutation/executor.go b/src/pkg/convergence/mutation/executor.go index 554800256b..6bb68a3124 100644 --- a/src/pkg/convergence/mutation/executor.go +++ b/src/pkg/convergence/mutation/executor.go @@ -1,6 +1,7 @@ package mutation import ( + "context" "errors" "fmt" "time" @@ -85,6 +86,10 @@ func (x Executor) now() time.Time { // operation whose latest attempt is Unknown replays as ErrNeedsReconciliation, // never as a success carrying an empty result. func (x Executor) Execute(e Effect, epoch uint64, holder string, effect EffectFunc) (Operation, error) { + return x.ExecuteContext(context.Background(), e, epoch, holder, effect) +} + +func (x Executor) ExecuteContext(ctx context.Context, e Effect, epoch uint64, holder string, effect EffectFunc) (Operation, error) { if !JournalingEnabled(x.Mode) { result, err := effect() if err != nil { @@ -107,7 +112,7 @@ func (x Executor) Execute(e Effect, epoch uint64, holder string, effect EffectFu } // Intent before effect, durably. - op, err := x.Journal.Begin(e, epoch, holder, now) + op, err := x.Journal.BeginContext(ctx, e, epoch, holder, now) if err != nil { if errors.Is(err, ErrAlreadyApplied) { // Dedup replay: the journal already holds this logical operation @@ -122,18 +127,24 @@ func (x Executor) Execute(e Effect, epoch uint64, holder string, effect EffectFu return Operation{}, err } + if err := ctx.Err(); err != nil { + // Physical persistence cannot be interrupted safely. If cancellation + // arrived during it, leave the durable intent for reconciliation, + // rather than starting an external call after the caller has gone. + return Operation{}, fmt.Errorf("mutation canceled after intent; reconciliation required: %w", err) + } result, effectErr := effect() after := x.now() if effectErr != nil && errors.Is(effectErr, effects.ErrNotApplied) { - if _, recErr := x.Journal.RecordResult(op.LogicalID, epoch, StatusNotApplied, "", after); recErr != nil { + if _, recErr := x.Journal.RecordResultContext(ctx, op.LogicalID, epoch, StatusNotApplied, "", after); recErr != nil { return Operation{}, fmt.Errorf("recording refused effect: %v (effect error: %w)", recErr, effectErr) } return Operation{}, effectErr } if effectErr != nil { // Uncertain: the call failed but the effect may exist externally. - if _, recErr := x.Journal.RecordResult(op.LogicalID, epoch, StatusUnknown, "", after); recErr != nil { + if _, recErr := x.Journal.RecordResultContext(ctx, op.LogicalID, epoch, StatusUnknown, "", after); recErr != nil { return Operation{}, fmt.Errorf("recording uncertain effect: %v (effect error: %w)", recErr, effectErr) } return Operation{}, fmt.Errorf("effect outcome uncertain, reconciliation required: %w", effectErr) @@ -144,11 +155,11 @@ func (x Executor) Execute(e Effect, epoch uint64, holder string, effect EffectFu // The effect happened but this owner may no longer speak for it: // record Unknown so the CURRENT owner reconciles the same logical // operation from authoritative external state — never a duplicate. - if _, recErr := x.Journal.RecordResult(op.LogicalID, epoch, StatusUnknown, "", after); recErr != nil { + if _, recErr := x.Journal.RecordResultContext(ctx, op.LogicalID, epoch, StatusUnknown, "", after); recErr != nil { return Operation{}, fmt.Errorf("recording unacknowledgeable effect: %v (fence: %w)", recErr, err) } return Operation{}, fmt.Errorf("acknowledgment fenced, reconciliation required: %w", err) } - return x.Journal.RecordResult(op.LogicalID, epoch, StatusApplied, result, after) + return x.Journal.RecordResultContext(ctx, op.LogicalID, epoch, StatusApplied, result, after) } diff --git a/src/pkg/convergence/mutation/journal.go b/src/pkg/convergence/mutation/journal.go index 3b44649e45..827b5bac2b 100644 --- a/src/pkg/convergence/mutation/journal.go +++ b/src/pkg/convergence/mutation/journal.go @@ -1,11 +1,13 @@ package mutation import ( + "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" + "log/slog" "os" "path/filepath" "sort" @@ -220,7 +222,7 @@ type persistedJournal struct { // Journal is the durable idempotent operation record: an in-memory index over // one JSON file, reloaded on boot, rewritten atomically on every accepted -// transition, serialized per logical-ID under the journal mutex and across +// transition, serialized under a transaction writer and across // processes under the exclusive .lock flock (refreshed from disk before every // transition, exactly like the claim ledger). Process memory is never // authority. Because every transition reloads and rewrites the whole file, @@ -233,7 +235,11 @@ type Journal struct { maxEntries int now func() time.Time mu sync.Mutex + writer chan struct{} + committed map[string]*Operation ops map[string]*Operation + // writeSnapshot is the atomic replacement seam; nil uses the filesystem. + writeSnapshot func(string, []byte) error } // OpenJournal loads the operation journal at path, creating an empty one when @@ -250,15 +256,44 @@ func OpenJournal(path string) (*Journal, error) { maxEntries: DefaultJournalMaxEntries, now: time.Now, ops: make(map[string]*Operation), + writer: make(chan struct{}, 1), } if err := j.reloadLocked(); err != nil { return nil, err } + j.publish() return j, nil } -// reloadLocked replaces the in-memory index with the on-disk snapshot. Callers -// hold j.mu. +// Only the transaction writer owns ops. Readers see a separate, immutable +// committed index, never a transition waiting for its durable replacement. +func (j *Journal) publish() { + j.mu.Lock() + j.committed = j.ops + j.mu.Unlock() +} + +func (j *Journal) transaction(ctx context.Context) (func(), error) { + started := time.Now() + select { + case j.writer <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + unlock, err := j.lockAndRefreshContext(ctx) + if err != nil { + <-j.writer + return nil, err + } + j.publish() + slog.Debug("mutation journal transaction admitted", "admission_duration", time.Since(started)) + return func() { + unlock() + <-j.writer + }, nil +} + +// reloadLocked replaces the writer's private index with the on-disk snapshot. func (j *Journal) reloadLocked() error { data, err := os.ReadFile(j.path) if errors.Is(err, os.ErrNotExist) { @@ -282,8 +317,9 @@ func (j *Journal) reloadLocked() error { if _, dup := ops[op.LogicalID]; dup { return fmt.Errorf("mutation journal %s holds conflicting entries for %s", j.path, op.LogicalID) } - cp := op.clone() - ops[op.LogicalID] = &cp + // JSON decoding already detached these maps and slices from every + // previous image. Keep the owned entry without a second deep copy. + ops[op.LogicalID] = &op } j.ops = ops return nil @@ -292,8 +328,8 @@ func (j *Journal) reloadLocked() error { // lockAndRefreshLocked takes the exclusive cross-process flock on the journal's // .lock sibling within the bounded wait, then reloads the on-disk snapshot so // the transition applies to the peer's latest persisted state. The returned -// func releases the lock. Callers hold j.mu. -func (j *Journal) lockAndRefreshLocked() (func(), error) { +// func releases the lock. Callers own the transaction writer, not j.mu. +func (j *Journal) lockAndRefreshContext(ctx context.Context) (func(), error) { lockPath := j.path + journalLockFileSuffix if err := os.MkdirAll(filepath.Dir(lockPath), 0o770); err != nil { return nil, fmt.Errorf("creating mutation journal lock directory: %w", err) @@ -304,6 +340,10 @@ func (j *Journal) lockAndRefreshLocked() (func(), error) { } deadline := time.Now().Add(j.lockTimeout) for { + if err := ctx.Err(); err != nil { + _ = f.Close() + return nil, err + } err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) if err == nil { break @@ -316,7 +356,12 @@ func (j *Journal) lockAndRefreshLocked() (func(), error) { _ = f.Close() return nil, fmt.Errorf("%s after %s: %w", lockPath, j.lockTimeout, ErrJournalLocked) } - time.Sleep(journalLockPollInterval) + select { + case <-ctx.Done(): + _ = f.Close() + return nil, ctx.Err() + case <-time.After(journalLockPollInterval): + } } if err := j.reloadLocked(); err != nil { _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) @@ -341,19 +386,47 @@ func (o Operation) clone() Operation { // Get returns a detached copy of the operation for a logical ID. func (j *Journal) Get(id string) (Operation, bool) { + // Refresh idle journals for peer-process replay, but never queue a reader + // behind a local writer's serialization, flock wait or filesystem I/O. + select { + case j.writer <- struct{}{}: + unlock, err := j.tryRefresh() + if err == nil { + j.publish() + unlock() + } + <-j.writer + default: + } j.mu.Lock() defer j.mu.Unlock() - unlock, err := j.lockAndRefreshLocked() - if err == nil { - defer unlock() - } - op, ok := j.ops[id] + op, ok := j.committed[id] if !ok { return Operation{}, false } return op.clone(), true } +func (j *Journal) tryRefresh() (func(), error) { + f, err := os.OpenFile(j.path+journalLockFileSuffix, os.O_RDWR, journalFileMode) + if err != nil { + return nil, err + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = f.Close() + return nil, err + } + unlock := func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + _ = f.Close() + } + if err := j.reloadLocked(); err != nil { + unlock() + return nil, err + } + return unlock, nil +} + // Begin persists operation INTENT — durably, BEFORE the external effect — and // records the authorized attempt with its owner epoch inside the entry. The // rules, in order: @@ -370,13 +443,15 @@ func (j *Journal) Get(id string) (Operation, bool) { // unchanged inputs can never mint a second logical ID because the ID is // derived from the effect alone. func (j *Journal) Begin(e Effect, epoch uint64, holder string, now time.Time) (Operation, error) { + return j.BeginContext(context.Background(), e, epoch, holder, now) +} + +func (j *Journal) BeginContext(ctx context.Context, e Effect, epoch uint64, holder string, now time.Time) (Operation, error) { if err := e.Validate(); err != nil { return Operation{}, err } id := e.LogicalID() - j.mu.Lock() - defer j.mu.Unlock() - unlock, err := j.lockAndRefreshLocked() + unlock, err := j.transaction(ctx) if err != nil { return Operation{}, err } @@ -414,12 +489,14 @@ func (j *Journal) Begin(e Effect, epoch uint64, holder string, now time.Time) (O // IDENTICAL applied acknowledgment deduplicates silently. // - status must be StatusApplied, StatusNotApplied, or StatusUnknown. func (j *Journal) RecordResult(id string, epoch uint64, status, result string, now time.Time) (Operation, error) { + return j.RecordResultContext(context.Background(), id, epoch, status, result, now) +} + +func (j *Journal) RecordResultContext(ctx context.Context, id string, epoch uint64, status, result string, now time.Time) (Operation, error) { if status != StatusApplied && status != StatusNotApplied && status != StatusUnknown { return Operation{}, fmt.Errorf("%w: %q is not a recordable status", ErrInvalidEffect, status) } - j.mu.Lock() - defer j.mu.Unlock() - unlock, err := j.lockAndRefreshLocked() + unlock, err := j.transaction(ctx) if err != nil { return Operation{}, err } @@ -475,9 +552,11 @@ type ExternalState struct { // regresses an Applied entry to NotApplied — terminal truth stands, and a // disagreeing later observation is a refusal, not a rewrite. func (j *Journal) Reconcile(id string, state ExternalState, now time.Time) (Operation, error) { - j.mu.Lock() - defer j.mu.Unlock() - unlock, err := j.lockAndRefreshLocked() + return j.ReconcileContext(context.Background(), id, state, now) +} + +func (j *Journal) ReconcileContext(ctx context.Context, id string, state ExternalState, now time.Time) (Operation, error) { + unlock, err := j.transaction(ctx) if err != nil { return Operation{}, err } @@ -523,8 +602,16 @@ func attemptOutcome(status string) string { } // commit installs the updated operation and persists atomically, rolling back -// the in-memory index when the durable write fails. Callers hold j.mu. +// the writer's private index when persistence fails. Readers keep the last +// committed image. Callers own the transaction writer and cross-process lock. func (j *Journal) commit(id string, work Operation) (Operation, error) { + // The refreshed index may already be visible to readers. Detach only + // its map before editing; entries themselves are immutable/replaced. + pending := make(map[string]*Operation, len(j.ops)+1) + for id, op := range j.ops { + pending[id] = op + } + j.ops = pending prior, hadPrior := j.ops[id] cp := work.clone() j.ops[id] = &cp @@ -537,6 +624,7 @@ func (j *Journal) commit(id string, work Operation) (Operation, error) { } return Operation{}, err } + j.publish() return work.clone(), nil } @@ -560,8 +648,8 @@ func isTerminal(status string) bool { // pruneLocked bounds the in-memory index (and so the next persisted // snapshot) before a commit: terminal entries past the retention window are // dropped, then the oldest terminal entries are evicted until the count fits -// maxEntries. Planned and Unknown entries are always kept. Callers hold j.mu -// and the cross-process lock. +// maxEntries. Planned and Unknown entries are always kept. Callers own the +// transaction writer and the cross-process lock. func (j *Journal) pruneLocked(now time.Time) { if j.retention > 0 { cutoff := now.Add(-j.retention) @@ -594,33 +682,48 @@ func (j *Journal) pruneLocked(now time.Time) { } } -func (j *Journal) persistLocked() error { +func (j *Journal) persistLocked() (persistErr error) { + started := time.Now() + defer func() { + slog.Debug("mutation journal snapshot replacement finished", "operations", len(j.ops), "persist_duration", time.Since(started), "error", persistErr) + }() all := make([]Operation, 0, len(j.ops)) for _, op := range j.ops { - all = append(all, op.clone()) + // The transaction writer owns ops; immutable entries can be marshaled + // without another deep copy of every effect input and attempt. + all = append(all, *op) } sort.Slice(all, func(i, k int) bool { return all[i].LogicalID < all[k].LogicalID }) - data, err := json.MarshalIndent(persistedJournal{Version: journalFormatVersion, Operations: all}, "", " ") + data, err := json.Marshal(persistedJournal{Version: journalFormatVersion, Operations: all}) if err != nil { return fmt.Errorf("marshaling mutation journal: %w", err) } - tmpPath := j.path + ".tmp" + if j.writeSnapshot != nil { + return j.writeSnapshot(j.path, data) + } + return replaceJournalSnapshot(j.path, data) +} + +func replaceJournalSnapshot(path string, data []byte) error { + tmpPath := path + ".tmp" if err := os.WriteFile(tmpPath, data, journalFileMode); err != nil { return fmt.Errorf("writing tmp mutation journal: %w", err) } - return os.Rename(tmpPath, j.path) + return os.Rename(tmpPath, path) } // syncPersistent is opt-in for journal-required effects. Keep legacy persistence // unchanged, but sync both the latest snapshot and its renamed directory entry // before these effects run and before acknowledging their result. func (j *Journal) syncPersistent() error { + return j.syncPersistentContext(context.Background()) +} + +func (j *Journal) syncPersistentContext(ctx context.Context) error { if j == nil || j.path == "" { return fmt.Errorf("persistent mutation journal path required") } - j.mu.Lock() - defer j.mu.Unlock() - unlock, err := j.lockAndRefreshLocked() + unlock, err := j.transaction(ctx) if err != nil { return err } diff --git a/src/pkg/convergence/mutation/journal_throughput_test.go b/src/pkg/convergence/mutation/journal_throughput_test.go new file mode 100644 index 0000000000..17ee110dc0 --- /dev/null +++ b/src/pkg/convergence/mutation/journal_throughput_test.go @@ -0,0 +1,250 @@ +package mutation + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/convergence/proof" +) + +const journalProgressDeadline = 3 * time.Second + +func waitJournalSignal(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(journalProgressDeadline): + t.Fatal("journal transition did not make progress") + } +} + +func TestJournalSlowWriterPublishesOnlyCommittedOrderedSnapshots(t *testing.T) { + path := filepath.Join(t.TempDir(), "journal.json") + j, err := OpenJournal(path) + if err != nil { + t.Fatal(err) + } + first := crossProcessEffect("acme/widgets#1", "first") + second := crossProcessEffect("acme/widgets#2", "second") + entered := make(chan struct{}) + release := make(chan struct{}) + secondEntered := make(chan struct{}) + secondRelease := make(chan struct{}) + writes := 0 + j.writeSnapshot = func(path string, data []byte) error { + writes++ + switch writes { + case 1: + close(entered) + <-release + case 2: + // A later image cannot overtake the first physical replacement. + disk, err := OpenJournal(path) + if err != nil { + return err + } + if _, ok := disk.Get(first.LogicalID()); !ok { + return errors.New("second writer overtook first commit") + } + close(secondEntered) + <-secondRelease + } + return replaceJournalSnapshot(path, data) + } + firstDone, secondDone := make(chan error, 1), make(chan error, 1) + go func() { _, err := j.Begin(first, 1, "a", time.Now()); firstDone <- err }() + waitJournalSignal(t, entered) + go func() { _, err := j.Begin(second, 1, "b", time.Now()); secondDone <- err }() + // This is the actual Get used by reconciliation and replay, not merely + // a proxy mutex check. Neither pending intent may be exposed as durable. + readDone := make(chan struct{}) + go func() { + defer close(readDone) + if _, ok := j.Get(first.LogicalID()); ok { + t.Error("uncommitted first intent leaked to reader") + } + }() + waitJournalSignal(t, readDone) + select { + case err := <-secondDone: + t.Fatalf("later dirty state falsely acknowledged: %v", err) + default: + } + close(release) + waitJournalSignal(t, secondEntered) + if err := <-firstDone; err != nil { + t.Fatal(err) + } + if _, ok := j.Get(first.LogicalID()); !ok { + t.Fatal("committed first intent unavailable while next writer is slow") + } + if _, ok := j.Get(second.LogicalID()); ok { + t.Fatal("older commit acknowledged newer pending state") + } + close(secondRelease) + if err := <-secondDone; err != nil { + t.Fatal(err) + } + reopened, err := OpenJournal(path) + if err != nil { + t.Fatal(err) + } + for _, e := range []Effect{first, second} { + if op, ok := reopened.Get(e.LogicalID()); !ok || op.Status != StatusPlanned { + t.Fatalf("restart lost unresolved intent: %+v, %v", op, ok) + } + } +} + +func TestJournalFailedWriterDoesNotLoseQueuedOrRetriedOperations(t *testing.T) { + j, err := OpenJournal(filepath.Join(t.TempDir(), "journal.json")) + if err != nil { + t.Fatal(err) + } + first := crossProcessEffect("acme/widgets#1", "first") + second := crossProcessEffect("acme/widgets#2", "second") + entered, release := make(chan struct{}), make(chan struct{}) + writes := 0 + j.writeSnapshot = func(path string, data []byte) error { + writes++ + if writes == 1 { + close(entered) + <-release + return os.ErrPermission + } + return replaceJournalSnapshot(path, data) + } + failed, queued := make(chan error, 1), make(chan error, 1) + go func() { _, err := j.Begin(first, 1, "a", time.Now()); failed <- err }() + waitJournalSignal(t, entered) + go func() { _, err := j.Begin(second, 1, "b", time.Now()); queued <- err }() + close(release) + if err := <-failed; !errors.Is(err, os.ErrPermission) { + t.Fatalf("write failure hidden: %v", err) + } + if err := <-queued; err != nil { + t.Fatal(err) + } + if _, ok := j.Get(first.LogicalID()); ok { + t.Fatal("failed write published an intent") + } + if _, err := j.Begin(first, 1, "a", time.Now()); err != nil { + t.Fatalf("failed intent not retryable: %v", err) + } + reopened, err := OpenJournal(j.path) + if err != nil { + t.Fatal(err) + } + for _, e := range []Effect{first, second} { + if _, ok := reopened.Get(e.LogicalID()); !ok { + t.Fatal("queued or retried operation lost on restart") + } + } +} + +func TestExecutorCanceledJournalAdmissionNeverInvokesExternalEffect(t *testing.T) { + x, claim, now := newExecutor(t, proof.ModeEnforce) + entry, err := x.Ledger.Acquire(claim, "alice", ttl, now) + if err != nil { + t.Fatal(err) + } + x.Journal.writer <- struct{}{} + defer func() { <-x.Journal.writer }() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + calls := 0 + if _, err := x.ExecuteContext(ctx, testEffect(), entry.Epoch, "alice", func() (string, error) { + calls++ + return "must-not-run", nil + }); !errors.Is(err, context.Canceled) || calls != 0 { + t.Fatalf("canceled optional caller executed through a busy journal: err=%v calls=%d", err, calls) + } +} + +func TestExecutorCancellationDuringIntentLeavesRestartReconciliation(t *testing.T) { + x, claim, now := newExecutor(t, proof.ModeEnforce) + entry, err := x.Ledger.Acquire(claim, "alice", ttl, now) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + x.Journal.writeSnapshot = func(path string, data []byte) error { + cancel() + return replaceJournalSnapshot(path, data) + } + calls := 0 + e := testEffect() + if _, err := x.ExecuteContext(ctx, e, entry.Epoch, "alice", func() (string, error) { + calls++ + return "must-not-run", nil + }); !errors.Is(err, context.Canceled) || calls != 0 { + t.Fatalf("effect ran after cancellation during intent: err=%v calls=%d", err, calls) + } + reopened, err := OpenJournal(x.Journal.path) + if err != nil { + t.Fatal(err) + } + if got, ok := reopened.Get(e.LogicalID()); !ok || got.Status != StatusPlanned { + t.Fatalf("restart lost unresolved durable intent: %+v %v", got, ok) + } + if _, err := reopened.Begin(e, entry.Epoch, "alice", now); !errors.Is(err, ErrNeedsReconciliation) { + t.Fatalf("canceled intent blindly replayed after restart: %v", err) + } +} + +func TestJournalCancellationWaitsOnlyForAdmissionNotOptimisticAcknowledgement(t *testing.T) { + j, err := OpenJournal(filepath.Join(t.TempDir(), "journal.json")) + if err != nil { + t.Fatal(err) + } + e := testEffect() + op, err := j.Begin(e, 1, "a", time.Now()) + if err != nil { + t.Fatal(err) + } + entered, release := make(chan struct{}), make(chan struct{}) + j.writeSnapshot = func(path string, data []byte) error { + close(entered) + <-release + return replaceJournalSnapshot(path, data) + } + done := make(chan error, 1) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + _, err := j.RecordResultContext(ctx, op.LogicalID, 1, StatusApplied, "applied", time.Now()) + done <- err + }() + waitJournalSignal(t, entered) + cancel() + if got, _ := j.Get(op.LogicalID); got.Status != StatusPlanned { + t.Fatal("canceled writer exposed an uncommitted acknowledgment") + } + if _, err := j.BeginContext(ctx, crossProcessEffect("acme/widgets#2", "second"), 1, "b", time.Now()); !errors.Is(err, context.Canceled) { + t.Fatalf("queued caller ignored cancellation: %v", err) + } + select { + case err := <-done: + t.Fatalf("physical replacement was optimistically acknowledged: %v", err) + default: + } + close(release) + if err := <-done; err != nil { + t.Fatal(err) + } + j.writeSnapshot = func(string, []byte) error { t.Error("unchanged replay rewrote journal"); return nil } + if _, err := j.RecordResult(op.LogicalID, 1, StatusApplied, "applied", time.Now()); err != nil { + t.Fatal(err) + } + reopened, err := OpenJournal(j.path) + if err != nil { + t.Fatal(err) + } + if got, _ := reopened.Get(op.LogicalID); got.Status != StatusApplied || got.Result != "applied" { + t.Fatalf("cancellation lost the completed durable result: %+v", got) + } +} diff --git a/src/pkg/dashboard/api_repo_item_decide_delegated.go b/src/pkg/dashboard/api_repo_item_decide_delegated.go index 6bbbb62e8a..2de7b59940 100644 --- a/src/pkg/dashboard/api_repo_item_decide_delegated.go +++ b/src/pkg/dashboard/api_repo_item_decide_delegated.go @@ -96,18 +96,20 @@ type delegatedDecisionRequest struct { // delegatedDecisionJob is the wire shape of one job. type delegatedDecisionJob struct { - ID string `json:"id"` - Repo string `json:"repo"` - Number int `json:"number"` - Type string `json:"type"` - Status string `json:"status"` - Requester string `json:"requester"` - StartedAt time.Time `json:"started_at"` - EndedAt *time.Time `json:"ended_at,omitempty"` - Decision *delegatedecision.Decision `json:"decision,omitempty"` - Notes []string `json:"notes,omitempty"` - Error string `json:"error,omitempty"` - Marker string `json:"marker,omitempty"` + ID string `json:"id"` + Repo string `json:"repo"` + Number int `json:"number"` + Type string `json:"type"` + Status string `json:"status"` + Requester string `json:"requester"` + StartedAt time.Time `json:"started_at"` + EndedAt *time.Time `json:"ended_at,omitempty"` + Decision *delegatedecision.Decision `json:"decision,omitempty"` + Notes []string `json:"notes,omitempty"` + Error string `json:"error,omitempty"` + Marker string `json:"marker,omitempty"` + ApprovalRecordedAt *time.Time `json:"approval_recorded_at,omitempty"` + ApprovalHeadSHA string `json:"approval_head_sha,omitempty"` // Result is the shared row-action outcome (what was decided, on which // item, what happens next), added for the dashboard's result toast. Result *RowActionResult `json:"result,omitempty"` @@ -496,6 +498,18 @@ func (s *Server) decideDelegatedOne(ctx context.Context, repoFull string, number s.audit.LogRecord(user, auditRepoItemDecisionDelegated+"_failed", auditDetail("repo", repoFull, "number", strconv.Itoa(number), "error", err.Error()), "", repoFull, number) return delegatedOutcome{Err: err} } + // Consult the applied native decision BEFORE invoking any paid decider. + // A posted AI verdict alone is not proof of durable application. + if in.Kind == delegatedecision.KindPR && !req.Force { + if approval, ok := repoItemDecisionChangeRequests().For(repoFull, number, in.HeadSHA); ok && + approval.IsApproval() && approval.Actor == user && approval.BodyDigest != "" && + approval.BodyDigest == review.PullRequestBodyDigest(in.Title, in.Body) { + d := delegatedecision.Decision{Verdict: delegatedecision.VerdictApprove, Rule: "recorded-approval", Text: "approve"} + result := rowActionNoopResult(rowActionTarget(repoFull, rowActionTargetPR, number), "native approval already recorded for this head", approval.RecordedAt, "") + result.Next = "Approval is recorded, not merge readiness: current review, CI, hold, sentinel and queue requirements still apply." + return delegatedOutcome{Decision: &d, Notes: []string{"native approval already applied; no paid re-decision or reporting replay"}, Result: &result} + } + } // A structured blocker — the App lacks a permission, a CLA must be // signed — is not a question the hive can answer: a direction would // un-park the issue only for the agent to re-park it with the same @@ -637,6 +651,8 @@ func (s *Server) gatherDelegatedInput(ctx context.Context, repoFull string, numb if err != nil { return in, fmt.Errorf("reading PR %s#%d: %w", repoFull, number, err) } + in.HeadSHA = strings.TrimSpace(pr.GetHead().GetSHA()) + in.Title, in.Body = pr.GetTitle(), pr.GetBody() for _, l := range pr.Labels { if name := l.GetName(); name != "" && !slices.Contains(in.Labels, name) { in.Labels = append(in.Labels, name) @@ -875,18 +891,69 @@ func delegatedLowRiskPath(path string) bool { return false } +const delegatedApprovalReportingTimeout = 15 * time.Second + // executeDelegatedDecision runs the allowed actions through the existing // GitHub helpers. It stops at the first failure and reports what landed. -func (s *Server) executeDelegatedDecision(ctx context.Context, in delegatedecision.Input, d delegatedecision.Decision, user string, mergerRole bool) ([]string, error) { +func (s *Server) executeDelegatedDecision(ctx context.Context, in delegatedecision.Input, d delegatedecision.Decision, user string, mergerRole bool) (notes []string, execErr error) { client := s.deps.GHClient repoFull, number := in.Repo, in.Number hiveID := s.delegateHiveID() marker := delegatedecision.Marker(hiveID) comment := delegatedecision.CommentBodyWith(d, hiveID, user, delegatedecision.CommentOptions{Kind: in.Kind, EffectTimeout: s.delegatedEffectTimeout()}) - var notes []string if d.Verdict == delegatedecision.VerdictRefused { return append(notes, "refused: "+d.Rationale), nil } + approvalApplied := false + defer func() { + if approvalApplied && execErr != nil { + notes = append(notes, "native approval recorded; external reporting incomplete: "+execErr.Error()) + s.audit.LogRecord(user, auditRepoItemDecisionDelegated+"_reporting_failed", auditDetail("repo", repoFull, "number", strconv.Itoa(number), "head", in.HeadSHA, "error", execErr.Error()), "", repoFull, number) + execErr = nil // The authoritative decision did land; do not re-decide it. + } + }() + if in.Kind == delegatedecision.KindPR && mergerRole && d.Verdict == delegatedecision.VerdictApprove && !d.KeepsPark && !in.SentinelAlert { + for _, action := range d.Actions { + if action.Kind != delegatedecision.ActionApproveReview { + continue + } + if in.HeadSHA == "" || strings.TrimSpace(in.Author) == "" { + return notes, errors.New("approval requires the head and author validated by the decider") + } + event, err := client.RecordOperatorPullRequestApprovalWith(ctx, repoFull, number, user, ghpkg.PullRequestApprovalOptions{ + HumanChangeRequests: repoItemDecisionChangeRequests(), LiftNeedsHuman: true, + ExpectedHead: in.HeadSHA, ExpectedBodyDigest: review.PullRequestBodyDigest(in.Title, in.Body), ExpectedAuthor: in.Author, + GuardSentinelLabel: s.sentinelLabel(), VerifyActorWrite: true, + Recorded: func(event ghpkg.PullRequestApprovalEvent) { + approvalApplied = event.Recorded + now := time.Now() + s.delegatedJobs.mu.Lock() + if job := s.delegatedJobs.jobs[delegatedJobKey(repoFull, number)]; job != nil { + job.ApprovalRecordedAt = &now + job.ApprovalHeadSHA = event.HeadSHA + } + s.delegatedJobs.mu.Unlock() + // The receipt projects the applied approval immediately, + // even while bounded optional reporting is still pending. + s.settleOperatorAction(repoFull, number, &delegatedDecisionJob{Status: delegatedJobStatusDone, Decision: &d, ApprovalHeadSHA: event.HeadSHA}) + }, + }) + if err != nil { + return notes, fmt.Errorf("recording validated approval: %w", err) + } + notes = append(notes, "native approval durably recorded on the validated head; all other merge gates remain") + if event.ReportingError != "" { + notes = append(notes, "native approval recorded; external reporting incomplete: "+event.ReportingError) + s.audit.LogRecord(user, auditRepoItemDecisionDelegated+"_reporting_failed", auditDetail("repo", repoFull, "number", strconv.Itoa(number), "head", in.HeadSHA, "error", event.ReportingError), "", repoFull, number) + } + break + } + } + if approvalApplied { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, delegatedApprovalReportingTimeout) + defer cancel() + } for _, a := range d.Actions { switch a.Kind { case delegatedecision.ActionCommentDecision: @@ -921,24 +988,18 @@ func (s *Server) executeDelegatedDecision(ctx context.Context, in delegatedecisi continue } if mergerRole { - queued, err := client.QueuePRAutoMergeWithNoteResult(ctx, repoFull, number, user, marker) + queued, err := client.QueuePRAutoMergeWithNoteResultAtHead(ctx, repoFull, number, user, marker, in.HeadSHA) if err != nil { return notes, fmt.Errorf("approving PR head: %w", err) } + if queued.ReportingError != "" { + notes = append(notes, "native approval recorded; Prow bridge incomplete: "+queued.ReportingError) + s.audit.LogRecord(user, auditRepoItemDecisionDelegated+"_reporting_failed", auditDetail("repo", repoFull, "number", strconv.Itoa(number), "head", in.HeadSHA, "error", queued.ReportingError), "", repoFull, number) + } if queued.ReviewSubmitted { notes = append(notes, "Approve review submitted on the head and queued for auto-merge (the engine merges per repo settings)") } else { - // The hive's own PR: no review is possible, so the - // operator's approval goes on record the way Approve - // records it and the self-authored lane merges on it. - if _, err := client.RecordOperatorPullRequestApprovalWith(ctx, repoFull, number, user, ghpkg.PullRequestApprovalOptions{ - HumanChangeRequests: repoItemDecisionChangeRequests(), - LiftNeedsHuman: true, - SentinelLabel: s.sentinelLabel(), - }); err != nil { - return notes, fmt.Errorf("approving PR: %w", err) - } - notes = append(notes, "approval recorded on the hive's own head and queued for auto-merge (the App cannot review its own pull request; the self-authored lane merges once CI is green)") + notes = append(notes, "queue attribution recorded on the hive's own head (the App cannot review its own pull request; native approval and all other gates are evaluated separately)") } } else { if _, err := client.RecordOperatorPullRequestApproval(ctx, repoFull, number, user); err != nil { diff --git a/src/pkg/dashboard/api_repo_item_decision.go b/src/pkg/dashboard/api_repo_item_decision.go index 7729b6cf8d..a5573127c7 100644 --- a/src/pkg/dashboard/api_repo_item_decision.go +++ b/src/pkg/dashboard/api_repo_item_decision.go @@ -725,7 +725,7 @@ func (s *Server) handlePullRequestDecision(ctx context.Context, w http.ResponseW err error ) var released []string - stage, commandsSkipped := "", "" + stage, commandsSkipped, reportingError := "", "", "" sentinelOpen, prow := false, false toContributor := "" if decision == repoItemDecisionApprove { @@ -738,6 +738,7 @@ func (s *Server) handlePullRequestDecision(ctx context.Context, w http.ResponseW head = event.HeadSHA released, stage, sentinelOpen, prow = repoItemApprovalReleased(event), event.Stage, event.SentinelLabelOpen, event.ProwManaged commandsSkipped = event.CommandsSkipped + reportingError = event.ReportingError } else { var event ghpkg.PullRequestDecisionEvent event, err = gh.RecordOperatorPullRequestDecision(ctx, repoFull, number, user, comment, repoItemDecisionChangeRequests()) @@ -779,6 +780,10 @@ func (s *Server) handlePullRequestDecision(ctx context.Context, w http.ResponseW result := repoItemDecisionResultWith(repoFull, number, "pr", decision, comment, "", "", repoItemApproveFacts{ User: user, Released: released, Stage: stage, SentinelOpen: sentinelOpen, SentinelLabel: s.sentinelLabel(), SoakMinutes: s.approveSoakMinutes(admitted != ""), Prow: prow, CommandsSkipped: commandsSkipped != "", ToContributor: toContributor, }) + if reportingError != "" { + s.audit.LogRecord(user, auditRepoItemDecisionApprove+"_reporting_failed", auditDetail("repo", repoFull, "number", numStr, "head", head, "error", reportingError), "", repoFull, number) + result.Next = "Native approval is recorded; external reporting is incomplete: " + reportingError + ". Current merge gates still apply." + } var rebase *StageTrackerRebase if decision == repoItemDecisionApprove { // An approval on a PR that is behind or conflicting does not diff --git a/src/pkg/dashboard/api_repo_item_decision_test.go b/src/pkg/dashboard/api_repo_item_decision_test.go index 325fda1bdb..2b2b57e698 100644 --- a/src/pkg/dashboard/api_repo_item_decision_test.go +++ b/src/pkg/dashboard/api_repo_item_decision_test.go @@ -75,6 +75,9 @@ type repoItemDecisionMock struct { issueState string // commentError / editError refuse POST /comments or PATCH /issues/N. commentError, editError int + statusError int + commentBlockText string + commentStarted chan struct{} // editPaths records each PATCH path alongside edits. editPaths []string } @@ -90,6 +93,11 @@ func newRepoItemDecisionServer(t *testing.T, m *repoItemDecisionMock) *Server { case r.Method == http.MethodGet && (strings.HasSuffix(path, "/collaborators/writer/permission") || strings.HasSuffix(path, "/collaborators/someone/permission")): _, _ = io.WriteString(w, `{"permission":"write"}`) case r.Method == http.MethodPost && strings.Contains(path, "/statuses/"): + if m.statusError != 0 { + w.WriteHeader(m.statusError) + _, _ = io.WriteString(w, `{"message":"Resource not accessible by integration"}`) + return + } _, _ = io.WriteString(w, `{}`) case r.Method == http.MethodGet && strings.HasSuffix(path, "/collaborators/reader/permission"): _, _ = io.WriteString(w, `{"permission":"read"}`) @@ -227,6 +235,11 @@ func newRepoItemDecisionServer(t *testing.T, m *repoItemDecisionMock) *Server { } var c map[string]string _ = json.NewDecoder(r.Body).Decode(&c) + if m.commentBlockText != "" && strings.Contains(c["body"], m.commentBlockText) { + close(m.commentStarted) + <-r.Context().Done() + return + } m.comments = append(m.comments, c["body"]) _, _ = io.WriteString(w, `{"id":1}`) case r.Method == http.MethodGet && strings.Contains(path, "/labels/"): diff --git a/src/pkg/dashboard/delegated_approval_ordering_test.go b/src/pkg/dashboard/delegated_approval_ordering_test.go new file mode 100644 index 0000000000..4445b05d4e --- /dev/null +++ b/src/pkg/dashboard/delegated_approval_ordering_test.go @@ -0,0 +1,189 @@ +package dashboard + +import ( + "context" + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/delegatedecision" + "github.com/hivecommons/hive/pkg/review" + "github.com/hivecommons/hive/pkg/stagelabels" +) + +const delegatedOrderingHead = "feedfacefeedfacefeedfacefeedfacefeedface" + +func TestDelegatedAppliedApprovalSurvivesReportingFailureAndRestartWithoutPaidRepeat(t *testing.T) { + for _, status := range []int{0, http.StatusForbidden} { + t.Run(http.StatusText(status), func(t *testing.T) { + mock := &repoItemDecisionMock{commentError: status} + s := newDelegatedServer(t, mock) + path := filepath.Join(t.TempDir(), "approvals.json") + repoItemDecisionChangeRequests = func() review.HumanChangeRequestStore { return review.NewHumanChangeRequests(path) } + decider := &fakeDecider{err: errors.New("use deterministic default")} + withDecider(s, decider, true) + job := decodeDelegatedJob(t, postDelegated(t, s, 8, "writer", `{"type":"pr"}`, map[string]string{"X-Hive-Role": "merger"})) + if job.Status != delegatedJobStatusDone || job.ApprovalRecordedAt == nil || job.Error != "" { + t.Fatalf("applied native decision did not complete: %+v", job) + } + if job.Result == nil || !strings.HasPrefix(job.Result.Outcome, "Native approval recorded") || strings.Contains(job.Result.Outcome, "`/hive decision` recorded") { + t.Fatalf("AI comment and applied native approval are conflated: %+v", job.Result) + } + approval, ok := repoItemDecisionChangeRequests().For("testorg/testrepo", 8, delegatedOrderingHead) + if !ok || !approval.IsApproval() || approval.Actor != "writer" { + t.Fatalf("exact-head approval missing after reporting: %+v %v", approval, ok) + } + if status != 0 && !strings.Contains(strings.Join(job.Notes, "\n"), "external reporting incomplete") { + t.Fatalf("partial reporting was hidden: %+v", job) + } + if status != 0 && !strings.Contains(job.Result.Next, "reporting is incomplete") { + t.Fatalf("next gate hides failed external reporting: %+v", job.Result) + } + calls := decider.calls + // A new process has neither the job nor the AI comment (the 403 + // case never posted it); only the persistent native approval. + s.delegatedJobs = delegatedDecisionJobs{} + out := s.decideDelegatedOne(context.Background(), "testorg/testrepo", 8, "pr", "writer", true, delegatedDecisionRequest{}) + if out.Err != nil || out.Result == nil || out.Result.Action != rowActionNoop || decider.calls != calls { + t.Fatalf("restart repeated a paid decision after durable application: %+v calls=%d -> %d", out, calls, decider.calls) + } + }) + } +} + +func TestDelegatedFailedNativeRecordCannotBecomeAppliedReplay(t *testing.T) { + mock := &repoItemDecisionMock{} + s := newDelegatedServer(t, mock) + path := filepath.Join(t.TempDir(), "approvals.json") + store := review.NewHumanChangeRequests(path) + repoItemDecisionChangeRequests = func() review.HumanChangeRequestStore { return store } + if err := os.Mkdir(path+".tmp", 0o755); err != nil { + t.Fatal(err) + } + body := `{"type":"pr"}` + headers := map[string]string{"X-Hive-Role": "merger"} + failed := decodeDelegatedJob(t, postDelegated(t, s, 8, "writer", body, headers)) + if failed.Status != delegatedJobStatusFailed || failed.ApprovalRecordedAt != nil { + t.Fatalf("failed persistence was acknowledged: %+v", failed) + } + if _, ok := store.For("testorg/testrepo", 8, delegatedOrderingHead); ok || len(mock.comments) != 0 { + t.Fatal("failed native record leaked an applied receipt or reporting") + } + if err := os.Remove(path + ".tmp"); err != nil { + t.Fatal(err) + } + retried := decodeDelegatedJob(t, postDelegated(t, s, 8, "writer", body, headers)) + if retried.Status != delegatedJobStatusDone || retried.ApprovalRecordedAt == nil || retried.Result == nil || retried.Result.Action == rowActionNoop { + t.Fatalf("same-process retry falsely replayed a failed approval: %+v", retried) + } + if got, ok := review.NewHumanChangeRequests(path).For("testorg/testrepo", 8, delegatedOrderingHead); !ok || !got.IsApproval() { + t.Fatal("healthy retry did not persist native approval") + } +} + +func TestDelegatedNativeApprovalAndProjectionPrecedeHungProwWarning(t *testing.T) { + started := make(chan struct{}) + mock := &repoItemDecisionMock{ + statusError: http.StatusForbidden, commentBlockText: "could not set the `prow/lgtm`", + commentStarted: started, prLabels: []string{stagelabels.NeedsDecision.Label(), "needs-human"}, + } + s := newDelegatedServer(t, mock) + delegatedDecisionRunAsync = true + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s.deps.Ctx = ctx + s.deps.GHClient.SetAutoMergeLabel("lgtm") + w := postDelegated(t, s, 8, "writer", `{"type":"pr","head":"`+delegatedOrderingHead+`"}`, map[string]string{"X-Hive-Role": "merger"}) + if w.Code != http.StatusAccepted { + t.Fatalf("job not accepted: %d %s", w.Code, w.Body.String()) + } + select { + case <-started: + case <-time.After(3 * time.Second): + t.Fatal("optional Prow warning was not reached") + } + job, _ := s.delegatedJobs.get(delegatedJobKey("testorg/testrepo", 8)) + if job.ApprovalRecordedAt == nil { + t.Fatal("hung bridge still withheld native decision checkpoint") + } + approval, ok := repoItemDecisionChangeRequests().For("testorg/testrepo", 8, delegatedOrderingHead) + if !ok || !approval.IsApproval() { + t.Fatal("native approval absent while optional bridge is blocked") + } + items := []StageTrackerItem{{Repo: "testorg/testrepo", Kind: "pr", Number: 8, Head: delegatedOrderingHead, Stage: 85, Modifiers: []string{modNeedsHuman, modNeedsDecision}}} + s.applyOperatorActions(items, time.Now()) + s.applyDecidedProjection(items, time.Now()) + if items[0].Stage == 85 || items[0].DecisionNeeded != nil || items[0].Decided == nil { + t.Fatalf("tracker still asks for the already-applied human decision: %+v", items[0]) + } + s.stageTrackerCacheMu.Lock() + stale := s.stageTrackerCache.stale + s.stageTrackerCacheMu.Unlock() + if !stale { + t.Fatal("native approval did not invalidate cached projection") + } + cancel() // Supported server lifecycle cancellation ends optional transport. + deadline := time.After(3 * time.Second) + for { + job, _ = s.delegatedJobs.get(delegatedJobKey("testorg/testrepo", 8)) + if job.Status != delegatedJobStatusRunning { + break + } + select { + case <-deadline: + t.Fatal("applied decision remained running after reporting cancellation") + case <-time.After(time.Millisecond): + } + } + if job.Status != delegatedJobStatusDone || job.Error != "" || !strings.Contains(strings.Join(job.Notes, "\n"), "Prow bridge incomplete") { + t.Fatalf("Prow reporting failure lost or reversed the decision: %+v", job) + } + if !strings.Contains(strings.Join(decisionAuditActions(s), ","), auditRepoItemDecisionDelegated+"_reporting_failed") { + t.Fatal("Prow reporting failure not audited") + } +} + +func TestDelegatedChangedApprovalTargetFailsBeforeDecisionComment(t *testing.T) { + for _, mode := range []string{"head", "body", "author", "actor"} { + t.Run(mode, func(t *testing.T) { + mock := &repoItemDecisionMock{} + s := newDelegatedServer(t, mock) + in := delegatedecision.Input{Repo: "testorg/testrepo", Number: 8, Kind: delegatedecision.KindPR, HeadSHA: delegatedOrderingHead, Author: "hive[bot]", HiveAuthored: true} + actor := "writer" + switch mode { + case "head": + in.HeadSHA = "old-head" + case "body": + in.Body = "old-body" + case "author": + in.Author = "old-author" + case "actor": + actor = "reader" + } + d := delegatedecision.Decision{Verdict: delegatedecision.VerdictApprove, Text: "approve", + Actions: []delegatedecision.Action{{Kind: delegatedecision.ActionCommentDecision}, {Kind: delegatedecision.ActionApproveReview}}} + if _, err := s.executeDelegatedDecision(context.Background(), in, d, actor, true); err == nil { + t.Fatal("changed approval target was accepted") + } + if _, ok := repoItemDecisionChangeRequests().For("testorg/testrepo", 8, delegatedOrderingHead); ok || len(mock.comments) != 0 { + t.Fatalf("changed target got native approval or success-shaped comment: comments=%v", mock.comments) + } + }) + } +} + +func TestDelegatedRejectedDecisionDoesNotRecordNativeApproval(t *testing.T) { + mock := &repoItemDecisionMock{prLabels: []string{"hold"}} + s := newDelegatedServer(t, mock) + job := decodeDelegatedJob(t, postDelegated(t, s, 8, "writer", `{"type":"pr"}`, map[string]string{"X-Hive-Role": "merger"})) + if job.Decision == nil || job.Decision.Verdict != delegatedecision.VerdictRefused || job.ApprovalRecordedAt != nil { + t.Fatalf("rejection behavior changed: %+v", job) + } + if _, ok := repoItemDecisionChangeRequests().For("testorg/testrepo", 8, delegatedOrderingHead); ok { + t.Fatal("rejected decision recorded approval") + } +} diff --git a/src/pkg/dashboard/operator_actions.go b/src/pkg/dashboard/operator_actions.go index f2760da846..332fa684c9 100644 --- a/src/pkg/dashboard/operator_actions.go +++ b/src/pkg/dashboard/operator_actions.go @@ -223,7 +223,7 @@ func (st *operatorActionStore) Settle(repo string, number int, action, outcome, // SettleDecision is Settle with the delegate's structured verdict and // rule kept on the record. -func (st *operatorActionStore) SettleDecision(repo string, number int, action, outcome, detail, verdict, rule string) (bool, error) { +func (st *operatorActionStore) SettleDecision(repo string, number int, action, outcome, detail, verdict, rule string, appliedHead ...string) (bool, error) { if st == nil { return false, nil } @@ -237,6 +237,9 @@ func (st *operatorActionStore) SettleDecision(repo string, number int, action, o list[i].Detail = detail list[i].Verdict = verdict list[i].Rule = rule + if len(appliedHead) > 0 && appliedHead[0] != "" { + list[i].HeadSHA = appliedHead[0] + } return true, st.persist() } } @@ -564,7 +567,7 @@ func (s *Server) settleOperatorAction(repoFull string, number int, job *delegate if job.Decision != nil { verdict, rule = string(job.Decision.Verdict), job.Decision.Rule } - if _, err := s.operatorActions().SettleDecision(repoFull, number, operatorActionLetHiveDecide, outcome, detail, verdict, rule); err != nil && s.logger != nil { + if _, err := s.operatorActions().SettleDecision(repoFull, number, operatorActionLetHiveDecide, outcome, detail, verdict, rule, job.ApprovalHeadSHA); err != nil && s.logger != nil { s.logger.Warn("operator action ledger write failed", "path", operatorActionsPath, "error", err) } s.invalidateStageTrackerCache() diff --git a/src/pkg/dashboard/row_action_result.go b/src/pkg/dashboard/row_action_result.go index 0ae35c4dce..587891d64e 100644 --- a/src/pkg/dashboard/row_action_result.go +++ b/src/pkg/dashboard/row_action_result.go @@ -209,5 +209,12 @@ func delegatedRowActionResult(in delegatedecision.Input, d delegatedecision.Deci out.Next = "Hive's merge sweep merges it on its next pass once CI is green and the head has soaked." } } + if in.Kind == delegatedecision.KindPR && d.Verdict == delegatedecision.VerdictApprove && note("native approval durably recorded") { + out.Outcome = "Native approval recorded for head `" + shortHead(in.HeadSHA) + "`" + ruleTag + "." + out.Next = "Current CI, review, hold, sentinel and queue requirements still apply; approval is not merge readiness." + if note("incomplete") { + out.Next = "GitHub/Prow reporting is incomplete and audited. Native approval stands; current merge gates still apply, without another model decision." + } + } return out } diff --git a/src/pkg/delegatedecision/policy.go b/src/pkg/delegatedecision/policy.go index d63b04ba65..730f2cb3b2 100644 --- a/src/pkg/delegatedecision/policy.go +++ b/src/pkg/delegatedecision/policy.go @@ -189,12 +189,13 @@ type CompetingPR struct { // Input is everything the delegate reads about one parked item. type Input struct { - Repo string - Number int - Kind Kind - Title string - Body string - Labels []string + Repo string + Number int + Kind Kind + Title string + Body string + HeadSHA string + Labels []string // Reasons are the park reasons; Question is the reviewer's decision // question, when one was recorded. Reasons []string diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index eb5d85539a..4e71e14600 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -3086,6 +3086,8 @@ func (c *Client) QueuePRAutoMergeWithNote(ctx context.Context, repo string, numb // operator's approval in the hive's own store so the self-authored lane // merges on it (hivecommons/hive#15666). type PRAutoMergeQueueResult struct { + // ReportingError is optional Prow bridge failure, not a lost approval. + ReportingError string HeadSHA string ReviewSubmitted bool SelfAuthored bool @@ -3129,6 +3131,12 @@ func (c *Client) pullRequestSelfAuthored(pr *gh.PullRequest) bool { // applied. The same fallback runs when GitHub answers that 422 for an // author the hive did not recognise as itself. func (c *Client) QueuePRAutoMergeWithNoteResult(ctx context.Context, repo string, number int, queuedBy, note string) (PRAutoMergeQueueResult, error) { + return c.QueuePRAutoMergeWithNoteResultAtHead(ctx, repo, number, queuedBy, note, "") +} + +// QueuePRAutoMergeWithNoteResultAtHead refuses a queue write if the head moved +// after the operator's decision was validated. +func (c *Client) QueuePRAutoMergeWithNoteResultAtHead(ctx context.Context, repo string, number int, queuedBy, note, expectedHead string) (PRAutoMergeQueueResult, error) { if c == nil { return PRAutoMergeQueueResult{}, ErrNoGitHubClient } @@ -3148,6 +3156,9 @@ func (c *Client) QueuePRAutoMergeWithNoteResult(ctx context.Context, repo string if headSHA == "" { return PRAutoMergeQueueResult{}, errors.New("PR head SHA is required for auto-merge approval") } + if expectedHead != "" && headSHA != expectedHead { + return PRAutoMergeQueueResult{}, errors.New("PR head changed since approval; queue reporting deferred") + } full := owner + "/" + repoName res := PRAutoMergeQueueResult{HeadSHA: headSHA, SelfAuthored: c.pullRequestSelfAuthored(pr)} if blocked := c.sentinelBlockedLabel(labelNames(pr.Labels)); blocked != "" { @@ -3213,7 +3224,9 @@ func (c *Client) QueuePRAutoMergeWithNoteResult(ctx context.Context, repo string return res, fmt.Errorf("adding %s label: %w", label, err) } if strings.EqualFold(label, prowLgtmLabel) { - c.bindProwLgtm(ctx, owner, repoName, number, headSHA, queuedBy) + if err := c.bindProwLgtm(ctx, owner, repoName, number, headSHA, queuedBy); err != nil { + res.ReportingError = err.Error() + } } return res, nil } @@ -3271,7 +3284,7 @@ const ( // bindProwLgtm records the `prow/lgtm` success status on headSHA, the commit // the queue review approved. The label and review are already in place, so a // failure is logged and surfaced on the PR but never returned. -func (c *Client) bindProwLgtm(ctx context.Context, owner, repoName string, number int, headSHA, queuedBy string) { +func (c *Client) bindProwLgtm(ctx context.Context, owner, repoName string, number int, headSHA, queuedBy string) error { sha7 := headSHA if len(sha7) > 7 { sha7 = sha7[:7] @@ -3294,17 +3307,21 @@ func (c *Client) bindProwLgtm(ctx context.Context, owner, repoName string, numbe return effects.Result{Provenance: owner + "/" + repoName + "@" + headSHA}, apiErr }) if err == nil { - return + return nil } if c.logger != nil { c.logger.Warn("could not set the prow/lgtm commit status; the GitHub App likely lacks the statuses: write permission", "repo", owner+"/"+repoName, "number", number, "error", err) } note := "Hive queued this PR but could not set the `prow/lgtm` commit status, so Prow may strip the `lgtm` label and the auto-merge queue may not hold." - if cerr := c.CreateIssueComment(ctx, owner+"/"+repoName, number, note); cerr != nil && c.logger != nil { - c.logger.Warn("could not post the prow/lgtm status warning comment", - "repo", owner+"/"+repoName, "number", number, "error", cerr) + if cerr := c.CreateIssueComment(ctx, owner+"/"+repoName, number, note); cerr != nil { + if c.logger != nil { + c.logger.Warn("could not post the prow/lgtm status warning comment", + "repo", owner+"/"+repoName, "number", number, "error", cerr) + } + return fmt.Errorf("Prow status reporting: %w; warning comment: %v", err, cerr) } + return fmt.Errorf("Prow status reporting: %w", err) } func (c *Client) ensureLabel(ctx context.Context, owner, repo, name string) error { diff --git a/src/pkg/github/pr_approval_ordering_test.go b/src/pkg/github/pr_approval_ordering_test.go new file mode 100644 index 0000000000..dba55dfc66 --- /dev/null +++ b/src/pkg/github/pr_approval_ordering_test.go @@ -0,0 +1,133 @@ +package github + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/review" + "github.com/hivecommons/hive/pkg/stageengine/stageenginetest" +) + +func TestOperatorApprovalCommitsBeforeForbiddenOrHungReporting(t *testing.T) { + for _, mode := range []string{"forbidden", "hung"} { + t.Run(mode, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "approvals.json") + store := review.NewHumanChangeRequests(path) + var reports atomic.Int32 + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pulls/") { + _, _ = w.Write([]byte(`{"state":"open","title":"fix","body":"body","user":{"login":"hive[bot]"},"head":{"sha":"head"},"labels":[]}`)) + return + } + if r.Method == http.MethodGet { + _, _ = w.Write([]byte(`[]`)) + return + } + // Reload from disk, not just memory: reporting is downstream + // of the exact native approval's persistent commit point. + _, _ = io.Copy(io.Discard, r.Body) + req, ok := review.NewHumanChangeRequests(path).For("org/repo", 1, "head") + if !ok || !req.IsApproval() || req.Actor != "owner" { + t.Error("external reporting started before native approval committed") + } + reports.Add(1) + if mode == "hung" { + select { + case <-r.Context().Done(): + case <-release: + } + return + } + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"message":"Resource not accessible by integration"}`)) + })) + defer server.Close() + defer close(release) + c := newTestClient(t, server, "org", []string{"repo"}) + c.SetStageFactSink(&stageenginetest.Recorder{}) + event, err := c.RecordOperatorPullRequestApprovalWith(context.Background(), "org/repo", 1, "owner", PullRequestApprovalOptions{ + HumanChangeRequests: store, ExpectedHead: "head", ExpectedBodyDigest: review.PullRequestBodyDigest("fix", "body"), ExpectedAuthor: "hive[bot]", + ReportingTimeout: 100 * time.Millisecond, + }) + if err != nil || !event.Recorded || event.ReportingError == "" || reports.Load() == 0 { + t.Fatalf("durable application and reporting failure must be distinct: %+v, %v", event, err) + } + if facts := prDecisionFacts(t, c); len(facts) != 1 || facts[0].Head != "head" { + t.Fatalf("native stage projection did not precede reporting: %+v", facts) + } + }) + } +} + +func TestOperatorApprovalRejectsChangedIdentityAndEmptyActorBeforeRecord(t *testing.T) { + for _, mode := range []string{"head", "body", "author", "empty-actor", "revoked-actor", "hold", "sentinel", "closed"} { + t.Run(mode, func(t *testing.T) { + var writes atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes.Add(1) + } + state := "open" + if mode == "closed" { + state = "closed" + } + if strings.HasSuffix(r.URL.Path, "/permission") { + _, _ = w.Write([]byte(`{"permission":"read"}`)) + return + } + var labels []map[string]string + if mode == "hold" || mode == "sentinel" { + labels = append(labels, map[string]string{"name": mode}) + } + _ = json.NewEncoder(w).Encode(map[string]any{"state": state, "title": "fix", "body": "body", "labels": labels, "user": map[string]string{"login": "hive[bot]"}, "head": map[string]string{"sha": "head"}}) + })) + defer server.Close() + c := newTestClient(t, server, "org", []string{"repo"}) + store := review.NewHumanChangeRequests("") + opts := PullRequestApprovalOptions{HumanChangeRequests: store, ExpectedHead: "head", ExpectedBodyDigest: review.PullRequestBodyDigest("fix", "body"), ExpectedAuthor: "hive[bot]"} + actor := "owner" + switch mode { + case "head": + opts.ExpectedHead = "old-head" + case "body": + opts.ExpectedBodyDigest = review.PullRequestBodyDigest("fix", "old-body") + case "author": + opts.ExpectedAuthor = "someone-else" + case "empty-actor": + actor = "" + case "revoked-actor", "hold", "sentinel": + opts.VerifyActorWrite = true + opts.GuardSentinelLabel = "sentinel" + } + event, err := c.RecordOperatorPullRequestApprovalWith(context.Background(), "org/repo", 1, actor, opts) + if err == nil || event.Recorded || store.Len() != 0 || writes.Load() != 0 { + t.Fatalf("changed or unauthorized target approved: %+v err=%v writes=%d", event, err, writes.Load()) + } + }) + } +} + +func TestQueueApprovalRefusesChangedHeadBeforeAnyMutation(t *testing.T) { + var writes atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes.Add(1) + } + _, _ = w.Write([]byte(`{"state":"open","user":{"login":"contributor"},"head":{"sha":"new"}}`)) + })) + defer server.Close() + c := newTestClient(t, server, "org", []string{"repo"}) + if _, err := c.QueuePRAutoMergeWithNoteResultAtHead(context.Background(), "org/repo", 1, "owner", "decision", "old"); err == nil || writes.Load() != 0 { + t.Fatalf("stale queue approval wrote externally: err=%v writes=%d", err, writes.Load()) + } +} diff --git a/src/pkg/github/pr_decision_operator.go b/src/pkg/github/pr_decision_operator.go index 2e1ce4e039..d3844ffd24 100644 --- a/src/pkg/github/pr_decision_operator.go +++ b/src/pkg/github/pr_decision_operator.go @@ -88,6 +88,9 @@ type PullRequestApprovalEvent struct { // so posting again would only draw the same "NOT APPROVED" reply. The // hive's own approval record is unaffected. CommandsSkipped string + // ReportingError preserves partial progress: the native approval stands, + // but comments, labels or stage reporting must be retried/reconciled. + ReportingError string } // PRApprovalCommandsSkippedProw is the CommandsSkipped reason for a Prow @@ -166,6 +169,17 @@ func (c *Client) prowRejectedHiveCommands(ctx context.Context, owner, name strin // posts the commands only, which is what the delegated "Let hive decide" // path keeps: a recommendation must not clear a person's gate. type PullRequestApprovalOptions struct { + // Expected identity binds an asynchronous decision to the facts it read. + ExpectedHead string + ExpectedBodyDigest string + ExpectedAuthor string + // Delegated decisions may not clear a newly arrived hold or sentinel. + GuardSentinelLabel string + VerifyActorWrite bool + // Recorded runs after the durable approval, before optional reporting. + Recorded func(PullRequestApprovalEvent) + // ReportingTimeout bounds lock admission and network reporting, not fsync. + ReportingTimeout time.Duration // HumanChangeRequests, when set, records the approval against the head // as a bare `approve` decision — the fact withRecordedDecision and the // stage engine read as "the requires_human verdict on this head has @@ -272,6 +286,7 @@ type prDecisionTarget struct { clanker bool author string labels []string + bodyDigest string } // SetAuthorKindLabels installs the live governor.labels.author resolver. @@ -294,6 +309,7 @@ func (c *Client) openPullRequestDecisionTarget(ctx context.Context, owner, name return prDecisionTarget{}, ErrPullRequestClosed } t := prDecisionTarget{author: strings.TrimSpace(safeGetLogin(pr.GetUser()))} + t.bodyDigest = review.PullRequestBodyDigest(pr.GetTitle(), pr.GetBody()) if pr.GetHead() != nil { t.head = strings.TrimSpace(pr.GetHead().GetSHA()) } @@ -340,9 +356,9 @@ func (c *Client) RecordOperatorPullRequestApproval(ctx context.Context, repo str } // RecordOperatorPullRequestApprovalWith is RecordOperatorPullRequestApproval -// plus what opts asks the person's approval to release, in this order: the -// commands are posted first (the durable, visible record), then the -// approval is recorded against the head, the park labels the person just +// plus what opts asks the person's approval to release: the validated native +// approval is recorded against the head BEFORE optional external reporting. +// Then the commands are posted, the park labels the person just // answered come off, the sentinel acknowledgment is written, and the stage // engine is given the decision so the row moves now rather than on the // next sweep. A failed label or stage write after the record is logged and @@ -354,47 +370,78 @@ func (c *Client) RecordOperatorPullRequestApprovalWith(ctx context.Context, repo return PullRequestApprovalEvent{}, err } actor = strings.TrimSpace(actor) - head, selfAuthored, labels, err := c.openPullRequestHeadLabels(ctx, owner, name, number) + target, err := c.openPullRequestDecisionTarget(ctx, owner, name, number) if err != nil { return PullRequestApprovalEvent{}, err } - full := owner + "/" + name - event := PullRequestApprovalEvent{Repo: full, Number: number, HeadSHA: head, Actor: actor, ProwManaged: prowManagedLabels(labels)} - prowRejected, err := c.prowRejectedHiveCommands(ctx, owner, name, number) - if err != nil { - return PullRequestApprovalEvent{}, fmt.Errorf("reading comments of %s/%s#%d before posting approval commands: %w", owner, name, number, err) - } - if prowRejected { - event.CommandsSkipped = PRApprovalCommandsSkippedProw - if c.logger != nil { - c.logger.Info("pull request approval commands not posted: Prow rejected the hive's earlier commands on this PR (bot is not an approver); the hive's own approval record stands", "repo", event.Repo, "pr", number, "actor", actor, "head", head) + head, selfAuthored, labels := target.head, target.selfAuthored, target.labels + if opts.ExpectedHead != "" && head != opts.ExpectedHead || + opts.ExpectedBodyDigest != "" && target.bodyDigest != opts.ExpectedBodyDigest || + opts.ExpectedAuthor != "" && !strings.EqualFold(target.author, opts.ExpectedAuthor) { + return PullRequestApprovalEvent{}, errors.New("approval target changed since the decision; gather and validate again") + } + if opts.VerifyActorWrite { + if labelPresent(labels, "hold") || opts.GuardSentinelLabel != "" && labelPresent(labels, opts.GuardSentinelLabel) { + return PullRequestApprovalEvent{}, errors.New("approval target gained a hold or sentinel; delegate must not release it") } - } else { - for _, command := range PRApprovalCommands { - if err := c.CreateIssueComment(ctx, full, number, renderPRApprovalCommandComment(command, actor, head)); err != nil { - return PullRequestApprovalEvent{}, fmt.Errorf("posting %s on %s/%s#%d: %w", command, owner, name, number, err) - } + permission, _, err := c.client.Repositories.GetPermissionLevel(ctx, owner, name, actor) + if err != nil { + return PullRequestApprovalEvent{}, fmt.Errorf("revalidating approval actor: %w", err) } - if c.logger != nil { - c.logger.Info("pull request approval commands posted by dashboard decision", "repo", event.Repo, "pr", number, "actor", actor, "head", head) + switch strings.ToLower(permission.GetPermission()) { + case "admin", "maintain", "write": + default: + return PullRequestApprovalEvent{}, errors.New("approval actor no longer has write access") } } - if opts.HumanChangeRequests != nil && head != "" { + full := owner + "/" + name + event := PullRequestApprovalEvent{Repo: full, Number: number, HeadSHA: head, Actor: actor, ProwManaged: prowManagedLabels(labels)} + if opts.HumanChangeRequests != nil { + if head == "" { + return event, ErrPullRequestHeadUnknown + } req := review.HumanChangeRequest{ - Repo: full, - Number: number, - HeadSHA: head, - Source: review.HumanChangeRequestSourceDecision, - SourceID: operatorDecisionMarkerID, - Actor: actor, - Body: prDecisionApproveBody, - Kind: review.HumanChangeRequestKindApprove, + Repo: full, Number: number, HeadSHA: head, BodyDigest: target.bodyDigest, + Source: review.HumanChangeRequestSourceDecision, SourceID: operatorDecisionMarkerID, + Actor: actor, Body: prDecisionApproveBody, Kind: review.HumanChangeRequestKindApprove, } + start := time.Now() if err := opts.HumanChangeRequests.Record(req); err != nil { return event, fmt.Errorf("recording dashboard approval on %s#%d: %w", full, number, err) } event.Recorded = true + if c.logger != nil { + c.logger.Info("native operator approval recorded; external reporting pending", "repo", full, "pr", number, "head", head, "actor", actor, "record_duration", time.Since(start)) + } + if opts.Recorded != nil { + opts.Recorded(event) + } + } + timeout := opts.ReportingTimeout + if timeout <= 0 { + timeout = defaultApprovalReportingTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + reportStarted := time.Now() + reportErr := c.reportOperatorPullRequestApproval(ctx, owner, name, number, actor, head, selfAuthored, labels, opts, &event) + if c.logger != nil { + c.logger.Info("operator approval external reporting finished", "repo", full, "pr", number, "head", head, "approval_recorded", event.Recorded, "report_duration", time.Since(reportStarted), "error", reportErr) } + if reportErr != nil { + if !event.Recorded { + return event, reportErr + } + event.ReportingError = reportErr.Error() + c.warnOperatorApproval("native approval stands; external reporting incomplete", full, number, head, reportErr) + } + return event, nil +} + +const defaultApprovalReportingTimeout = 15 * time.Second + +func (c *Client) reportOperatorPullRequestApproval(ctx context.Context, owner, name string, number int, actor, head string, selfAuthored bool, labels []string, opts PullRequestApprovalOptions, event *PullRequestApprovalEvent) error { + full := owner + "/" + name if opts.LiftNeedsHuman { event.NeedsHumanLifted = slices.Contains(c.liftPRDecisionParkLabels(ctx, full, number, head, labels), issueNeedsHumanLabel) } @@ -417,7 +464,26 @@ func (c *Client) RecordOperatorPullRequestApprovalWith(ctx context.Context, repo event.Stage = c.applyPRDecisionStage(ctx, owner, name, number, head, actor, prDecisionApprove, selfAuthored) c.logPRDecision(full, number, head, actor, prDecisionApprove, event.Stage, false) } - return event, nil + prowRejected, err := c.prowRejectedHiveCommands(ctx, owner, name, number) + if err != nil { + return fmt.Errorf("reading comments of %s/%s#%d before posting approval commands: %w", owner, name, number, err) + } + if prowRejected { + event.CommandsSkipped = PRApprovalCommandsSkippedProw + if c.logger != nil { + c.logger.Info("pull request approval commands not posted: Prow rejected the hive's earlier commands on this PR (bot is not an approver); the hive's own approval record stands", "repo", event.Repo, "pr", number, "actor", actor, "head", head) + } + } else { + for _, command := range PRApprovalCommands { + if err := c.CreateIssueComment(ctx, full, number, renderPRApprovalCommandComment(command, actor, head)); err != nil { + return fmt.Errorf("posting %s on %s/%s#%d: %w", command, owner, name, number, err) + } + } + if c.logger != nil { + c.logger.Info("pull request approval commands posted by dashboard decision", "repo", event.Repo, "pr", number, "actor", actor, "head", head) + } + } + return ctx.Err() } // prowManagedLabels reports whether labels carry Prow's plugin markers: diff --git a/src/pkg/review/human_approval_commit_test.go b/src/pkg/review/human_approval_commit_test.go new file mode 100644 index 0000000000..1beb7b328f --- /dev/null +++ b/src/pkg/review/human_approval_commit_test.go @@ -0,0 +1,41 @@ +package review + +import ( + "os" + "path/filepath" + "testing" +) + +func TestFailedNativeApprovalRecordDoesNotBecomeAppliedReplay(t *testing.T) { + path := filepath.Join(t.TempDir(), "decisions.json") + store := NewHumanChangeRequests(path) + original := HumanChangeRequest{Repo: "org/repo", Number: 1, HeadSHA: "head", Source: HumanChangeRequestSourceDecision, SourceID: 1, Actor: "owner", Body: "fix it", Kind: HumanChangeRequestKindChanges} + if err := store.Record(original); err != nil { + t.Fatal(err) + } + // A directory at the replacement path deterministically refuses a write, + // even on root CI runners; no filesystem permission assumptions. + if err := os.Mkdir(path+".tmp", 0o755); err != nil { + t.Fatal(err) + } + approval := original + approval.Kind, approval.Body, approval.BodyDigest = HumanChangeRequestKindApprove, "approve", PullRequestBodyDigest("title", "body") + if err := store.Record(approval); err == nil { + t.Fatal("broken persistent write was acknowledged") + } + if got, ok := store.For("org/repo", 1, "head"); !ok || got.IsApproval() || got.Body != original.Body { + t.Fatalf("failed approval leaked into applied-replay lookup: %+v %v", got, ok) + } + if got, ok := NewHumanChangeRequests(path).For("org/repo", 1, "head"); !ok || got.IsApproval() { + t.Fatalf("failed approval lost the prior disk record: %+v %v", got, ok) + } + if err := os.Remove(path + ".tmp"); err != nil { + t.Fatal(err) + } + if err := store.Record(approval); err != nil { + t.Fatal(err) + } + if got, ok := NewHumanChangeRequests(path).For("org/repo", 1, "head"); !ok || !got.IsApproval() || got.BodyDigest != approval.BodyDigest { + t.Fatalf("healthy retry did not persist native approval identity: %+v %v", got, ok) + } +} diff --git a/src/pkg/review/human_change_request.go b/src/pkg/review/human_change_request.go index 3f88ad8955..e4b5f9a313 100644 --- a/src/pkg/review/human_change_request.go +++ b/src/pkg/review/human_change_request.go @@ -58,6 +58,8 @@ type HumanChangeRequest struct { Repo string `json:"repo"` Number int `json:"number"` HeadSHA string `json:"head_sha"` + // BodyDigest binds dashboard approval replay to the validated title/body. + BodyDigest string `json:"body_digest,omitempty"` // Source is HumanChangeRequestSourceReview, ...Decision or ...StageLabel. Source string `json:"source"` // SourceID is the review ID or comment ID: with HeadSHA it is the @@ -188,10 +190,30 @@ func (s *HumanChangeRequests) persist() error { return err } tmp := s.path + ".tmp" - if err := os.WriteFile(tmp, data, 0o644); err != nil { + file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + return err + } + if _, err := file.Write(data); err != nil { + _ = file.Close() + return err + } + if err := file.Sync(); err != nil { + _ = file.Close() + return err + } + if err := file.Close(); err != nil { + return err + } + if err := os.Rename(tmp, s.path); err != nil { + return err + } + directory, err := os.Open(filepath.Dir(s.path)) + if err != nil { return err } - return os.Rename(tmp, s.path) + defer directory.Close() + return directory.Sync() } // Record implements HumanChangeRequestStore. @@ -210,7 +232,8 @@ func (s *HumanChangeRequests) Record(req HumanChangeRequest) error { s.mu.Lock() defer s.mu.Unlock() s.load() - kept := s.items[:0] + prior := s.items + kept := make([]HumanChangeRequest, 0, len(s.items)+1) for _, item := range s.items { if item.Number == req.Number && strings.EqualFold(item.Repo, req.Repo) { if item.HeadSHA != req.HeadSHA || (item.Source == req.Source && item.SourceID == req.SourceID) { @@ -220,7 +243,11 @@ func (s *HumanChangeRequests) Record(req HumanChangeRequest) error { kept = append(kept, item) } s.items = append(kept, req) - return s.persist() + if err := s.persist(); err != nil { + s.items = prior + return err + } + return nil } // For implements HumanChangeRequestStore. From 7b2cf658da0efd91810e262bfe9bafa02927a16d Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Mon, 2 Nov 2026 15:26:27 -0500 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=93=96=20docs:=20reanchor=20rank=20ci?= =?UTF-8?q?tation=20after=20approval=20helper=20change?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- src/docs/scale-envelope.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index 3dd2f95a60..011690f25d 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:5040-5050`, descending `AgeMinutes` within each tier). + (`RankActionableIssues`, `src/pkg/github/client.go:5058-5068`, descending `AgeMinutes` within each tier). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes → refactors/docs → tests). From b3d37184de009451964105f0f1dac570654e3c9b Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Mon, 2 Nov 2026 15:32:31 -0500 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=90=9B=20fix:=20settle=20direct=20ope?= =?UTF-8?q?rator=20queue=20decisions=20before=20reporting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- src/pkg/dashboard/api_repo_item_decision.go | 53 +++++++++------ .../delegated_approval_ordering_test.go | 67 +++++++++++++++++++ src/pkg/github/pr_decision_operator.go | 15 ++--- 3 files changed, 106 insertions(+), 29 deletions(-) diff --git a/src/pkg/dashboard/api_repo_item_decision.go b/src/pkg/dashboard/api_repo_item_decision.go index a5573127c7..ddd5422fcb 100644 --- a/src/pkg/dashboard/api_repo_item_decision.go +++ b/src/pkg/dashboard/api_repo_item_decision.go @@ -611,33 +611,44 @@ func (s *Server) handlePullRequestMergeDecision(ctx context.Context, w http.Resp // between the approval landing and the merge (hivecommons/hive#15136). admitted := s.admitPullRequestHead(repoFull, number, state.HeadSHA, user) if decision == repoItemDecisionQueueMerge { - queued, err := gh.QueuePRAutoMergeWithNoteResult(ctx, repoFull, number, user, renderRepoItemDecisionQueueNote(user, comment)) + event, err := gh.RecordOperatorPullRequestApprovalWith(ctx, repoFull, number, user, ghpkg.PullRequestApprovalOptions{ + HumanChangeRequests: repoItemDecisionChangeRequests(), LiftNeedsHuman: true, + ExpectedHead: state.HeadSHA, ExpectedAuthor: author, GuardSentinelLabel: s.sentinelLabel(), VerifyActorWrite: true, + Recorded: func(ghpkg.PullRequestApprovalEvent) { s.invalidateStageTrackerForWebhook() }, + }) if err != nil { - jsonError(w, err.Error(), http.StatusBadGateway) + jsonError(w, "recording validated queue approval: "+err.Error(), http.StatusBadGateway) return } - released := []string{"admitted"} - if !queued.ReviewSubmitted { - // The hive's own PR: the App cannot review it, so the operator's - // approval is recorded in the hive's store exactly as the plain - // Approve decision records it; the self-authored lane merges on - // that record once CI is green (hivecommons/hive#15666). - event, err := gh.RecordOperatorPullRequestApprovalWith(ctx, repoFull, number, user, ghpkg.PullRequestApprovalOptions{ - HumanChangeRequests: repoItemDecisionChangeRequests(), - LiftNeedsHuman: true, - SentinelLabel: s.sentinelLabel(), - }) - if err != nil { - jsonError(w, "recording approval on the hive's own pull request: "+err.Error(), http.StatusBadGateway) - return - } - released = append(repoItemApprovalReleased(event), released...) - } + reportCtx, cancel := context.WithTimeout(ctx, delegatedApprovalReportingTimeout) + defer cancel() + queued, queueErr := gh.QueuePRAutoMergeWithNoteResultAtHead(reportCtx, repoFull, number, user, renderRepoItemDecisionQueueNote(user, comment), event.HeadSHA) + released := append(repoItemApprovalReleased(event), "admitted") label := gh.AutoMergeLabel() s.audit.LogRecord(user, auditRepoItemDecisionQueue, auditDetail("repo", repoFull, "number", numStr, "type", "pr", "head", state.HeadSHA, "author", author, "label", label, "admitted_by", admitted, "review_submitted", strconv.FormatBool(queued.ReviewSubmitted)), "", repoFull, number) s.invalidateStageTrackerForWebhook() - jsonResponse(w, repoItemDecisionResponse{OK: true, Repo: repoFull, Number: number, Decision: decision, Status: "queued-for-merge", Label: label, Head: state.HeadSHA, AdmittedBy: admitted, Released: released, - Result: repoItemDecisionResult(repoFull, number, "pr", decision, comment, "", label, nil)}) + status := "queued-for-merge" + result := repoItemDecisionResult(repoFull, number, "pr", decision, comment, "", label, nil) + var reporting []string + if event.ReportingError != "" { + reporting = append(reporting, event.ReportingError) + } + if queueErr != nil { + reporting = append(reporting, queueErr.Error()) + } + if queued.ReportingError != "" { + reporting = append(reporting, queued.ReportingError) + } + if len(reporting) > 0 { + detail := strings.Join(reporting, "; ") + status = "approval-recorded-reporting-incomplete" + result.Action = rowActionApproved + result.Outcome = "Native approval recorded for head `" + shortHead(event.HeadSHA) + "`; queue reporting is incomplete." + result.Next = detail + ". Current merge gates still apply; the recorded approval does not need another model decision." + s.audit.LogRecord(user, auditRepoItemDecisionQueue+"_reporting_failed", auditDetail("repo", repoFull, "number", numStr, "head", event.HeadSHA, "error", detail), "", repoFull, number) + } + jsonResponse(w, repoItemDecisionResponse{OK: true, Repo: repoFull, Number: number, Decision: decision, Status: status, Label: label, Head: event.HeadSHA, AdmittedBy: admitted, Released: released, + Result: result}) return } // A head GitHub reports behind or conflicting cannot merge (405): the diff --git a/src/pkg/dashboard/delegated_approval_ordering_test.go b/src/pkg/dashboard/delegated_approval_ordering_test.go index 4445b05d4e..6ed8ee2a4b 100644 --- a/src/pkg/dashboard/delegated_approval_ordering_test.go +++ b/src/pkg/dashboard/delegated_approval_ordering_test.go @@ -2,6 +2,7 @@ package dashboard import ( "context" + "encoding/json" "errors" "net/http" "os" @@ -176,6 +177,72 @@ func TestDelegatedChangedApprovalTargetFailsBeforeDecisionComment(t *testing.T) } } +func TestOperatorQueueApprovalCommitsBeforeForbiddenReporting(t *testing.T) { + for _, mode := range []string{"comment", "prow"} { + t.Run(mode, func(t *testing.T) { + mock := &repoItemDecisionMock{} + if mode == "comment" { + mock.commentError = http.StatusForbidden + } else { + mock.statusError = http.StatusForbidden + } + s := newRepoItemDecisionServer(t, mock) + s.deps.GHClient.SetAutoMergeLabel("lgtm") + w := postRepoItemDecisionAs(t, s, 8, "writer", "merger", `{"decision":"queue-merge","type":"pr"}`) + var response repoItemDecisionResponse + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if w.Code != http.StatusOK || response.Status != "approval-recorded-reporting-incomplete" || + response.Result == nil || response.Result.Action != rowActionApproved || !strings.Contains(response.Result.Next, "merge gates still apply") { + t.Fatalf("native queue approval was withheld or falsely claimed queued: %d %+v", w.Code, response) + } + if approval, ok := repoItemDecisionChangeRequests().For("testorg/testrepo", 8, delegatedOrderingHead); !ok || !approval.IsApproval() || approval.Actor != "writer" { + t.Fatal("optional queue reporting prevented exact-head operator approval") + } + if len(mock.merges) != 0 { + t.Fatal("queue approval bypassed the merge engine") + } + }) + } +} + +func TestOperatorQueueApprovalCompletesAfterHungBridgeCancellation(t *testing.T) { + started := make(chan struct{}) + mock := &repoItemDecisionMock{statusError: http.StatusForbidden, commentBlockText: "could not set the `prow/lgtm`", commentStarted: started} + s := newRepoItemDecisionServer(t, mock) + s.deps.GHClient.SetAutoMergeLabel("lgtm") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s.deps.Ctx = ctx + done := make(chan []byte, 1) + go func() { + w := postRepoItemDecisionAs(t, s, 8, "writer", "merger", `{"decision":"queue-merge","type":"pr"}`) + done <- w.Body.Bytes() + }() + select { + case <-started: + case <-time.After(3 * time.Second): + t.Fatal("operator queue did not reach optional bridge") + } + if approval, ok := repoItemDecisionChangeRequests().For("testorg/testrepo", 8, delegatedOrderingHead); !ok || !approval.IsApproval() { + t.Fatal("operator queue still records approval after optional bridge") + } + cancel() + select { + case body := <-done: + var response repoItemDecisionResponse + if err := json.Unmarshal(body, &response); err != nil { + t.Fatal(err) + } + if response.Status != "approval-recorded-reporting-incomplete" || response.Result == nil || response.Result.Action != rowActionApproved { + t.Fatalf("operator queue did not report applied decision separately: %s", body) + } + case <-time.After(3 * time.Second): + t.Fatal("operator queue remained blocked after lifecycle cancellation") + } +} + func TestDelegatedRejectedDecisionDoesNotRecordNativeApproval(t *testing.T) { mock := &repoItemDecisionMock{prLabels: []string{"hold"}} s := newDelegatedServer(t, mock) diff --git a/src/pkg/github/pr_decision_operator.go b/src/pkg/github/pr_decision_operator.go index d3844ffd24..453ab74f87 100644 --- a/src/pkg/github/pr_decision_operator.go +++ b/src/pkg/github/pr_decision_operator.go @@ -166,8 +166,8 @@ func (c *Client) prowRejectedHiveCommands(ctx context.Context, owner, name strin // beyond posting the commands (hivecommons/hive: #14821, #14822, crux#30 // sat at stage/085 after the owner approved from the dashboard — the // comments were read by nobody but the workflow-path fence). The zero value -// posts the commands only, which is what the delegated "Let hive decide" -// path keeps: a recommendation must not clear a person's gate. +// posts the commands only: a recommendation without release authority must +// not clear a person's gate. type PullRequestApprovalOptions struct { // Expected identity binds an asynchronous decision to the facts it read. ExpectedHead string @@ -358,12 +358,11 @@ func (c *Client) RecordOperatorPullRequestApproval(ctx context.Context, repo str // RecordOperatorPullRequestApprovalWith is RecordOperatorPullRequestApproval // plus what opts asks the person's approval to release: the validated native // approval is recorded against the head BEFORE optional external reporting. -// Then the commands are posted, the park labels the person just -// answered come off, the sentinel acknowledgment is written, and the stage -// engine is given the decision so the row moves now rather than on the -// next sweep. A failed label or stage write after the record is logged and -// reported on the event, not returned: the record stands, and the lanes -// derive the same answer from it on their next pass. +// Park labels and sentinel acknowledgment are reconciled, then the stage +// engine is given the decision and the command comments are reported. +// Reporting is bounded; its failures do not undo a recorded approval. +// Failed label/stage writes remain logged and reflected in the event, and +// transport failures appear in ReportingError for the caller's audit/receipt. func (c *Client) RecordOperatorPullRequestApprovalWith(ctx context.Context, repo string, number int, actor string, opts PullRequestApprovalOptions) (PullRequestApprovalEvent, error) { owner, name, err := c.operatorPRTarget(repo, number, actor, "RecordOperatorPullRequestApproval") if err != nil { From 2a2b8daaaa5a726d9e03912f19bcd413fb6183e1 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Mon, 2 Nov 2026 15:41:18 -0500 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=90=9B=20fix:=20align=20decision=20re?= =?UTF-8?q?gression=20with=20native-first=20reporting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- .../dashboard/delegated_decision_human_merge_test.go | 11 ++++++++++- src/pkg/github/pr_decision_operator.go | 8 -------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/pkg/dashboard/delegated_decision_human_merge_test.go b/src/pkg/dashboard/delegated_decision_human_merge_test.go index 7e65c373b5..c6064cda65 100644 --- a/src/pkg/dashboard/delegated_decision_human_merge_test.go +++ b/src/pkg/dashboard/delegated_decision_human_merge_test.go @@ -92,9 +92,18 @@ func TestDelegatedDecisionUntrustedReadyForHumanMergeDecides(t *testing.T) { if len(mock.merges) != 0 { t.Fatalf("the dashboard merges nothing itself: %v", mock.merges) } - if len(mock.comments) == 0 || !strings.Contains(mock.comments[0], tc.wantRule) { + decisionComment := false + for _, comment := range mock.comments { + if job.Marker != "" && strings.Contains(comment, tc.wantRule) && strings.Contains(comment, job.Marker) { + decisionComment = true + } + } + if !decisionComment { t.Fatalf("decision comment must carry rule %q: %v", tc.wantRule, mock.comments) } + if !tc.wantAdopt && job.ApprovalRecordedAt == nil { + t.Fatal("approved native decision must be recorded before optional comments") + } // The ✓ hover reads the ledger detail: recorded from the response // on a synchronous run, settled from the job on an async one. diff --git a/src/pkg/github/pr_decision_operator.go b/src/pkg/github/pr_decision_operator.go index 453ab74f87..b239777715 100644 --- a/src/pkg/github/pr_decision_operator.go +++ b/src/pkg/github/pr_decision_operator.go @@ -268,14 +268,6 @@ func ParsePRApprovalCommandComment(body string) (command, actor, head string, ok return command, actor, head, true } -// openPullRequestHeadLabels returns the PR's head SHA, whether the hive -// opened it, and its labels as read in the same fetch, or -// ErrPullRequestClosed / a not-found error. -func (c *Client) openPullRequestHeadLabels(ctx context.Context, owner, name string, number int) (head string, selfAuthored bool, labels []string, err error) { - t, err := c.openPullRequestDecisionTarget(ctx, owner, name, number) - return t.head, t.selfAuthored, t.labels, err -} - // prDecisionTarget is what one open-PR read tells a dashboard decision: // the head, whether the hive opened the PR, and — when it did not — // whether a contributor's hive did (author/clanker), so a direction has a