From c83634df9ba5d834206c381dda3dafea01f58543 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:01:46 -0500 Subject: [PATCH 1/2] fix: proxy decision path denies on callback failure Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- ...fixed-proxy-policy-callback-fail-closed.md | 1 + src/crux/proxy/github_proxy_policy.go | 62 +++++++++- src/crux/proxy/policy_callback_guard.go | 107 ++++++++++++++++++ src/crux/proxy/policy_callback_guard_test.go | 90 +++++++++++++++ 4 files changed, 254 insertions(+), 6 deletions(-) create mode 100644 changelog.d/fixed-proxy-policy-callback-fail-closed.md create mode 100644 src/crux/proxy/policy_callback_guard.go create mode 100644 src/crux/proxy/policy_callback_guard_test.go diff --git a/changelog.d/fixed-proxy-policy-callback-fail-closed.md b/changelog.d/fixed-proxy-policy-callback-fail-closed.md new file mode 100644 index 0000000000..dcb0d4b397 --- /dev/null +++ b/changelog.d/fixed-proxy-policy-callback-fail-closed.md @@ -0,0 +1 @@ +- **Proxy write gates fail closed when a policy check fails.** If the repo pause, agent repo scope, auto-merge or write_surface.enforce check panics or hangs, the proxy now refuses that one request with a 403 naming the failed check, instead of crashing the whole proxy or holding the request open. diff --git a/src/crux/proxy/github_proxy_policy.go b/src/crux/proxy/github_proxy_policy.go index 7db0f15c48..9e578fbaae 100644 --- a/src/crux/proxy/github_proxy_policy.go +++ b/src/crux/proxy/github_proxy_policy.go @@ -71,39 +71,89 @@ func repoScopePath(req *http.Request) string { } // repoPauseRefusal is RepoPauseRefusal against this proxy's current predicate. +// A predicate that panics or times out counts as paused (see +// policy_callback_guard.go). func (p *GitHubProxy) repoPauseRefusal(method, path string) (string, bool) { p.pausedMu.RLock() paused := p.repoPaused p.pausedMu.RUnlock() - return RepoPauseRefusal(paused, method, path) + failed := false + reason, refused := RepoPauseRefusal(p.guardRepoPredicate("repo pause", paused, true, &failed), method, path) + if failed { + return policyCallbackFailedReason("repo pause"), true + } + return reason, refused } +// autoMergeRefusal is RepoAutoMergeRefusal against the current predicate. A +// predicate that panics or times out counts as auto-merge off. func (p *GitHubProxy) autoMergeRefusal(method, path string) (string, bool) { p.autoMergeMu.RLock() enabled := p.repoAutoMergeOn p.autoMergeMu.RUnlock() - return RepoAutoMergeRefusal(enabled, method, path) + failed := false + reason, refused := RepoAutoMergeRefusal(p.guardRepoPredicate("repo auto-merge", enabled, false, &failed), method, path) + if failed { + return policyCallbackFailedReason("repo auto-merge"), true + } + return reason, refused } func (p *GitHubProxy) graphQLAutoMergeRefusal(body []byte) (string, bool) { p.autoMergeMu.RLock() enabled := p.repoAutoMergeOn p.autoMergeMu.RUnlock() - return GraphQLAutoMergeRefusal(enabled, body) + failed := false + reason, refused := GraphQLAutoMergeRefusal(p.guardRepoPredicate("repo auto-merge", enabled, false, &failed), body) + if failed { + return policyCallbackFailedReason("repo auto-merge"), true + } + return reason, refused } // agentRepoScopeRefusal is AgentRepoScopeRefusal against the current predicate. +// A predicate that panics or times out counts as not scoped. func (p *GitHubProxy) agentRepoScopeRefusal(agentName, method, path string) (string, bool) { p.scopeMu.RLock() serves := p.agentServesRepo p.scopeMu.RUnlock() - return AgentRepoScopeRefusal(serves, agentName, method, path) + if serves == nil { + return AgentRepoScopeRefusal(nil, agentName, method, path) + } + failed := false + guarded := func(agent, repo string) bool { + v, ok := p.callPolicyPredicate("agent repo scope", func() bool { return serves(agent, repo) }) + if !ok { + failed = true + return false + } + return v + } + reason, refused := AgentRepoScopeRefusal(guarded, agentName, method, path) + if failed { + return policyCallbackFailedReason("agent repo scope"), true + } + return reason, refused } +// writeSurfaceEnforcedFunc returns the live write_surface.enforce predicate +// wrapped so a call that panics or times out reports the lane as enforced: +// its direct writes are refused and its GitHub hosts are intercepted, the +// deny direction for both callers. func (p *GitHubProxy) writeSurfaceEnforcedFunc() func(agent string) bool { p.writeSurfaceMu.RLock() - defer p.writeSurfaceMu.RUnlock() - return p.writeSurfaceEnforced + enforced := p.writeSurfaceEnforced + p.writeSurfaceMu.RUnlock() + if enforced == nil { + return nil + } + return func(agent string) bool { + v, ok := p.callPolicyPredicate("write_surface.enforce", func() bool { return enforced(agent) }) + if !ok { + return true + } + return v + } } // laneWriteSurfaceEnforced reports whether agentName's lane is enforced. diff --git a/src/crux/proxy/policy_callback_guard.go b/src/crux/proxy/policy_callback_guard.go new file mode 100644 index 0000000000..4f0cde5b8e --- /dev/null +++ b/src/crux/proxy/policy_callback_guard.go @@ -0,0 +1,107 @@ +package proxy + +import ( + "fmt" + "time" +) + +// The per-request write gates (repo pause, agent repo scope, repo auto-merge +// policy, write_surface.enforce) each consult a live predicate the hive +// installs through a Set*Func setter. Those predicates are host code reading +// live config, and the proxy used to call them bare on the request goroutine. +// That left two failure branches on the decision path that did not deny: +// +// - A panic. No goroutine above the MITM request loop recovers, so a panic +// in a predicate took down the whole proxy process (every agent's egress) +// instead of refusing the one request it was evaluating. +// - A hang. A predicate blocked on a lock or a slow store held the agent's +// request open indefinitely with no answer at all. +// +// The predicates return a plain bool, so there is no error value to +// mishandle; a panic or a timeout is how such a callback fails. Every gate +// now calls its predicate through callPolicyPredicate and treats either +// failure as the deny answer for that gate: paused, not scoped, auto-merge +// off, lane enforced. Fail closed, same direction as the dial-time +// classifier in plain_forward_guard.go refusing an address it cannot parse. +// +// Intentional exceptions, not routed through this guard: +// +// - githubBudgetCap / githubReserveFloor / githubReserveSnapshot are rate +// limits, not authorization; they default to config values when unset +// and a wrong answer costs quota, not a policy bypass. +// - agentTokenSource / linearCredential supply credentials rather than +// decide; a missing credential already makes the upstream refuse. +// - writeRefusedAudit and canaryLeakFunc are sinks called after a decision +// has been made; they cannot turn a deny into an allow. + +// policyCallbackTimeout bounds one policy predicate call. The predicates read +// in-memory config, so anything near this is already a fault. A var only so +// the fault-injection test can shorten it. +var policyCallbackTimeout = 2 * time.Second + +// policyCallbackFailedReason is the agent-facing 403 body when a gate could +// not be evaluated. It deliberately does not reuse the gate's own directive +// ("repository X is PAUSED by the operator"), which would assert an operator +// decision nobody made. +func policyCallbackFailedReason(gate string) string { + return "the proxy could not evaluate the " + gate + " policy for this request (the policy check failed or timed out), so the write is refused: policy checks fail closed. Retry shortly; if it keeps happening, report it to the operator as a hive fault." +} + +// callPolicyPredicate runs fn with panic recovery and policyCallbackTimeout. +// ok is false when fn panicked or did not return in time; the caller must +// then take its deny branch. On timeout the goroutine running fn is left to +// finish on its own (Go cannot cancel it); the buffered channel lets it exit +// without a reader. +func (p *GitHubProxy) callPolicyPredicate(gate string, fn func() bool) (result, ok bool) { + type outcome struct { + value bool + panicked bool + cause any + } + done := make(chan outcome, 1) + go func() { + defer func() { + if r := recover(); r != nil { + done <- outcome{panicked: true, cause: r} + } + }() + done <- outcome{value: fn()} + }() + timer := time.NewTimer(policyCallbackTimeout) + defer timer.Stop() + select { + case o := <-done: + if o.panicked { + p.logPolicyCallbackFailure(gate, fmt.Sprintf("panic: %v", o.cause)) + return false, false + } + return o.value, true + case <-timer.C: + p.logPolicyCallbackFailure(gate, "timed out after "+policyCallbackTimeout.String()) + return false, false + } +} + +func (p *GitHubProxy) logPolicyCallbackFailure(gate, cause string) { + if p == nil || p.logger == nil { + return + } + p.logger.Error("proxy: policy callback failed; denying the request", "gate", gate, "cause", cause) +} + +// guardRepoPredicate wraps a func(repo) bool gate predicate so a failed call +// answers onFail (the gate's deny value) and sets *failed. A nil fn stays nil +// so the Refusal functions keep their "no predicate installed" semantics. +func (p *GitHubProxy) guardRepoPredicate(gate string, fn func(string) bool, onFail bool, failed *bool) func(string) bool { + if fn == nil { + return nil + } + return func(repo string) bool { + v, ok := p.callPolicyPredicate(gate, func() bool { return fn(repo) }) + if !ok { + *failed = true + return onFail + } + return v + } +} diff --git a/src/crux/proxy/policy_callback_guard_test.go b/src/crux/proxy/policy_callback_guard_test.go new file mode 100644 index 0000000000..659c56535b --- /dev/null +++ b/src/crux/proxy/policy_callback_guard_test.go @@ -0,0 +1,90 @@ +package proxy + +import ( + "strings" + "testing" + "time" +) + +// TestPolicyCallbackFailureDenies injects faults into every live policy +// predicate on the write-gate decision path and asserts the gate refuses. +// Before the guard, the panic rows crashed the test binary (no recover on the +// request path) and the hang rows blocked until the test timeout. +func TestPolicyCallbackFailureDenies(t *testing.T) { + prev := policyCallbackTimeout + policyCallbackTimeout = 50 * time.Millisecond + release := make(chan struct{}) + t.Cleanup(func() { + close(release) + policyCallbackTimeout = prev + }) + + type fault struct { + name string + hit func() + } + faults := []fault{ + {"panic", func() { panic("injected policy fault") }}, + {"timeout", func() { <-release }}, + } + + const mergeGraphQL = `{"query":"mutation { mergePullRequest(input:{pullRequestId:\"PR_kw\"}) { clientMutationId } }","variables":{"repo":"acme/app"}}` + + gates := []struct { + name string + check func(p *GitHubProxy, hit func()) (string, bool) + }{ + {"repo pause", func(p *GitHubProxy, hit func()) (string, bool) { + p.SetRepoPausedFunc(func(string) bool { hit(); return false }) + return p.repoPauseRefusal("POST", "/repos/acme/app/issues") + }}, + {"agent repo scope", func(p *GitHubProxy, hit func()) (string, bool) { + p.SetAgentRepoScopeFunc(func(string, string) bool { hit(); return true }) + return p.agentRepoScopeRefusal("scanner", "POST", "/repos/acme/app/issues") + }}, + {"repo auto-merge REST", func(p *GitHubProxy, hit func()) (string, bool) { + p.SetRepoAutoMergeEnabledFunc(func(string) bool { hit(); return true }) + return p.autoMergeRefusal("PUT", "/repos/acme/app/pulls/7/merge") + }}, + {"repo auto-merge GraphQL", func(p *GitHubProxy, hit func()) (string, bool) { + p.SetRepoAutoMergeEnabledFunc(func(string) bool { hit(); return true }) + return p.graphQLAutoMergeRefusal([]byte(mergeGraphQL)) + }}, + {"write_surface.enforce", func(p *GitHubProxy, hit func()) (string, bool) { + p.SetWriteSurfaceEnforceFunc(func(string) bool { hit(); return false }) + reason, _, refused := WriteSurfaceEnforceRefusal(p.writeSurfaceEnforcedFunc(), "scanner", "POST", "/repos/acme/app/issues", "") + return reason, refused + }}, + {"write_surface.enforce MITM", func(p *GitHubProxy, hit func()) (string, bool) { + p.SetWriteSurfaceEnforceFunc(func(string) bool { hit(); return false }) + return "", p.laneWriteSurfaceEnforced("scanner") + }}, + } + + for _, g := range gates { + for _, f := range faults { + t.Run(g.name+"/"+f.name, func(t *testing.T) { + p := &GitHubProxy{} + reason, denied := g.check(p, f.hit) + if !denied { + t.Fatalf("%s callback %s: gate allowed the request; want deny", g.name, f.name) + } + if strings.Contains(reason, "PAUSED by the operator") || strings.Contains(reason, "is NOT scoped") { + t.Fatalf("failure reason asserts an operator decision nobody made: %q", reason) + } + }) + } + } + + // A healthy predicate still decides: the guard must not turn every + // answer into a deny. + p := &GitHubProxy{} + p.SetRepoPausedFunc(func(string) bool { return false }) + if reason, denied := p.repoPauseRefusal("POST", "/repos/acme/app/issues"); denied { + t.Fatalf("healthy pause predicate refused: %q", reason) + } + p.SetAgentRepoScopeFunc(func(string, string) bool { return true }) + if reason, denied := p.agentRepoScopeRefusal("scanner", "POST", "/repos/acme/app/issues"); denied { + t.Fatalf("healthy scope predicate refused: %q", reason) + } +} From e36ab6337b320c615a2bf826c6a06cedf3ac940a Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:04:19 -0500 Subject: [PATCH 2/2] fix: write_surface.enforce callback failure returns policy-failed reason Per maintainer decision on #19005: when the write_surface.enforce predicate panics or times out, the REST and GraphQL arms now return policyCallbackFailedReason("write_surface.enforce") via a failure flag, like the other gates, instead of the normal write_surface.enforce directive. The test asserts each gate's reason equals policyCallbackFailedReason(gate). Hive-Fix-Attempt: 1/6 Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/crux/proxy/github_proxy_mitm.go | 4 +-- src/crux/proxy/github_proxy_policy.go | 35 ++++++++++++++++++-- src/crux/proxy/policy_callback_guard_test.go | 22 ++++++++---- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/src/crux/proxy/github_proxy_mitm.go b/src/crux/proxy/github_proxy_mitm.go index 362263b0e2..37ed4cabbb 100644 --- a/src/crux/proxy/github_proxy_mitm.go +++ b/src/crux/proxy/github_proxy_mitm.go @@ -114,7 +114,7 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str blockCode = DenyReasonRepoAutoMergeOff blockJSON = true } else if !isLinear { - if reason, kind, refused := WriteSurfaceEnforceRefusal(p.writeSurfaceEnforcedFunc(), agentName, req.Method, req.URL.Path, req.URL.RawQuery); refused { + if reason, kind, refused := p.writeSurfaceEnforceRefusal(agentName, req.Method, req.URL.Path, req.URL.RawQuery); refused { blocked = true blockReason = reason blockCode = DenyReasonWriteSurfaceEnforced @@ -161,7 +161,7 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str blockCode = DenyReasonRepoAutoMergeOff blockJSON = true } else if agentName != internalCallerName { - if reason, refused := WriteSurfaceGraphQLRefusal(p.writeSurfaceEnforcedFunc(), agentName, body); refused { + if reason, refused := p.writeSurfaceGraphQLRefusal(agentName, body); refused { blocked = true blockReason = reason blockCode = DenyReasonWriteSurfaceEnforced diff --git a/src/crux/proxy/github_proxy_policy.go b/src/crux/proxy/github_proxy_policy.go index 9e578fbaae..ecbe86cd2e 100644 --- a/src/crux/proxy/github_proxy_policy.go +++ b/src/crux/proxy/github_proxy_policy.go @@ -139,8 +139,10 @@ func (p *GitHubProxy) agentRepoScopeRefusal(agentName, method, path string) (str // writeSurfaceEnforcedFunc returns the live write_surface.enforce predicate // wrapped so a call that panics or times out reports the lane as enforced: // its direct writes are refused and its GitHub hosts are intercepted, the -// deny direction for both callers. -func (p *GitHubProxy) writeSurfaceEnforcedFunc() func(agent string) bool { +// deny direction for both callers. failed, when non-nil, is set on a failed +// call so the caller can answer with policyCallbackFailedReason instead of +// the normal write_surface.enforce directive. +func (p *GitHubProxy) writeSurfaceEnforcedFunc(failed *bool) func(agent string) bool { p.writeSurfaceMu.RLock() enforced := p.writeSurfaceEnforced p.writeSurfaceMu.RUnlock() @@ -150,18 +152,45 @@ func (p *GitHubProxy) writeSurfaceEnforcedFunc() func(agent string) bool { return func(agent string) bool { v, ok := p.callPolicyPredicate("write_surface.enforce", func() bool { return enforced(agent) }) if !ok { + if failed != nil { + *failed = true + } return true } return v } } +// writeSurfaceEnforceRefusal is WriteSurfaceEnforceRefusal against the current +// predicate. A failed predicate refuses with policyCallbackFailedReason, not +// the write_surface.enforce directive, which would assert a lane setting the +// proxy could not read. +func (p *GitHubProxy) writeSurfaceEnforceRefusal(agent, method, path, rawQuery string) (reason, kind string, refused bool) { + failed := false + reason, kind, refused = WriteSurfaceEnforceRefusal(p.writeSurfaceEnforcedFunc(&failed), agent, method, path, rawQuery) + if failed { + return policyCallbackFailedReason("write_surface.enforce"), kind, true + } + return reason, kind, refused +} + +// writeSurfaceGraphQLRefusal is WriteSurfaceGraphQLRefusal against the current +// predicate, with the same failure reason as writeSurfaceEnforceRefusal. +func (p *GitHubProxy) writeSurfaceGraphQLRefusal(agent string, body []byte) (string, bool) { + failed := false + reason, refused := WriteSurfaceGraphQLRefusal(p.writeSurfaceEnforcedFunc(&failed), agent, body) + if failed { + return policyCallbackFailedReason("write_surface.enforce"), true + } + return reason, refused +} + // laneWriteSurfaceEnforced reports whether agentName's lane is enforced. func (p *GitHubProxy) laneWriteSurfaceEnforced(agentName string) bool { if agentName == "" || agentName == internalCallerName { return false } - enforced := p.writeSurfaceEnforcedFunc() + enforced := p.writeSurfaceEnforcedFunc(nil) return enforced != nil && enforced(agentName) } diff --git a/src/crux/proxy/policy_callback_guard_test.go b/src/crux/proxy/policy_callback_guard_test.go index 659c56535b..6282996074 100644 --- a/src/crux/proxy/policy_callback_guard_test.go +++ b/src/crux/proxy/policy_callback_guard_test.go @@ -32,30 +32,35 @@ func TestPolicyCallbackFailureDenies(t *testing.T) { gates := []struct { name string + gate string // policy name in policyCallbackFailedReason; "" = bool-only check, no reason check func(p *GitHubProxy, hit func()) (string, bool) }{ - {"repo pause", func(p *GitHubProxy, hit func()) (string, bool) { + {"repo pause", "repo pause", func(p *GitHubProxy, hit func()) (string, bool) { p.SetRepoPausedFunc(func(string) bool { hit(); return false }) return p.repoPauseRefusal("POST", "/repos/acme/app/issues") }}, - {"agent repo scope", func(p *GitHubProxy, hit func()) (string, bool) { + {"agent repo scope", "agent repo scope", func(p *GitHubProxy, hit func()) (string, bool) { p.SetAgentRepoScopeFunc(func(string, string) bool { hit(); return true }) return p.agentRepoScopeRefusal("scanner", "POST", "/repos/acme/app/issues") }}, - {"repo auto-merge REST", func(p *GitHubProxy, hit func()) (string, bool) { + {"repo auto-merge REST", "repo auto-merge", func(p *GitHubProxy, hit func()) (string, bool) { p.SetRepoAutoMergeEnabledFunc(func(string) bool { hit(); return true }) return p.autoMergeRefusal("PUT", "/repos/acme/app/pulls/7/merge") }}, - {"repo auto-merge GraphQL", func(p *GitHubProxy, hit func()) (string, bool) { + {"repo auto-merge GraphQL", "repo auto-merge", func(p *GitHubProxy, hit func()) (string, bool) { p.SetRepoAutoMergeEnabledFunc(func(string) bool { hit(); return true }) return p.graphQLAutoMergeRefusal([]byte(mergeGraphQL)) }}, - {"write_surface.enforce", func(p *GitHubProxy, hit func()) (string, bool) { + {"write_surface.enforce", "write_surface.enforce", func(p *GitHubProxy, hit func()) (string, bool) { p.SetWriteSurfaceEnforceFunc(func(string) bool { hit(); return false }) - reason, _, refused := WriteSurfaceEnforceRefusal(p.writeSurfaceEnforcedFunc(), "scanner", "POST", "/repos/acme/app/issues", "") + reason, _, refused := p.writeSurfaceEnforceRefusal("scanner", "POST", "/repos/acme/app/issues", "") return reason, refused }}, - {"write_surface.enforce MITM", func(p *GitHubProxy, hit func()) (string, bool) { + {"write_surface.enforce GraphQL", "write_surface.enforce", func(p *GitHubProxy, hit func()) (string, bool) { + p.SetWriteSurfaceEnforceFunc(func(string) bool { hit(); return false }) + return p.writeSurfaceGraphQLRefusal("scanner", []byte(mergeGraphQL)) + }}, + {"write_surface.enforce MITM", "", func(p *GitHubProxy, hit func()) (string, bool) { p.SetWriteSurfaceEnforceFunc(func(string) bool { hit(); return false }) return "", p.laneWriteSurfaceEnforced("scanner") }}, @@ -69,6 +74,9 @@ func TestPolicyCallbackFailureDenies(t *testing.T) { if !denied { t.Fatalf("%s callback %s: gate allowed the request; want deny", g.name, f.name) } + if g.gate != "" && reason != policyCallbackFailedReason(g.gate) { + t.Fatalf("%s callback %s: reason = %q; want policyCallbackFailedReason(%q)", g.name, f.name, reason, g.gate) + } if strings.Contains(reason, "PAUSED by the operator") || strings.Contains(reason, "is NOT scoped") { t.Fatalf("failure reason asserts an operator decision nobody made: %q", reason) }