Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-proxy-policy-callback-fail-closed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **Proxy write gates fail closed when a policy check fails.** If the repo pause, agent repo scope, auto-merge or write_surface.enforce check panics or hangs, the proxy now refuses that one request with a 403 naming the failed check, instead of crashing the whole proxy or holding the request open.
4 changes: 2 additions & 2 deletions src/crux/proxy/github_proxy_mitm.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str
blockCode = DenyReasonRepoAutoMergeOff
blockJSON = true
} else if !isLinear {
if reason, kind, refused := WriteSurfaceEnforceRefusal(p.writeSurfaceEnforcedFunc(), agentName, req.Method, req.URL.Path, req.URL.RawQuery); refused {
if reason, kind, refused := p.writeSurfaceEnforceRefusal(agentName, req.Method, req.URL.Path, req.URL.RawQuery); refused {
blocked = true
blockReason = reason
blockCode = DenyReasonWriteSurfaceEnforced
Expand Down Expand Up @@ -161,7 +161,7 @@ func (p *GitHubProxy) proxyHTTPHost(client net.Conn, upstream net.Conn, host str
blockCode = DenyReasonRepoAutoMergeOff
blockJSON = true
} else if agentName != internalCallerName {
if reason, refused := WriteSurfaceGraphQLRefusal(p.writeSurfaceEnforcedFunc(), agentName, body); refused {
if reason, refused := p.writeSurfaceGraphQLRefusal(agentName, body); refused {
blocked = true
blockReason = reason
blockCode = DenyReasonWriteSurfaceEnforced
Expand Down
95 changes: 87 additions & 8 deletions src/crux/proxy/github_proxy_policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,47 +71,126 @@ func repoScopePath(req *http.Request) string {
}

// repoPauseRefusal is RepoPauseRefusal against this proxy's current predicate.
// A predicate that panics or times out counts as paused (see
// policy_callback_guard.go).
func (p *GitHubProxy) repoPauseRefusal(method, path string) (string, bool) {
p.pausedMu.RLock()
paused := p.repoPaused
p.pausedMu.RUnlock()
return RepoPauseRefusal(paused, method, path)
failed := false
reason, refused := RepoPauseRefusal(p.guardRepoPredicate("repo pause", paused, true, &failed), method, path)
if failed {
return policyCallbackFailedReason("repo pause"), true
}
return reason, refused
}

// autoMergeRefusal is RepoAutoMergeRefusal against the current predicate. A

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Advisory: the failed-flag + guardRepoPredicate + policyCallbackFailedReason sequence is repeated in three gates here, and agentRepoScopeRefusal hand-rolls its own closure. A small shared helper would reduce drift risk for future gates. Not blocking.

// predicate that panics or times out counts as auto-merge off.
func (p *GitHubProxy) autoMergeRefusal(method, path string) (string, bool) {
p.autoMergeMu.RLock()
enabled := p.repoAutoMergeOn
p.autoMergeMu.RUnlock()
return RepoAutoMergeRefusal(enabled, method, path)
failed := false
reason, refused := RepoAutoMergeRefusal(p.guardRepoPredicate("repo auto-merge", enabled, false, &failed), method, path)
if failed {
return policyCallbackFailedReason("repo auto-merge"), true
}
return reason, refused
}

func (p *GitHubProxy) graphQLAutoMergeRefusal(body []byte) (string, bool) {
p.autoMergeMu.RLock()
enabled := p.repoAutoMergeOn
p.autoMergeMu.RUnlock()
return GraphQLAutoMergeRefusal(enabled, body)
failed := false
reason, refused := GraphQLAutoMergeRefusal(p.guardRepoPredicate("repo auto-merge", enabled, false, &failed), body)
if failed {
return policyCallbackFailedReason("repo auto-merge"), true
}
return reason, refused
}

// agentRepoScopeRefusal is AgentRepoScopeRefusal against the current predicate.
// A predicate that panics or times out counts as not scoped.
func (p *GitHubProxy) agentRepoScopeRefusal(agentName, method, path string) (string, bool) {
p.scopeMu.RLock()
serves := p.agentServesRepo
p.scopeMu.RUnlock()
return AgentRepoScopeRefusal(serves, agentName, method, path)
if serves == nil {
return AgentRepoScopeRefusal(nil, agentName, method, path)
}
failed := false
guarded := func(agent, repo string) bool {
v, ok := p.callPolicyPredicate("agent repo scope", func() bool { return serves(agent, repo) })
if !ok {
failed = true
return false
}
return v
}
reason, refused := AgentRepoScopeRefusal(guarded, agentName, method, path)
if failed {
return policyCallbackFailedReason("agent repo scope"), true
}
return reason, refused
}

func (p *GitHubProxy) writeSurfaceEnforcedFunc() func(agent string) bool {
// writeSurfaceEnforcedFunc returns the live write_surface.enforce predicate
// wrapped so a call that panics or times out reports the lane as enforced:
// its direct writes are refused and its GitHub hosts are intercepted, the
// deny direction for both callers. failed, when non-nil, is set on a failed
// call so the caller can answer with policyCallbackFailedReason instead of
// the normal write_surface.enforce directive.
func (p *GitHubProxy) writeSurfaceEnforcedFunc(failed *bool) func(agent string) bool {
p.writeSurfaceMu.RLock()
defer p.writeSurfaceMu.RUnlock()
return p.writeSurfaceEnforced
enforced := p.writeSurfaceEnforced
p.writeSurfaceMu.RUnlock()
if enforced == nil {
return nil
}
return func(agent string) bool {
v, ok := p.callPolicyPredicate("write_surface.enforce", func() bool { return enforced(agent) })
if !ok {
if failed != nil {
*failed = true
}
return true
}
return v
}
}

// writeSurfaceEnforceRefusal is WriteSurfaceEnforceRefusal against the current
// predicate. A failed predicate refuses with policyCallbackFailedReason, not
// the write_surface.enforce directive, which would assert a lane setting the
// proxy could not read.
func (p *GitHubProxy) writeSurfaceEnforceRefusal(agent, method, path, rawQuery string) (reason, kind string, refused bool) {
failed := false
reason, kind, refused = WriteSurfaceEnforceRefusal(p.writeSurfaceEnforcedFunc(&failed), agent, method, path, rawQuery)
if failed {
return policyCallbackFailedReason("write_surface.enforce"), kind, true
}
return reason, kind, refused
}

// writeSurfaceGraphQLRefusal is WriteSurfaceGraphQLRefusal against the current
// predicate, with the same failure reason as writeSurfaceEnforceRefusal.
func (p *GitHubProxy) writeSurfaceGraphQLRefusal(agent string, body []byte) (string, bool) {
failed := false
reason, refused := WriteSurfaceGraphQLRefusal(p.writeSurfaceEnforcedFunc(&failed), agent, body)
if failed {
return policyCallbackFailedReason("write_surface.enforce"), true
}
return reason, refused
}

// laneWriteSurfaceEnforced reports whether agentName's lane is enforced.
func (p *GitHubProxy) laneWriteSurfaceEnforced(agentName string) bool {
if agentName == "" || agentName == internalCallerName {
return false
}
enforced := p.writeSurfaceEnforcedFunc()
enforced := p.writeSurfaceEnforcedFunc(nil)
return enforced != nil && enforced(agentName)
}

Expand Down
107 changes: 107 additions & 0 deletions src/crux/proxy/policy_callback_guard.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package proxy

import (
"fmt"
"time"
)

// The per-request write gates (repo pause, agent repo scope, repo auto-merge
// policy, write_surface.enforce) each consult a live predicate the hive
// installs through a Set*Func setter. Those predicates are host code reading
// live config, and the proxy used to call them bare on the request goroutine.
// That left two failure branches on the decision path that did not deny:
//
// - A panic. No goroutine above the MITM request loop recovers, so a panic
// in a predicate took down the whole proxy process (every agent's egress)
// instead of refusing the one request it was evaluating.
// - A hang. A predicate blocked on a lock or a slow store held the agent's
// request open indefinitely with no answer at all.
//
// The predicates return a plain bool, so there is no error value to
// mishandle; a panic or a timeout is how such a callback fails. Every gate
// now calls its predicate through callPolicyPredicate and treats either
// failure as the deny answer for that gate: paused, not scoped, auto-merge
// off, lane enforced. Fail closed, same direction as the dial-time
// classifier in plain_forward_guard.go refusing an address it cannot parse.
//
// Intentional exceptions, not routed through this guard:
//
// - githubBudgetCap / githubReserveFloor / githubReserveSnapshot are rate
// limits, not authorization; they default to config values when unset
// and a wrong answer costs quota, not a policy bypass.
// - agentTokenSource / linearCredential supply credentials rather than
// decide; a missing credential already makes the upstream refuse.
// - writeRefusedAudit and canaryLeakFunc are sinks called after a decision
// has been made; they cannot turn a deny into an allow.

// policyCallbackTimeout bounds one policy predicate call. The predicates read
// in-memory config, so anything near this is already a fault. A var only so
// the fault-injection test can shorten it.
var policyCallbackTimeout = 2 * time.Second

// policyCallbackFailedReason is the agent-facing 403 body when a gate could
// not be evaluated. It deliberately does not reuse the gate's own directive
// ("repository X is PAUSED by the operator"), which would assert an operator
// decision nobody made.
func policyCallbackFailedReason(gate string) string {
return "the proxy could not evaluate the " + gate + " policy for this request (the policy check failed or timed out), so the write is refused: policy checks fail closed. Retry shortly; if it keeps happening, report it to the operator as a hive fault."
}

// callPolicyPredicate runs fn with panic recovery and policyCallbackTimeout.
// ok is false when fn panicked or did not return in time; the caller must
// then take its deny branch. On timeout the goroutine running fn is left to
// finish on its own (Go cannot cancel it); the buffered channel lets it exit
// without a reader.
func (p *GitHubProxy) callPolicyPredicate(gate string, fn func() bool) (result, ok bool) {
type outcome struct {
value bool
panicked bool
cause any
}
done := make(chan outcome, 1)
go func() {
defer func() {
if r := recover(); r != nil {
done <- outcome{panicked: true, cause: r}
}
}()
done <- outcome{value: fn()}
}()
timer := time.NewTimer(policyCallbackTimeout)
defer timer.Stop()
select {
case o := <-done:
if o.panicked {
p.logPolicyCallbackFailure(gate, fmt.Sprintf("panic: %v", o.cause))
return false, false
}
return o.value, true
case <-timer.C:
p.logPolicyCallbackFailure(gate, "timed out after "+policyCallbackTimeout.String())
return false, false
}
}

func (p *GitHubProxy) logPolicyCallbackFailure(gate, cause string) {
if p == nil || p.logger == nil {
return
}
p.logger.Error("proxy: policy callback failed; denying the request", "gate", gate, "cause", cause)
}

// guardRepoPredicate wraps a func(repo) bool gate predicate so a failed call
// answers onFail (the gate's deny value) and sets *failed. A nil fn stays nil
// so the Refusal functions keep their "no predicate installed" semantics.
func (p *GitHubProxy) guardRepoPredicate(gate string, fn func(string) bool, onFail bool, failed *bool) func(string) bool {
if fn == nil {
return nil
}
return func(repo string) bool {
v, ok := p.callPolicyPredicate(gate, func() bool { return fn(repo) })
if !ok {
*failed = true
return onFail
}
return v
}
}
98 changes: 98 additions & 0 deletions src/crux/proxy/policy_callback_guard_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package proxy

import (
"strings"
"testing"
"time"
)

// TestPolicyCallbackFailureDenies injects faults into every live policy
// predicate on the write-gate decision path and asserts the gate refuses.
// Before the guard, the panic rows crashed the test binary (no recover on the
// request path) and the hang rows blocked until the test timeout.
func TestPolicyCallbackFailureDenies(t *testing.T) {
prev := policyCallbackTimeout
policyCallbackTimeout = 50 * time.Millisecond
release := make(chan struct{})
t.Cleanup(func() {
close(release)
policyCallbackTimeout = prev
})

type fault struct {
name string
hit func()
}
faults := []fault{
{"panic", func() { panic("injected policy fault") }},
{"timeout", func() { <-release }},
}

const mergeGraphQL = `{"query":"mutation { mergePullRequest(input:{pullRequestId:\"PR_kw\"}) { clientMutationId } }","variables":{"repo":"acme/app"}}`

gates := []struct {
name string
gate string // policy name in policyCallbackFailedReason; "" = bool-only check, no reason
check func(p *GitHubProxy, hit func()) (string, bool)
}{
{"repo pause", "repo pause", func(p *GitHubProxy, hit func()) (string, bool) {
p.SetRepoPausedFunc(func(string) bool { hit(); return false })
return p.repoPauseRefusal("POST", "/repos/acme/app/issues")
}},
{"agent repo scope", "agent repo scope", func(p *GitHubProxy, hit func()) (string, bool) {
p.SetAgentRepoScopeFunc(func(string, string) bool { hit(); return true })
return p.agentRepoScopeRefusal("scanner", "POST", "/repos/acme/app/issues")
}},
{"repo auto-merge REST", "repo auto-merge", func(p *GitHubProxy, hit func()) (string, bool) {
p.SetRepoAutoMergeEnabledFunc(func(string) bool { hit(); return true })
return p.autoMergeRefusal("PUT", "/repos/acme/app/pulls/7/merge")
}},
{"repo auto-merge GraphQL", "repo auto-merge", func(p *GitHubProxy, hit func()) (string, bool) {
p.SetRepoAutoMergeEnabledFunc(func(string) bool { hit(); return true })
return p.graphQLAutoMergeRefusal([]byte(mergeGraphQL))
}},
{"write_surface.enforce", "write_surface.enforce", func(p *GitHubProxy, hit func()) (string, bool) {
p.SetWriteSurfaceEnforceFunc(func(string) bool { hit(); return false })
reason, _, refused := p.writeSurfaceEnforceRefusal("scanner", "POST", "/repos/acme/app/issues", "")
return reason, refused
}},
{"write_surface.enforce GraphQL", "write_surface.enforce", func(p *GitHubProxy, hit func()) (string, bool) {
p.SetWriteSurfaceEnforceFunc(func(string) bool { hit(); return false })
return p.writeSurfaceGraphQLRefusal("scanner", []byte(mergeGraphQL))
}},
{"write_surface.enforce MITM", "", func(p *GitHubProxy, hit func()) (string, bool) {
p.SetWriteSurfaceEnforceFunc(func(string) bool { hit(); return false })
return "", p.laneWriteSurfaceEnforced("scanner")
}},
}

for _, g := range gates {
for _, f := range faults {
t.Run(g.name+"/"+f.name, func(t *testing.T) {
p := &GitHubProxy{}
reason, denied := g.check(p, f.hit)
if !denied {
t.Fatalf("%s callback %s: gate allowed the request; want deny", g.name, f.name)
}
if g.gate != "" && reason != policyCallbackFailedReason(g.gate) {
t.Fatalf("%s callback %s: reason = %q; want policyCallbackFailedReason(%q)", g.name, f.name, reason, g.gate)
}
if strings.Contains(reason, "PAUSED by the operator") || strings.Contains(reason, "is NOT scoped") {
t.Fatalf("failure reason asserts an operator decision nobody made: %q", reason)
}
})
}
}

// A healthy predicate still decides: the guard must not turn every
// answer into a deny.
p := &GitHubProxy{}
p.SetRepoPausedFunc(func(string) bool { return false })
if reason, denied := p.repoPauseRefusal("POST", "/repos/acme/app/issues"); denied {
t.Fatalf("healthy pause predicate refused: %q", reason)
}
p.SetAgentRepoScopeFunc(func(string, string) bool { return true })
if reason, denied := p.agentRepoScopeRefusal("scanner", "POST", "/repos/acme/app/issues"); denied {
t.Fatalf("healthy scope predicate refused: %q", reason)
}
}
Loading