Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-19224-proxy-seam-fail-closed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **More proxy checks fail closed when they cannot run.** If the review-thread ownership check, the GitHub lookup behind the reference-repo and review-thread gates, or the canary scan panics or hangs, the proxy now refuses that one request with a 403 instead of crashing the whole proxy or holding the request open. A canary scan that cannot finish refuses the write even in advisory canary mode. The results of the egress audit's error-path review, and the accepted risks, are recorded in the security model ([#19224](https://github.com/hivecommons/hive/issues/19224)).
18 changes: 17 additions & 1 deletion src/crux/proxy/github_proxy_policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"net/http"
"strings"

"github.com/hivecommons/hive/crux"
"github.com/hivecommons/hive/crux/agent"
)

Expand Down Expand Up @@ -370,7 +371,22 @@ func (p *GitHubProxy) inspectCanaryEgress(agentName string, req *http.Request) (
}
req.Body = io.NopCloser(bytes.NewReader(body))
req.ContentLength = int64(len(body))
if leak, ok := p.canaryRegistry.Scan(agentName, string(body), "github:"+req.Method+" "+req.URL.Path); ok {
type scanResult struct {
leak crux.CanaryLeak
found bool
}
scanner, text, source := p.canaryRegistry, string(body), "github:"+req.Method+" "+req.URL.Path
res, scanned := callPolicySeam(p, "ioscan canary scan", canaryScanTimeout, func() scanResult {
leak, found := scanner.Scan(agentName, text, source)
return scanResult{leak, found}
})
if !scanned {
// Fail closed (#19224), whatever canaryFailClosed says: a scan that
// panicked or did not finish is a body we could not scan, the same
// answer as a body we could not read.
return "ioscan canary scan failed", true, true
}
if leak, ok := res.leak, res.found; ok {
if p.canaryLeakFunc != nil {
p.canaryLeakFunc(leak)
}
Expand Down
53 changes: 44 additions & 9 deletions src/crux/proxy/policy_callback_guard.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,21 +24,48 @@ import (
// off, lane enforced. Fail closed, same direction as the dial-time
// classifier in plain_forward_guard.go refusing an address it cannot parse.
//
// The same guard (callPolicySeam) covers the seams on the decision path that
// are not Set*Func predicates (#19224):
//
// - ReviewThreadGuard.OwnPR and HiveLogins (review-thread gate): a failure
// refuses the reply or delete.
// - referenceLookup, the GitHub GET behind the reference-repo own-item and
// default-branch checks and the review-thread comment lookup: it already
// refused on an error; a panic or a call past referenceLookupTimeout is
// now reported as that error.
// - CanaryScanner.Scan: a scan that panics or does not finish within
// canaryScanTimeout refuses the write, the same answer as a body the scan
// could not read.
//
// Intentional exceptions, not routed through this guard:
//
// - githubBudgetCap / githubReserveFloor / githubReserveSnapshot are rate
// limits, not authorization; they default to config values when unset
// and a wrong answer costs quota, not a policy bypass.
// - agentTokenSource / linearCredential supply credentials rather than
// decide; a missing credential already makes the upstream refuse.
// - writeRefusedAudit and canaryLeakFunc are sinks called after a decision
// - ReferenceRepoPolicy.Token supplies a credential; "" sends the lookup
// unauthenticated and GitHub then answers 404 for a private repo, which
// refuses.
// - writeRefusedAudit, canaryLeakFunc and ReviewThreadGuard.OnRefused are sinks called after a decision
// has been made; they cannot turn a deny into an allow.

// policyCallbackTimeout bounds one policy predicate call. The predicates read
// in-memory config, so anything near this is already a fault. A var only so
// the fault-injection test can shorten it.
var policyCallbackTimeout = 2 * time.Second

// canaryScanTimeout bounds one canary scan of a write body. The scan is
// in-process but works over up to maxGitHubWriteBodyScan bytes and their
// decoded variants, so it gets more room than a config predicate. A var only
// so the fault-injection test can shorten it.
var canaryScanTimeout = 10 * time.Second

// referenceLookupTimeout bounds one referenceLookup call. The default lookup
// carries its own 10s client timeout; this is the backstop for a lookup that
// ignores it. A var only so the fault-injection test can shorten it.
var referenceLookupTimeout = 15 * time.Second

// policyCallbackFailedReason is the agent-facing 403 body when a gate could
// not be evaluated. It deliberately does not reuse the gate's own directive
// ("repository X is PAUSED by the operator"), which would assert an operator
Expand All @@ -49,12 +76,19 @@ func policyCallbackFailedReason(gate string) string {

// callPolicyPredicate runs fn with panic recovery and policyCallbackTimeout.
// ok is false when fn panicked or did not return in time; the caller must
// then take its deny branch. On timeout the goroutine running fn is left to
// finish on its own (Go cannot cancel it); the buffered channel lets it exit
// without a reader.
// then take its deny branch.
func (p *GitHubProxy) callPolicyPredicate(gate string, fn func() bool) (result, ok bool) {
return callPolicySeam(p, gate, policyCallbackTimeout, fn)
}

// callPolicySeam is callPolicyPredicate for a seam of any result type and its
// own deadline (#19224). ok is false when fn panicked or did not return within
// timeout; the zero T is returned and the caller must take its deny branch.
// On timeout the goroutine running fn is left to finish on its own (Go cannot
// cancel it); the buffered channel lets it exit without a reader.
func callPolicySeam[T any](p *GitHubProxy, gate string, timeout time.Duration, fn func() T) (result T, ok bool) {
type outcome struct {
value bool
value T
panicked bool
cause any
}
Expand All @@ -67,18 +101,19 @@ func (p *GitHubProxy) callPolicyPredicate(gate string, fn func() bool) (result,
}()
done <- outcome{value: fn()}
}()
timer := time.NewTimer(policyCallbackTimeout)
timer := time.NewTimer(timeout)
defer timer.Stop()
var zero T
select {
case o := <-done:
if o.panicked {
p.logPolicyCallbackFailure(gate, fmt.Sprintf("panic: %v", o.cause))
return false, false
return zero, false
}
return o.value, true
case <-timer.C:
p.logPolicyCallbackFailure(gate, "timed out after "+policyCallbackTimeout.String())
return false, false
p.logPolicyCallbackFailure(gate, "timed out after "+timeout.String())
return zero, false
}
}

Expand Down
188 changes: 188 additions & 0 deletions src/crux/proxy/policy_seam_fault_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
package proxy

import (
"errors"
"io"
"log/slog"
"net/http"
"strings"
"testing"
"time"

"github.com/hivecommons/hive/crux"
)

// seamFaultScanner is a CanaryScanner whose Scan raises the row's fault
// first. A nil fault is a healthy scanner that finds nothing.
type seamFaultScanner struct{ f *seamFault }

func (s seamFaultScanner) Scan(string, string, string) (crux.CanaryLeak, bool) {
s.f.raise()
return crux.CanaryLeak{}, false
}

// seamFaultBody is a request body whose Read fails.
type seamFaultBody struct{}

func (seamFaultBody) Read([]byte) (int, error) { return 0, errors.New("injected body read fault") }

var errSeamFault = errors.New("injected seam error")

// seamFault is one injected failure: an error, a panic, or a call that does
// not return before the seam's deadline. A nil *seamFault is healthy.
type seamFault struct {
kind string // "error", "panic" or "timeout"
release <-chan struct{}
}

// raise performs the fault. It returns errSeamFault for the error kind, nil
// when healthy, and never returns normally for panic (or until release closes
// for timeout).
func (f *seamFault) raise() error {
if f == nil {
return nil
}
switch f.kind {
case "error":
return errSeamFault
case "panic":
panic("injected policy seam fault")
case "timeout":
<-f.release
}
return nil
}

// TestPolicySeamFaultInjectionDenies covers the decision-path seams that are
// not Set*Func predicates (#19224; TestPolicyCallbackFailureDenies covers
// those). Every seam is driven with each fault it can raise (an error, a
// panic, a call past its deadline) and the gate must refuse. Each seam also
// runs healthy first and must allow the same request, so a refusal is the
// fault's doing and not a blanket deny. Before the guard, the panic rows
// crashed the test binary and the timeout rows blocked until the test
// timeout.
func TestPolicySeamFaultInjectionDenies(t *testing.T) {
prevPolicy, prevScan, prevLookup := policyCallbackTimeout, canaryScanTimeout, referenceLookupTimeout
policyCallbackTimeout, canaryScanTimeout, referenceLookupTimeout = 50*time.Millisecond, 50*time.Millisecond, 50*time.Millisecond
release := make(chan struct{})
t.Cleanup(func() {
close(release)
policyCallbackTimeout, canaryScanTimeout, referenceLookupTimeout = prevPolicy, prevScan, prevLookup
})
errKind := &seamFault{kind: "error"}
panicKind := &seamFault{kind: "panic"}
timeoutKind := &seamFault{kind: "timeout", release: release}

replyPath := "/repos/cncf/techdocs/pulls/comments/" + rtReplyID + "/replies"
deleteOwnPath := "/repos/cncf/techdocs/issues/comments/" + rtCommentA
refPol := ReferenceRepoPolicy{Repo: "hivecommons/docs", Issues: true, Login: "hivecommons-hive[bot]", Token: func() (string, bool) { return "ref-tok", true }}

seams := []struct {
name string
faults []*seamFault
// check evaluates the gate with the seam raising f (nil = healthy)
// and reports the refusal reason and whether it refused.
check func(t *testing.T, f *seamFault) (string, bool)
// failReason, when set, is the exact reason a faulted call must give.
failReason string
}{
{
name: "review-thread OwnPR",
faults: []*seamFault{panicKind, timeoutKind},
check: func(t *testing.T, f *seamFault) (string, bool) {
p := newRTProxy(t, newRTFixture(time.Now()), nil)
g := p.reviewThreadGuard()
g.OwnPR = func(repo string, n int) (string, bool) {
_ = f.raise()
return rtOwnPR(repo, n)
}
p.SetReviewThreadGuard(g)
return p.reviewThreadRefusal(rtReq("POST", replyPath, `{"body":"x"}`), rtOwner)
},
failReason: policyCallbackFailedReason(reviewOwnershipGate),
},
{
name: "review-thread HiveLogins",
faults: []*seamFault{panicKind, timeoutKind},
check: func(t *testing.T, f *seamFault) (string, bool) {
p := newRTProxy(t, newRTFixture(time.Now()), nil)
g := p.reviewThreadGuard()
g.HiveLogins = func() []string {
_ = f.raise()
return []string{rtHiveBot}
}
p.SetReviewThreadGuard(g)
return p.reviewThreadRefusal(rtReq("DELETE", deleteOwnPath, ""), rtOwner)
},
failReason: policyCallbackFailedReason(reviewOwnershipGate),
},
{
name: "review-thread comment lookup",
faults: []*seamFault{errKind, panicKind, timeoutKind},
check: func(t *testing.T, f *seamFault) (string, bool) {
fx := newRTFixture(time.Now())
p := newRTProxy(t, fx, nil)
p.referenceLookup = func(url, token string) ([]byte, error) {
if err := f.raise(); err != nil {
return nil, err
}
return fx.lookup(url, token)
}
return p.reviewThreadRefusal(rtReq("POST", replyPath, `{"body":"x"}`), rtOwner)
},
},
{
name: "reference repo own-item lookup",
faults: []*seamFault{errKind, panicKind, timeoutKind},
check: func(t *testing.T, f *seamFault) (string, bool) {
p := newReferenceTestProxy(t, refPol, func(string, string) ([]byte, error) {
if err := f.raise(); err != nil {
return nil, err
}
return []byte(`{"user":{"login":"hivecommons-hive[bot]"}}`), nil
})
return p.referenceRepoRefusal(refReq("POST", "/repos/hivecommons/docs/issues/12/comments", "", nil))
},
},
{
// Advisory canary mode (failClosed=false) still refuses a scan
// that could not run: advisory governs what a found leak does,
// and an unscanned body has not been found clean.
name: "canary scan",
faults: []*seamFault{errKind, panicKind, timeoutKind},
check: func(t *testing.T, f *seamFault) (string, bool) {
p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))}
req, _ := http.NewRequest(http.MethodPost, "https://api.github.com/repos/org/repo/issues/1/comments", strings.NewReader(`{"body":"hello"}`))
scanFault := f
if f != nil && f.kind == "error" {
// Scan has no error return; this seam's error branch is
// the body it would scan being unreadable.
req.Body = io.NopCloser(seamFaultBody{})
scanFault = nil
}
p.SetCanaryScanner(true, false, seamFaultScanner{f: scanFault}, nil)
reason, deny, _ := p.inspectCanaryEgress("scanner", req)
return reason, deny
},
},
}

for _, s := range seams {
t.Run(s.name+"/healthy", func(t *testing.T) {
if reason, denied := s.check(t, nil); denied {
t.Fatalf("healthy %s refused: %q", s.name, reason)
}
})
for _, f := range s.faults {
t.Run(s.name+"/"+f.kind, func(t *testing.T) {
reason, denied := s.check(t, f)
if !denied {
t.Fatalf("%s %s: gate allowed the request; want deny", s.name, f.kind)
}
if s.failReason != "" && reason != s.failReason {
t.Fatalf("%s %s: reason = %q; want %q", s.name, f.kind, reason, s.failReason)
}
})
}
}
}
28 changes: 25 additions & 3 deletions src/crux/proxy/reference_repos.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"bytes"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
Expand Down Expand Up @@ -355,13 +356,34 @@ func defaultReferenceLookup(url, token string) ([]byte, error) {
return body, nil
}

// referenceLookupOrDefault returns the lookup wrapped in the policy-seam
// guard (#19224): a lookup that panics or outlives referenceLookupTimeout
// answers an error, which every caller already turns into a refusal.
func (p *GitHubProxy) referenceLookupOrDefault() referenceLookupFunc {
if p.referenceLookup != nil {
return p.referenceLookup
lookup := p.referenceLookup
if lookup == nil {
lookup = defaultReferenceLookup
}
return func(url, token string) ([]byte, error) {
type answer struct {
body []byte
err error
}
a, ok := callPolicySeam(p, "reference lookup", referenceLookupTimeout, func() answer {
body, err := lookup(url, token)
return answer{body, err}
})
if !ok {
return nil, errReferenceLookupFailed
}
return a.body, a.err
}
return defaultReferenceLookup
}

// errReferenceLookupFailed is the error a guarded lookup answers when the
// lookup itself panicked or timed out.
var errReferenceLookupFailed = errors.New("the GitHub lookup failed or timed out")

func (p *GitHubProxy) referenceToken(pol ReferenceRepoPolicy) string {
if pol.Token == nil {
return ""
Expand Down
Loading
Loading