Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-18377-proxy-proof-warn-throttle.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Dashboard: the "rejected hub-proxied request with unverified proxy proof" warning is now logged at most once a minute per reason and path (with a suppressed count) instead of once per request; the request is still rejected with 403 (#18377).
45 changes: 45 additions & 0 deletions src/pkg/dashboard/proxy_proof_warn_gate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package dashboard

import (
"sync"
"time"
)

// proxyProofWarnInterval is the minimum gap between WARN logs for the same
// rejection reason and path. A hub/spoke token mismatch rejects every request
// the hub proxies, which flooded the log (99 in 10m, #18377).
const proxyProofWarnInterval = time.Minute

// proxyProofWarnGate dedupes the proxy-proof rejection WARN. The 403 itself is
// never throttled; only the log line is.
var proxyProofWarnGate = &warnGate{interval: proxyProofWarnInterval}

type warnGate struct {
mu sync.Mutex
interval time.Duration
last map[string]time.Time
suppressed map[string]int
}

// allow reports whether key may log now, and how many logs for key were
// suppressed since the last allowed one.
func (g *warnGate) allow(key string, now time.Time) (suppressed int, ok bool) {
g.mu.Lock()
defer g.mu.Unlock()
if g.last == nil {
g.last = map[string]time.Time{}
g.suppressed = map[string]int{}
}
if t, seen := g.last[key]; seen && now.Sub(t) < g.interval {
g.suppressed[key]++
return 0, false
}
if len(g.last) >= 256 { // bound memory against forged paths
g.last = map[string]time.Time{}
g.suppressed = map[string]int{}
}
n := g.suppressed[key]
delete(g.suppressed, key)
g.last[key] = now
return n, true
}
26 changes: 26 additions & 0 deletions src/pkg/dashboard/proxy_proof_warn_gate_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package dashboard

import (
"testing"
"time"
)

func TestWarnGateDedupesWithinInterval(t *testing.T) {
g := &warnGate{interval: time.Minute}
t0 := time.Unix(1000, 0)
if _, ok := g.allow("k", t0); !ok {
t.Fatal("first call must log")
}
for i := 0; i < 5; i++ {
if _, ok := g.allow("k", t0.Add(time.Second)); ok {
t.Fatal("repeat within interval must be suppressed")
}
}
if _, ok := g.allow("other", t0); !ok {
t.Fatal("distinct key must log")
}
n, ok := g.allow("k", t0.Add(2*time.Minute))
if !ok || n != 5 {
t.Fatalf("got n=%d ok=%v, want 5 true", n, ok)
}
}
6 changes: 4 additions & 2 deletions src/pkg/dashboard/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -2089,8 +2089,10 @@ func (s *Server) securityHeaders(next http.Handler) http.Handler {
// auth-signin redirect, which is how #9785 looped. The page names the cause so
// the hive's operator can act on it instead of seeing a browser error.
func writeProxyProofRejected(w http.ResponseWriter, r *http.Request, reason string) {
slog.Warn("dashboard: rejected hub-proxied request with unverified proxy proof; hub and spoke dashboard tokens disagree or hub omitted the proof",
"reason", reason, "path", r.URL.Path, "user", r.Header.Get("X-Hive-User"))
if n, ok := proxyProofWarnGate.allow(reason+"|"+r.URL.Path, time.Now()); ok {
slog.Warn("dashboard: rejected hub-proxied request with unverified proxy proof; hub and spoke dashboard tokens disagree or hub omitted the proof",
"reason", reason, "path", r.URL.Path, "user", r.Header.Get("X-Hive-User"), "suppressed", n)
}
w.Header().Set("Cache-Control", "no-store")
if strings.HasPrefix(r.URL.Path, "/api/") {
jsonStatusResponse(w, http.StatusForbidden, map[string]string{"error": reason})
Expand Down
Loading