Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **A hosted hive's own error is no longer hidden behind "hive unavailable".** The hosted ingress replaces any 502 with its generic "this hive is restarting" page, so an error the spoke actually answered (a Discord join refused for a missing permission, a failed upstream call) read as an outage. On a request that came through the hosted ingress, a JSON 502 from a dashboard API handler is now sent as 424 with the original status in `X-Hive-Status`, which the ingress passes through with the spoke's message; the generic page still appears when the spoke cannot be reached ([#18598](https://github.com/hivecommons/hive/issues/18598)).
110 changes: 110 additions & 0 deletions src/pkg/dashboard/hosted_gateway_status.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package dashboard

import (
"bufio"
"io"
"net"
"net/http"
"strings"
)

// The hosted Ingress in front of a hub-proxied hive intercepts 502 and 503
// (custom-http-errors "502,503", pkg/hub/saas_provision.go) and replaces the
// body with the generic "hive unavailable" page, so the owner is told the
// hive is restarting. That page is right when the spoke is unreachable (the
// Ingress itself produced the status), and wrong when the spoke answered a
// 502 carrying its own error: a Discord join Quorum refused for a missing
// permission read as "hive is restarting" (hivecommons/hive#18598), as a
// GitHub 422 on a hold label once did (#17037).
//
// withHostedGatewayPassthrough fixes that for every API route at once: on a
// request that came through the hosted Ingress (it carries the hub's
// X-Hive-Proxy-Auth proof), a JSON 502 a handler writes goes out as 424
// Failed Dependency, which the Ingress passes through with its body, and the
// original status rides along in X-Hive-Status. A spoke the Ingress cannot
// reach still gets the generic page. 503 is left alone: the dashboard's own
// clients read a spoke 503 ("not enabled") by status, and the generic page's
// retry is the right answer for a hive that is starting.
const (
// hostedGatewayPassthroughStatus is the status a spoke-answered JSON 502
// is sent under on a hosted request; the Ingress does not intercept it.
hostedGatewayPassthroughStatus = http.StatusFailedDependency
// hostedGatewayOriginalStatusHeader names the status the handler wrote.
hostedGatewayOriginalStatusHeader = "X-Hive-Status"
)

// isHostedGatewayRequest reports whether r came through the hosted Ingress
// to an API route. The proof header's presence is enough here: this only
// picks the status an error is sent under, it grants nothing.
func isHostedGatewayRequest(r *http.Request) bool {
return r.Header.Get(proxyAuthHeader) != "" && strings.HasPrefix(r.URL.Path, "/api/")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Presence-only check lets any anonymous caller send a junk X-Hive-Proxy-Auth and unmask 502 bodies on public /api/ routes (contribute, gh-user-auth, v1...), which the ingress used to hide. Validate the proof like authenticate does (server.go:2170):

Suggested change
return r.Header.Get(proxyAuthHeader) != "" && strings.HasPrefix(r.URL.Path, "/api/")
func (s *Server) isHostedGatewayRequest(r *http.Request) bool {
proof := r.Header.Get(proxyAuthHeader)
return proof != "" && s.authToken != "" && secureCompare(proof, s.authToken) && strings.HasPrefix(r.URL.Path, "/api/")
}

and call s.isHostedGatewayRequest(r) at line 46. The test must then set the proof to the server's auth token.

}

func (s *Server) withHostedGatewayPassthrough(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !isHostedGatewayRequest(r) {
next.ServeHTTP(w, r)
return
}
next.ServeHTTP(&hostedGatewayStatusWriter{ResponseWriter: w}, r)
})
}

// hostedGatewayStatusWriter rewrites a JSON 502 to the pass-through status.
type hostedGatewayStatusWriter struct {
http.ResponseWriter
wrote bool
}

func (w *hostedGatewayStatusWriter) Unwrap() http.ResponseWriter {
return w.ResponseWriter
}

func (w *hostedGatewayStatusWriter) WriteHeader(status int) {
if !w.wrote {
w.wrote = true
if status == http.StatusBadGateway && isJSONContentType(w.Header().Get("Content-Type")) {
w.Header().Set(hostedGatewayOriginalStatusHeader, "502")
status = hostedGatewayPassthroughStatus
}
}
w.ResponseWriter.WriteHeader(status)
}

func isJSONContentType(ct string) bool {
return strings.HasPrefix(strings.ToLower(strings.TrimSpace(ct)), "application/json")
}

func (w *hostedGatewayStatusWriter) Write(b []byte) (int, error) {
if !w.wrote {
w.WriteHeader(http.StatusOK)
}
return w.ResponseWriter.Write(b)
}

func (w *hostedGatewayStatusWriter) Flush() {
if !w.wrote {
w.WriteHeader(http.StatusOK)
}
if f, ok := w.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}

func (w *hostedGatewayStatusWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
h, ok := w.ResponseWriter.(http.Hijacker)
if !ok {
return nil, nil, http.ErrNotSupported
}
return h.Hijack()
}

func (w *hostedGatewayStatusWriter) ReadFrom(r io.Reader) (int64, error) {
if !w.wrote {
w.WriteHeader(http.StatusOK)
}
if rf, ok := w.ResponseWriter.(io.ReaderFrom); ok {
return rf.ReadFrom(r)
}
return io.Copy(w.ResponseWriter, r)
}
95 changes: 95 additions & 0 deletions src/pkg/dashboard/hosted_gateway_status_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package dashboard

import (
"net/http"
"net/http/httptest"
"strings"
"testing"
)

// A JSON 502 the spoke answers on a hosted request must reach the owner with
// its own body, not the Ingress's generic "hive unavailable" page
// (hivecommons/hive#18598).
func TestHostedGatewayPassthroughStatus(t *testing.T) {
s := NewServer(0, nil)
const refusal = "quorum meeting.join: blocked: the bot has no permission to connect to that voice channel"
jsonFail := func(code int) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { jsonError(w, refusal, code) })
}
htmlFail := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusBadGateway)
_, _ = w.Write([]byte("<html>upstream</html>"))
})
implicitOK := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"ok":true}`)) })

cases := []struct {
name string
h http.Handler
path string
hosted bool
wantStatus int
wantOrig string
}{
{"hosted JSON 502 passes through", jsonFail(http.StatusBadGateway), "/api/meetings/mtg-1/approve", true, hostedGatewayPassthroughStatus, "502"},
{"self-hosted keeps 502", jsonFail(http.StatusBadGateway), "/api/meetings/mtg-1/approve", false, http.StatusBadGateway, ""},
{"non-API path keeps 502", jsonFail(http.StatusBadGateway), "/snapshot", true, http.StatusBadGateway, ""},
{"non-JSON 502 keeps 502", htmlFail, "/api/x", true, http.StatusBadGateway, ""},
{"503 is left alone", jsonFail(http.StatusServiceUnavailable), "/api/meetings", true, http.StatusServiceUnavailable, ""},
{"other errors unchanged", jsonFail(http.StatusConflict), "/api/meetings", true, http.StatusConflict, ""},
{"implicit 200 unchanged", implicitOK, "/api/meetings", true, http.StatusOK, ""},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, c.path, nil)
if c.hosted {
req.Header.Set(proxyAuthHeader, "proof")
}
rec := httptest.NewRecorder()
s.withHostedGatewayPassthrough(c.h).ServeHTTP(rec, req)
if rec.Code != c.wantStatus {
t.Fatalf("status = %d, want %d", rec.Code, c.wantStatus)
}
if got := rec.Header().Get(hostedGatewayOriginalStatusHeader); got != c.wantOrig {
t.Errorf("%s = %q, want %q", hostedGatewayOriginalStatusHeader, got, c.wantOrig)
}
if c.wantOrig != "" && !strings.Contains(rec.Body.String(), refusal) {
t.Errorf("body lost the spoke's error: %s", rec.Body.String())
}
})
}
}

// The wrapper is in the served handler chain, and keeps Flush and Unwrap
// working for streaming handlers.
func TestHostedGatewayPassthroughWiredAndStreams(t *testing.T) {
s := newMinimalServer(t)
s.mux.HandleFunc("GET /api/test-gateway-502", func(w http.ResponseWriter, _ *http.Request) {
jsonError(w, "upstream said no", http.StatusBadGateway)
})
req := httptest.NewRequest(http.MethodGet, "/api/test-gateway-502", nil)
req.Header.Set(proxyAuthHeader, "proof")
rec := httptest.NewRecorder()
s.Handler().ServeHTTP(rec, req)
// authenticate may refuse a wrong proof first; either way the Ingress
// must never see a JSON 502 from a hosted request.
if rec.Code == http.StatusBadGateway {
t.Fatalf("hosted JSON 502 reached the Ingress unchanged: %s", rec.Body.String())
}

gw := &hostedGatewayStatusWriter{ResponseWriter: httptest.NewRecorder()}
gw.Flush()
if !gw.wrote {
t.Error("Flush before WriteHeader must commit a 200")
}
if _, ok := gw.Unwrap().(*httptest.ResponseRecorder); !ok {
t.Error("Unwrap must return the wrapped writer")
}
if _, _, err := gw.Hijack(); err == nil {
t.Error("Hijack on a non-hijacker must fail")
}
n, err := (&hostedGatewayStatusWriter{ResponseWriter: httptest.NewRecorder()}).ReadFrom(strings.NewReader("abc"))
if err != nil || n != 3 {
t.Errorf("ReadFrom = %d, %v", n, err)
}
}
4 changes: 2 additions & 2 deletions src/pkg/dashboard/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -1860,7 +1860,7 @@ func (s *Server) Start() error {

// authenticate is outermost so the identity headers it injects from a
// per-user session are visible to roleEnforcement's read-only write-gate.
handler := s.withServerTiming(s.noIndex(s.authenticate(s.roleEnforcement(s.securityHeaders(s.mux)))))
handler := s.withServerTiming(s.withHostedGatewayPassthrough(s.noIndex(s.authenticate(s.roleEnforcement(s.securityHeaders(s.mux))))))

const dashboardReadTimeout = 30 * time.Second
const dashboardIdleTimeout = 120 * time.Second
Expand Down Expand Up @@ -2797,7 +2797,7 @@ async function poll(interval,flowId){
</script></body></html>`

func (s *Server) Handler() http.Handler {
return s.noIndex(s.authenticate(s.roleEnforcement(s.securityHeaders(s.mux))))
return s.withHostedGatewayPassthrough(s.noIndex(s.authenticate(s.roleEnforcement(s.securityHeaders(s.mux)))))
}

func (s *Server) roleEnforcement(next http.Handler) http.Handler {
Expand Down
Loading