From 639b145eb73fb45d142196e9e1243ff13a1fc98b Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 21:28:10 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20dashboard:=20a=20hosted=20spoke'?= =?UTF-8?q?s=20JSON=20502=20reaches=20the=20owner=20instead=20of=20the=20i?= =?UTF-8?q?ngress=20page?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hosted ingress intercepts 502/503 and serves its generic 'hive unavailable' page, so a 502 the spoke answered with its own error read as an outage. On a request carrying the hub's X-Hive-Proxy-Auth proof, a JSON 502 from an /api/ handler now goes out as 424 (passed through by the ingress) with X-Hive-Status: 502. Unreachable spokes still get the generic page; 503 is unchanged because the UI reads spoke 503s by status. Refs #18598 Signed-off-by: Andy Anderson --- ...18598-hosted-spoke-502-body-passthrough.md | 1 + src/pkg/dashboard/hosted_gateway_status.go | 110 ++++++++++++++++++ .../dashboard/hosted_gateway_status_test.go | 95 +++++++++++++++ src/pkg/dashboard/server.go | 4 +- 4 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-18598-hosted-spoke-502-body-passthrough.md create mode 100644 src/pkg/dashboard/hosted_gateway_status.go create mode 100644 src/pkg/dashboard/hosted_gateway_status_test.go diff --git a/changelog.d/fixed-18598-hosted-spoke-502-body-passthrough.md b/changelog.d/fixed-18598-hosted-spoke-502-body-passthrough.md new file mode 100644 index 0000000000..29363da464 --- /dev/null +++ b/changelog.d/fixed-18598-hosted-spoke-502-body-passthrough.md @@ -0,0 +1 @@ +- **A hosted hive's own error is no longer hidden behind "hive unavailable".** The hosted ingress replaces any 502 with its generic "this hive is restarting" page, so an error the spoke actually answered (a Discord join refused for a missing permission, a failed upstream call) read as an outage. On a request that came through the hosted ingress, a JSON 502 from a dashboard API handler is now sent as 424 with the original status in `X-Hive-Status`, which the ingress passes through with the spoke's message; the generic page still appears when the spoke cannot be reached ([#18598](https://github.com/hivecommons/hive/issues/18598)). diff --git a/src/pkg/dashboard/hosted_gateway_status.go b/src/pkg/dashboard/hosted_gateway_status.go new file mode 100644 index 0000000000..1a9bf115c3 --- /dev/null +++ b/src/pkg/dashboard/hosted_gateway_status.go @@ -0,0 +1,110 @@ +package dashboard + +import ( + "bufio" + "io" + "net" + "net/http" + "strings" +) + +// The hosted Ingress in front of a hub-proxied hive intercepts 502 and 503 +// (custom-http-errors "502,503", pkg/hub/saas_provision.go) and replaces the +// body with the generic "hive unavailable" page, so the owner is told the +// hive is restarting. That page is right when the spoke is unreachable (the +// Ingress itself produced the status), and wrong when the spoke answered a +// 502 carrying its own error: a Discord join Quorum refused for a missing +// permission read as "hive is restarting" (hivecommons/hive#18598), as a +// GitHub 422 on a hold label once did (#17037). +// +// withHostedGatewayPassthrough fixes that for every API route at once: on a +// request that came through the hosted Ingress (it carries the hub's +// X-Hive-Proxy-Auth proof), a JSON 502 a handler writes goes out as 424 +// Failed Dependency, which the Ingress passes through with its body, and the +// original status rides along in X-Hive-Status. A spoke the Ingress cannot +// reach still gets the generic page. 503 is left alone: the dashboard's own +// clients read a spoke 503 ("not enabled") by status, and the generic page's +// retry is the right answer for a hive that is starting. +const ( + // hostedGatewayPassthroughStatus is the status a spoke-answered JSON 502 + // is sent under on a hosted request; the Ingress does not intercept it. + hostedGatewayPassthroughStatus = http.StatusFailedDependency + // hostedGatewayOriginalStatusHeader names the status the handler wrote. + hostedGatewayOriginalStatusHeader = "X-Hive-Status" +) + +// isHostedGatewayRequest reports whether r came through the hosted Ingress +// to an API route. The proof header's presence is enough here: this only +// picks the status an error is sent under, it grants nothing. +func isHostedGatewayRequest(r *http.Request) bool { + return r.Header.Get(proxyAuthHeader) != "" && strings.HasPrefix(r.URL.Path, "/api/") +} + +func (s *Server) withHostedGatewayPassthrough(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !isHostedGatewayRequest(r) { + next.ServeHTTP(w, r) + return + } + next.ServeHTTP(&hostedGatewayStatusWriter{ResponseWriter: w}, r) + }) +} + +// hostedGatewayStatusWriter rewrites a JSON 502 to the pass-through status. +type hostedGatewayStatusWriter struct { + http.ResponseWriter + wrote bool +} + +func (w *hostedGatewayStatusWriter) Unwrap() http.ResponseWriter { + return w.ResponseWriter +} + +func (w *hostedGatewayStatusWriter) WriteHeader(status int) { + if !w.wrote { + w.wrote = true + if status == http.StatusBadGateway && isJSONContentType(w.Header().Get("Content-Type")) { + w.Header().Set(hostedGatewayOriginalStatusHeader, "502") + status = hostedGatewayPassthroughStatus + } + } + w.ResponseWriter.WriteHeader(status) +} + +func isJSONContentType(ct string) bool { + return strings.HasPrefix(strings.ToLower(strings.TrimSpace(ct)), "application/json") +} + +func (w *hostedGatewayStatusWriter) Write(b []byte) (int, error) { + if !w.wrote { + w.WriteHeader(http.StatusOK) + } + return w.ResponseWriter.Write(b) +} + +func (w *hostedGatewayStatusWriter) Flush() { + if !w.wrote { + w.WriteHeader(http.StatusOK) + } + if f, ok := w.ResponseWriter.(http.Flusher); ok { + f.Flush() + } +} + +func (w *hostedGatewayStatusWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + h, ok := w.ResponseWriter.(http.Hijacker) + if !ok { + return nil, nil, http.ErrNotSupported + } + return h.Hijack() +} + +func (w *hostedGatewayStatusWriter) ReadFrom(r io.Reader) (int64, error) { + if !w.wrote { + w.WriteHeader(http.StatusOK) + } + if rf, ok := w.ResponseWriter.(io.ReaderFrom); ok { + return rf.ReadFrom(r) + } + return io.Copy(w.ResponseWriter, r) +} diff --git a/src/pkg/dashboard/hosted_gateway_status_test.go b/src/pkg/dashboard/hosted_gateway_status_test.go new file mode 100644 index 0000000000..2195fdb0f0 --- /dev/null +++ b/src/pkg/dashboard/hosted_gateway_status_test.go @@ -0,0 +1,95 @@ +package dashboard + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// A JSON 502 the spoke answers on a hosted request must reach the owner with +// its own body, not the Ingress's generic "hive unavailable" page +// (hivecommons/hive#18598). +func TestHostedGatewayPassthroughStatus(t *testing.T) { + s := NewServer(0, nil) + const refusal = "quorum meeting.join: blocked: the bot has no permission to connect to that voice channel" + jsonFail := func(code int) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { jsonError(w, refusal, code) }) + } + htmlFail := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusBadGateway) + _, _ = w.Write([]byte("upstream")) + }) + implicitOK := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"ok":true}`)) }) + + cases := []struct { + name string + h http.Handler + path string + hosted bool + wantStatus int + wantOrig string + }{ + {"hosted JSON 502 passes through", jsonFail(http.StatusBadGateway), "/api/meetings/mtg-1/approve", true, hostedGatewayPassthroughStatus, "502"}, + {"self-hosted keeps 502", jsonFail(http.StatusBadGateway), "/api/meetings/mtg-1/approve", false, http.StatusBadGateway, ""}, + {"non-API path keeps 502", jsonFail(http.StatusBadGateway), "/snapshot", true, http.StatusBadGateway, ""}, + {"non-JSON 502 keeps 502", htmlFail, "/api/x", true, http.StatusBadGateway, ""}, + {"503 is left alone", jsonFail(http.StatusServiceUnavailable), "/api/meetings", true, http.StatusServiceUnavailable, ""}, + {"other errors unchanged", jsonFail(http.StatusConflict), "/api/meetings", true, http.StatusConflict, ""}, + {"implicit 200 unchanged", implicitOK, "/api/meetings", true, http.StatusOK, ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, c.path, nil) + if c.hosted { + req.Header.Set(proxyAuthHeader, "proof") + } + rec := httptest.NewRecorder() + s.withHostedGatewayPassthrough(c.h).ServeHTTP(rec, req) + if rec.Code != c.wantStatus { + t.Fatalf("status = %d, want %d", rec.Code, c.wantStatus) + } + if got := rec.Header().Get(hostedGatewayOriginalStatusHeader); got != c.wantOrig { + t.Errorf("%s = %q, want %q", hostedGatewayOriginalStatusHeader, got, c.wantOrig) + } + if c.wantOrig != "" && !strings.Contains(rec.Body.String(), refusal) { + t.Errorf("body lost the spoke's error: %s", rec.Body.String()) + } + }) + } +} + +// The wrapper is in the served handler chain, and keeps Flush and Unwrap +// working for streaming handlers. +func TestHostedGatewayPassthroughWiredAndStreams(t *testing.T) { + s := newMinimalServer(t) + s.mux.HandleFunc("GET /api/test-gateway-502", func(w http.ResponseWriter, _ *http.Request) { + jsonError(w, "upstream said no", http.StatusBadGateway) + }) + req := httptest.NewRequest(http.MethodGet, "/api/test-gateway-502", nil) + req.Header.Set(proxyAuthHeader, "proof") + rec := httptest.NewRecorder() + s.Handler().ServeHTTP(rec, req) + // authenticate may refuse a wrong proof first; either way the Ingress + // must never see a JSON 502 from a hosted request. + if rec.Code == http.StatusBadGateway { + t.Fatalf("hosted JSON 502 reached the Ingress unchanged: %s", rec.Body.String()) + } + + gw := &hostedGatewayStatusWriter{ResponseWriter: httptest.NewRecorder()} + gw.Flush() + if !gw.wrote { + t.Error("Flush before WriteHeader must commit a 200") + } + if _, ok := gw.Unwrap().(*httptest.ResponseRecorder); !ok { + t.Error("Unwrap must return the wrapped writer") + } + if _, _, err := gw.Hijack(); err == nil { + t.Error("Hijack on a non-hijacker must fail") + } + n, err := (&hostedGatewayStatusWriter{ResponseWriter: httptest.NewRecorder()}).ReadFrom(strings.NewReader("abc")) + if err != nil || n != 3 { + t.Errorf("ReadFrom = %d, %v", n, err) + } +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index f86b9265dd..246f608aa8 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -1860,7 +1860,7 @@ func (s *Server) Start() error { // authenticate is outermost so the identity headers it injects from a // per-user session are visible to roleEnforcement's read-only write-gate. - handler := s.withServerTiming(s.noIndex(s.authenticate(s.roleEnforcement(s.securityHeaders(s.mux))))) + handler := s.withServerTiming(s.withHostedGatewayPassthrough(s.noIndex(s.authenticate(s.roleEnforcement(s.securityHeaders(s.mux)))))) const dashboardReadTimeout = 30 * time.Second const dashboardIdleTimeout = 120 * time.Second @@ -2797,7 +2797,7 @@ async function poll(interval,flowId){ ` func (s *Server) Handler() http.Handler { - return s.noIndex(s.authenticate(s.roleEnforcement(s.securityHeaders(s.mux)))) + return s.withHostedGatewayPassthrough(s.noIndex(s.authenticate(s.roleEnforcement(s.securityHeaders(s.mux))))) } func (s *Server) roleEnforcement(next http.Handler) http.Handler {