From a8577c8ca219c4891a4ab60db48ad6fa32afb7d6 Mon Sep 17 00:00:00 2001 From: bsian03 Date: Tue, 16 Sep 2025 18:23:37 +0100 Subject: [PATCH 1/4] Fix webhooks and reactions 404 issues (#21) * Fix webhook fast 404 when getting deleted webhook message * encode emojis --- lib/discord.go | 14 ++++++++++++-- lib/queue.go | 17 +++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index cf99f86..b7bafdd 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -5,15 +5,17 @@ import ( "crypto/tls" "encoding/json" "errors" - "github.com/sirupsen/logrus" "io" "io/ioutil" "math" "net" "net/http" + "net/url" "strconv" "strings" "time" + + "github.com/sirupsen/logrus" ) var client *http.Client @@ -213,6 +215,15 @@ func GetBotUser(token string) (*BotUserResponse, error) { } func doDiscordReq(ctx context.Context, path string, method string, body io.ReadCloser, header http.Header, query string) (*http.Response, error) { + route := GetMetricsPath(path) + if route == "/channels/!/messages/!/reactions/!/!" { + segs := strings.Split(path, "/") + unescaped, _ := url.PathUnescape(segs[7]) + if segs[7] == unescaped { + segs[7] = url.PathEscape(segs[7]) + } + path = strings.Join(segs, "/") + } discordReq, err := http.NewRequestWithContext(ctx, method, "https://discord.com"+path+"?"+query, body) if err != nil { return nil, err @@ -229,7 +240,6 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC } if err == nil { - route := GetMetricsPath(path) status := discordResp.Status method := discordResp.Request.Method elapsed := time.Since(startTime).Seconds() diff --git a/lib/queue.go b/lib/queue.go index 203383a..eab5eb8 100644 --- a/lib/queue.go +++ b/lib/queue.go @@ -1,17 +1,21 @@ package lib import ( + "bytes" "context" "errors" - "github.com/Clever/leakybucket" - "github.com/Clever/leakybucket/memory" - "github.com/sirupsen/logrus" + "io" + "io/ioutil" "net/http" "strconv" "strings" "sync" "sync/atomic" "time" + + "github.com/Clever/leakybucket" + "github.com/Clever/leakybucket/memory" + "github.com/sirupsen/logrus" ) type QueueItem struct { @@ -272,6 +276,11 @@ func return401(item *QueueItem) { item.doneChan <- nil } +func isUnknownWebhook(_body io.ReadCloser) bool { + body, _ := ioutil.ReadAll(_body); + return bytes.Contains(body, []byte("\"code\": 10015")) +} + func isInteraction(url string) bool { parts := strings.Split(strings.SplitN(url, "?", 1)[0], "/") for _, p := range parts { @@ -348,7 +357,7 @@ func (q *RequestQueue) subscribe(ch *QueueChannel, path string, pathHash uint64) }).Warn("Unexpected 429") } - if resp.StatusCode == 404 && strings.HasPrefix(path, "/webhooks/") && !isInteraction(item.Req.URL.String()) { + if resp.StatusCode == 404 && isUnknownWebhook(resp.Body) && !isInteraction(item.Req.URL.String()) { logger.WithFields(logrus.Fields{ "bucket": path, "route": item.Req.URL.String(), From 48c8455a0c8e7e548495d71d62484e04b001467a Mon Sep 17 00:00:00 2001 From: Germano Eichenberg Date: Tue, 16 Sep 2025 15:46:42 -0300 Subject: [PATCH 2/4] Fix reaction encoding and webhook 404 detection --- lib/discord.go | 36 ++++++++++++++++++++++++++++++++---- lib/queue.go | 23 +++++++++++++++++++---- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index b7bafdd..32e7613 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -1,6 +1,7 @@ package lib import ( + "bytes" "context" "crypto/tls" "encoding/json" @@ -218,11 +219,22 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC route := GetMetricsPath(path) if route == "/channels/!/messages/!/reactions/!/!" { segs := strings.Split(path, "/") - unescaped, _ := url.PathUnescape(segs[7]) - if segs[7] == unescaped { - segs[7] = url.PathEscape(segs[7]) + emojiIdx := -1 + for idx, seg := range segs { + if seg == "reactions" { + emojiIdx = idx + 1 + break + } + } + if emojiIdx >= 0 && emojiIdx < len(segs) { + emojiSegment := segs[emojiIdx] + if emojiSegment != "" { + if unescaped, err := url.PathUnescape(emojiSegment); err == nil && emojiSegment == unescaped { + segs[emojiIdx] = url.PathEscape(emojiSegment) + path = strings.Join(segs, "/") + } + } } - path = strings.Join(segs, "/") } discordReq, err := http.NewRequestWithContext(ctx, method, "https://discord.com"+path+"?"+query, body) if err != nil { @@ -281,11 +293,27 @@ func ProcessRequest(ctx context.Context, item *QueueItem) (*http.Response, error "discordBucket": discordResp.Header.Get("x-ratelimit-bucket"), }).Debug("Discord request") + var respBody []byte + if discordResp.Body != nil { + respBody, err = ioutil.ReadAll(discordResp.Body) + if err != nil { + res.WriteHeader(500) + _, _ = res.Write([]byte(err.Error())) + return nil, err + } + _ = discordResp.Body.Close() + discordResp.Body = io.NopCloser(bytes.NewReader(respBody)) + } + err = CopyResponseToResponseWriter(discordResp, item.Res) if err != nil { return nil, err } + if discordResp.Body != nil { + discordResp.Body = io.NopCloser(bytes.NewReader(respBody)) + } + return discordResp, nil } diff --git a/lib/queue.go b/lib/queue.go index eab5eb8..e46ab94 100644 --- a/lib/queue.go +++ b/lib/queue.go @@ -276,8 +276,10 @@ func return401(item *QueueItem) { item.doneChan <- nil } -func isUnknownWebhook(_body io.ReadCloser) bool { - body, _ := ioutil.ReadAll(_body); +func isUnknownWebhook(body []byte) bool { + if len(body) == 0 { + return false + } return bytes.Contains(body, []byte("\"code\": 10015")) } @@ -338,7 +340,6 @@ func (q *RequestQueue) subscribe(ch *QueueChannel, path string, pathHash uint64) item.errChan <- err continue } - item.doneChan <- resp if resp.StatusCode == 429 && scope != "shared" { logger.WithFields(logrus.Fields{ @@ -357,7 +358,19 @@ func (q *RequestQueue) subscribe(ch *QueueChannel, path string, pathHash uint64) }).Warn("Unexpected 429") } - if resp.StatusCode == 404 && isUnknownWebhook(resp.Body) && !isInteraction(item.Req.URL.String()) { + var bodyBytes []byte + var bodyReadErr error + shouldCheckWebhook := resp.StatusCode == 404 && !isInteraction(item.Req.URL.String()) + if shouldCheckWebhook && resp.Body != nil { + bodyBytes, bodyReadErr = ioutil.ReadAll(resp.Body) + _ = resp.Body.Close() + resp.Body = io.NopCloser(bytes.NewReader(bodyBytes)) + if bodyReadErr != nil { + logger.WithError(bodyReadErr).Debug("Failed to read body for webhook detection") + } + } + + if shouldCheckWebhook && bodyReadErr == nil && isUnknownWebhook(bodyBytes) { logger.WithFields(logrus.Fields{ "bucket": path, "route": item.Req.URL.String(), @@ -366,6 +379,8 @@ func (q *RequestQueue) subscribe(ch *QueueChannel, path string, pathHash uint64) ret404 = true } + item.doneChan <- resp + if resp.StatusCode == 401 && !isInteraction(item.Req.URL.String()) && q.queueType != NoAuth { // Permanently lock this queue logger.WithFields(logrus.Fields{ From 406a6d85ec979df5beb702f93e05343b82934bcc Mon Sep 17 00:00:00 2001 From: Germano Eichenberg Date: Tue, 16 Sep 2025 15:54:15 -0300 Subject: [PATCH 3/4] Fix reaction emoji index and restore webhook detection --- lib/discord.go | 38 ++++++++------------------------------ lib/http.go | 4 +++- lib/queue.go | 23 ++++------------------- 3 files changed, 15 insertions(+), 50 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index 32e7613..7efb4d5 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -1,7 +1,6 @@ package lib import ( - "bytes" "context" "crypto/tls" "encoding/json" @@ -219,21 +218,16 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC route := GetMetricsPath(path) if route == "/channels/!/messages/!/reactions/!/!" { segs := strings.Split(path, "/") - emojiIdx := -1 - for idx, seg := range segs { - if seg == "reactions" { - emojiIdx = idx + 1 - break - } + emojiIdx := 7 + if strings.HasPrefix(path, "/") { + emojiIdx = 8 } - if emojiIdx >= 0 && emojiIdx < len(segs) { - emojiSegment := segs[emojiIdx] - if emojiSegment != "" { - if unescaped, err := url.PathUnescape(emojiSegment); err == nil && emojiSegment == unescaped { - segs[emojiIdx] = url.PathEscape(emojiSegment) - path = strings.Join(segs, "/") - } + if emojiIdx < len(segs) { + unescaped, _ := url.PathUnescape(segs[emojiIdx]) + if segs[emojiIdx] == unescaped { + segs[emojiIdx] = url.PathEscape(segs[emojiIdx]) } + path = strings.Join(segs, "/") } } discordReq, err := http.NewRequestWithContext(ctx, method, "https://discord.com"+path+"?"+query, body) @@ -293,27 +287,11 @@ func ProcessRequest(ctx context.Context, item *QueueItem) (*http.Response, error "discordBucket": discordResp.Header.Get("x-ratelimit-bucket"), }).Debug("Discord request") - var respBody []byte - if discordResp.Body != nil { - respBody, err = ioutil.ReadAll(discordResp.Body) - if err != nil { - res.WriteHeader(500) - _, _ = res.Write([]byte(err.Error())) - return nil, err - } - _ = discordResp.Body.Close() - discordResp.Body = io.NopCloser(bytes.NewReader(respBody)) - } - err = CopyResponseToResponseWriter(discordResp, item.Res) if err != nil { return nil, err } - if discordResp.Body != nil { - discordResp.Body = io.NopCloser(bytes.NewReader(respBody)) - } - return discordResp, nil } diff --git a/lib/http.go b/lib/http.go index eaa1ecd..67a2784 100644 --- a/lib/http.go +++ b/lib/http.go @@ -1,6 +1,7 @@ package lib import ( + "bytes" "io/ioutil" "net/http" "strings" @@ -34,5 +35,6 @@ func CopyResponseToResponseWriter(resp *http.Response, respWriter *http.Response if err != nil { return err } + resp.Body = ioutil.NopCloser(bytes.NewReader(body)) return nil -} \ No newline at end of file +} diff --git a/lib/queue.go b/lib/queue.go index e46ab94..eab5eb8 100644 --- a/lib/queue.go +++ b/lib/queue.go @@ -276,10 +276,8 @@ func return401(item *QueueItem) { item.doneChan <- nil } -func isUnknownWebhook(body []byte) bool { - if len(body) == 0 { - return false - } +func isUnknownWebhook(_body io.ReadCloser) bool { + body, _ := ioutil.ReadAll(_body); return bytes.Contains(body, []byte("\"code\": 10015")) } @@ -340,6 +338,7 @@ func (q *RequestQueue) subscribe(ch *QueueChannel, path string, pathHash uint64) item.errChan <- err continue } + item.doneChan <- resp if resp.StatusCode == 429 && scope != "shared" { logger.WithFields(logrus.Fields{ @@ -358,19 +357,7 @@ func (q *RequestQueue) subscribe(ch *QueueChannel, path string, pathHash uint64) }).Warn("Unexpected 429") } - var bodyBytes []byte - var bodyReadErr error - shouldCheckWebhook := resp.StatusCode == 404 && !isInteraction(item.Req.URL.String()) - if shouldCheckWebhook && resp.Body != nil { - bodyBytes, bodyReadErr = ioutil.ReadAll(resp.Body) - _ = resp.Body.Close() - resp.Body = io.NopCloser(bytes.NewReader(bodyBytes)) - if bodyReadErr != nil { - logger.WithError(bodyReadErr).Debug("Failed to read body for webhook detection") - } - } - - if shouldCheckWebhook && bodyReadErr == nil && isUnknownWebhook(bodyBytes) { + if resp.StatusCode == 404 && isUnknownWebhook(resp.Body) && !isInteraction(item.Req.URL.String()) { logger.WithFields(logrus.Fields{ "bucket": path, "route": item.Req.URL.String(), @@ -379,8 +366,6 @@ func (q *RequestQueue) subscribe(ch *QueueChannel, path string, pathHash uint64) ret404 = true } - item.doneChan <- resp - if resp.StatusCode == 401 && !isInteraction(item.Req.URL.String()) && q.queueType != NoAuth { // Permanently lock this queue logger.WithFields(logrus.Fields{ From 268caa11873cc1b582841557d5a1a8df1d8a84d1 Mon Sep 17 00:00:00 2001 From: Germano Eichenberg Date: Tue, 16 Sep 2025 16:26:52 -0300 Subject: [PATCH 4/4] Add tests for request and response body handling --- lib/process_request_test.go | 139 ++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 lib/process_request_test.go diff --git a/lib/process_request_test.go b/lib/process_request_test.go new file mode 100644 index 0000000..c34de15 --- /dev/null +++ b/lib/process_request_test.go @@ -0,0 +1,139 @@ +package lib + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/sirupsen/logrus" +) + +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (f roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func ensureTestLogger() { + if logger == nil { + SetLogger(logrus.New()) + } +} + +func TestDoDiscordReqForwardsRequestBody(t *testing.T) { + ensureTestLogger() + + expectedBody := "payload" + header := make(http.Header) + header.Set("Content-Type", "application/json") + + var capturedBody string + var capturedContentType string + + originalClient := client + client = &http.Client{Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + data, err := io.ReadAll(req.Body) + if err != nil { + return nil, err + } + capturedBody = string(data) + capturedContentType = req.Header.Get("Content-Type") + _ = req.Body.Close() + + return &http.Response{ + StatusCode: http.StatusNoContent, + Status: "204 No Content", + Header: make(http.Header), + Body: io.NopCloser(bytes.NewBuffer(nil)), + Request: req, + }, nil + })} + t.Cleanup(func() { client = originalClient }) + + resp, err := doDiscordReq(context.Background(), "/api/v10/webhooks/123", http.MethodPost, io.NopCloser(bytes.NewBufferString(expectedBody)), header, "") + if err != nil { + t.Fatalf("doDiscordReq returned error: %v", err) + } + defer resp.Body.Close() + + if capturedBody != expectedBody { + t.Fatalf("expected upstream to receive body %q, got %q", expectedBody, capturedBody) + } + if capturedContentType != "application/json" { + t.Fatalf("expected Content-Type header to be forwarded, got %q", capturedContentType) + } +} + +func TestProcessRequestPreservesResponseBody(t *testing.T) { + ensureTestLogger() + + const ( + expectedRequestBody = "request body" + upstreamBody = "{\"message\":\"Unknown Webhook\",\"code\":10015}" + ) + + var capturedBody string + + originalClient := client + client = &http.Client{Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + data, err := io.ReadAll(req.Body) + if err != nil { + return nil, err + } + capturedBody = string(data) + _ = req.Body.Close() + + return &http.Response{ + StatusCode: http.StatusNotFound, + Status: "404 Not Found", + Header: http.Header{ + "Content-Type": {"application/json"}, + "X-Ratelimit-Scope": {"user"}, + }, + Body: io.NopCloser(bytes.NewBufferString(upstreamBody)), + Request: req, + }, nil + })} + t.Cleanup(func() { client = originalClient }) + + originalTimeout := contextTimeout + contextTimeout = time.Second + t.Cleanup(func() { contextTimeout = originalTimeout }) + + req := httptest.NewRequest(http.MethodPost, "http://localhost/api/v10/webhooks/123", bytes.NewBufferString(expectedRequestBody)) + recorder := httptest.NewRecorder() + writer := http.ResponseWriter(recorder) + + item := &QueueItem{ + Req: req, + Res: &writer, + } + + resp, err := ProcessRequest(context.Background(), item) + if err != nil { + t.Fatalf("ProcessRequest returned error: %v", err) + } + defer resp.Body.Close() + + if capturedBody != expectedRequestBody { + t.Fatalf("expected upstream to receive body %q, got %q", expectedRequestBody, capturedBody) + } + if recorder.Code != http.StatusNotFound { + t.Fatalf("unexpected status written to client: got %d want %d", recorder.Code, http.StatusNotFound) + } + if recorder.Body.String() != upstreamBody { + t.Fatalf("unexpected body written to client: got %q want %q", recorder.Body.String(), upstreamBody) + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("failed to read response body: %v", err) + } + if string(bodyBytes) != upstreamBody { + t.Fatalf("expected preserved response body %q, got %q", upstreamBody, string(bodyBytes)) + } +}