diff --git a/CONFIG.md b/CONFIG.md index 6ad38c8..437772f 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -75,6 +75,13 @@ Default: false In the future, this will be the only possible behavior. +##### ALLOW_CONCURRENT_REQUESTS +Allows nirn to perform concurrent requests to Discord endpoints, instead of one at a time per queue. This might have the unintended side effect of a small increase in 429's if Discord updates buckets on the fly. + +If you do not care about throughput or do not make a lot of requests to the same endpoint that might take Discord a while to answer, then it would be fine to keep this off. + +Default: true + ## Unstable env vars Collection of env vars that may be removed at any time, mainly used for Discord introducing new behaviour on their edge api versions diff --git a/Dockerfile b/Dockerfile index c35da2e..f32356a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,17 @@ -FROM golang:alpine as app-builder +FROM --platform=$BUILDPLATFORM golang:alpine as app-builder WORKDIR /go/src/app COPY . . -RUN CGO_ENABLED=0 go install -ldflags '-extldflags "-static"' -tags timetzdata -buildvcs=false + +ARG TARGETOS +ARG TARGETARCH + +RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags '-extldflags "-static"' -tags timetzdata -buildvcs=false FROM scratch -COPY --from=app-builder /go/bin/nirn-proxy /nirn-proxy +COPY --from=app-builder /go/src/app/nirn-proxy /nirn-proxy # the tls certificates: # NB: this pulls directly from the upstream image, which already has ca-certificates: COPY --from=alpine:latest /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ EXPOSE 9000 EXPOSE 8080 -ENTRYPOINT ["/nirn-proxy"] \ No newline at end of file +ENTRYPOINT ["/nirn-proxy"] diff --git a/README.md b/README.md index f7d8ec9..dbc8a60 100644 --- a/README.md +++ b/README.md @@ -24,26 +24,27 @@ The proxy sits between the client and discord. Instead of pointing to discord.co Configuration options are -| Variable | Value | Default | -|-----------------|---------------------------------------------|---------| -| LOG_LEVEL | panic, fatal, error, warn, info, debug, trace | info | -| PORT | number | 8080 | -| METRICS_PORT | number | 9000 | -| ENABLE_METRICS | boolean | true | -| ENABLE_PPROF | boolean | false | -| BUFFER_SIZE | number | 50 | -| OUTBOUND_IP | string | "" | -| BIND_IP | string | 0.0.0.0 | -| REQUEST_TIMEOUT | number (milliseconds) | 5000 | -| CLUSTER_PORT | number | 7946 | -| CLUSTER_MEMBERS | string list (comma separated) | "" | -| CLUSTER_DNS | string | "" | -| MAX_BEARER_COUNT| number | 1024 | -| DISABLE_HTTP_2 | bool | true | -| BOT_RATELIMIT_OVERRIDES | string list (comma separated) | "" | -| DISABLE_GLOBAL_RATELIMIT_DETECTION | boolean | false | - -Information on each config var can be found [here](https://github.com/germanoeich/nirn-proxy/blob/main/CONFIG.md) +| Variable | Value | Default | +|------------------------------------|-----------------------------------------------|---------| +| LOG_LEVEL | panic, fatal, error, warn, info, debug, trace | info | +| PORT | number | 8080 | +| METRICS_PORT | number | 9000 | +| ENABLE_METRICS | boolean | true | +| ENABLE_PPROF | boolean | false | +| BUFFER_SIZE | number | 50 | +| OUTBOUND_IP | string | "" | +| BIND_IP | string | 0.0.0.0 | +| REQUEST_TIMEOUT | number (milliseconds) | 5000 | +| CLUSTER_PORT | number | 7946 | +| CLUSTER_MEMBERS | string list (comma separated) | "" | +| CLUSTER_DNS | string | "" | +| MAX_BEARER_COUNT | number | 1024 | +| DISABLE_HTTP_2 | bool | true | +| BOT_RATELIMIT_OVERRIDES | string list (comma separated) | "" | +| DISABLE_GLOBAL_RATELIMIT_DETECTION | boolean | false | +| ALLOW_CONCURRENT_REQUESTS | boolean | true | + +Information on each config var can be found [here](CONFIG.md) .env files are loaded if present @@ -101,14 +102,14 @@ This will vary depending on your usage, how many unique routes you see, etc. For ### Metrics / Health -| Key | Labels | Description | -|------------------------------------|----------------------------------------|------------------------------------------------------------| -|nirn_proxy_error | none | Counter for errors | -|nirn_proxy_requests | method, status, route, clientId | Histogram that keeps track of all request metrics | -|nirn_proxy_open_connections | route, method | Gauge for open client connections with the proxy | -|nirn_proxy_requests_routed_sent | none | Counter for requests routed to other nodes | -|nirn_proxy_requests_routed_received | none | Counter for requests received from other nodes | -|nirn_proxy_requests_routed_error | none | Counter for requests routed that failed | +| Key | Labels | Description | +|-------------------------------------|---------------------------------|---------------------------------------------------| +| nirn_proxy_error | none | Counter for errors | +| nirn_proxy_requests | method, status, route, clientId | Histogram that keeps track of all request metrics | +| nirn_proxy_open_connections | route, method | Gauge for open client connections with the proxy | +| nirn_proxy_requests_routed_sent | none | Counter for requests routed to other nodes | +| nirn_proxy_requests_routed_received | none | Counter for requests received from other nodes | +| nirn_proxy_requests_routed_error | none | Counter for requests routed that failed | Note: 429s can produce two status: 429 Too Many Requests or 429 Shared. The latter is only produced for requests that return with the x-ratelimit-scope header set to "shared", which means they don't count towards the cloudflare firewall limit and thus should not be used for alerts, etc. diff --git a/go.mod b/go.mod index d25e63e..c0d0148 100644 --- a/go.mod +++ b/go.mod @@ -1,36 +1,38 @@ module github.com/germanoeich/nirn-proxy -go 1.18 +go 1.25 require ( - github.com/Clever/leakybucket v1.2.0 - github.com/hashicorp/golang-lru v0.5.4 - github.com/hashicorp/memberlist v0.3.1 - github.com/joho/godotenv v1.4.0 - github.com/prometheus/client_golang v1.11.0 - github.com/sirupsen/logrus v1.8.1 + github.com/Clever/leakybucket v1.3.0 + github.com/hashicorp/golang-lru v1.0.2 + github.com/hashicorp/memberlist v0.5.4 + github.com/joho/godotenv v1.5.1 + github.com/prometheus/client_golang v1.23.2 + github.com/sirupsen/logrus v1.9.3 ) require ( - github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect + github.com/armon/go-metrics v0.4.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect - github.com/golang/protobuf v1.4.3 // indirect - github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-immutable-radix v1.0.0 // indirect - github.com/hashicorp/go-msgpack v0.5.3 // indirect - github.com/hashicorp/go-multierror v1.0.0 // indirect - github.com/hashicorp/go-sockaddr v1.0.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/miekg/dns v1.1.26 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.26.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-metrics v0.5.4 // indirect + github.com/hashicorp/go-msgpack/v2 v2.1.5 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/miekg/dns v1.1.70 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect - golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect - golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/protobuf v1.26.0-rc.1 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/mod v0.32.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/tools v0.41.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect ) diff --git a/go.sum b/go.sum index fd2c392..adfe9a3 100644 --- a/go.sum +++ b/go.sum @@ -1,32 +1,32 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/Clever/leakybucket v1.2.0 h1:tj9bHR6QS6c5Crszv+EP66NcbJxLabwZ90CUqNlFsSw= -github.com/Clever/leakybucket v1.2.0/go.mod h1:gZbI9EI3nNh9loJzrwobjtPUh3fuOT2Q6GgqtBHFuc4= +github.com/Clever/leakybucket v1.3.0 h1:GSj9YT5iTni8MCql6kqWCafX4AFOEytstreD4y3Z8Jc= +github.com/Clever/leakybucket v1.3.0/go.mod h1:oiP7sa8A5USud+zeGa5rkkxPYMFof65eNIik7ohMoPk= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/aws/aws-sdk-go v1.29.31/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/garyburd/redigo v1.3.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -38,61 +38,75 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY= +github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI= +github.com/hashicorp/go-msgpack/v2 v2.1.5 h1:Ue879bPnutj/hXfmUk6s/jtIK90XxgiUIcXRl656T44= +github.com/hashicorp/go-msgpack/v2 v2.1.5/go.mod h1:bjCsRXpZ7NsJdk45PoCQnzRGDaK8TKm5ZnDI/9y3J4M= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= +github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1vWZfM= -github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= -github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/memberlist v0.5.4 h1:40YY+3qq2tAUhZIMEK8kqusKZBBjdwJ3NUjvYkcxh74= +github.com/hashicorp/memberlist v0.5.4/go.mod h1:OgN6xiIo6RlHUWk+ALjP9e32xWCoQrsOCmHrWCm2MWA= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.70 h1:DZ4u2AV35VJxdD9Fo9fIWm119BsQL5cZU1cQ9s0LkqA= +github.com/miekg/dns v1.1.70/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -100,86 +114,92 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -187,15 +207,19 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/lib/bucket.go b/lib/bucket.go new file mode 100644 index 0000000..7edc17f --- /dev/null +++ b/lib/bucket.go @@ -0,0 +1,312 @@ +package lib + +import ( + "context" + "math" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +// isClose checks if two durations are within `diff` seconds of difference +func isClose(a, b float64, absTol float64) bool { + return math.Abs(a-b) <= absTol +} + +func calculateFixedWindow(resetAt, resetAfter float64) (time.Duration, time.Time) { + increaseAt := time.Unix(0, int64(resetAt*1_000_000_000)) + period := time.Duration(resetAfter*1_000) * time.Millisecond + + return period, increaseAt +} + +func calculateSlidingWindow(remaining, limit int64, resetAfter float64) (time.Duration, time.Time) { + // slidePeriod = resetAfter / (limit - remaining) + slidePeriod := time.Duration(math.Ceil((resetAfter/float64(limit-remaining))*1_000)) * time.Millisecond + increaseAt := time.Now().Add(slidePeriod) + + return slidePeriod, increaseAt +} + +func isFirstValidHeaders(remaining, limit int64) bool { + return remaining == limit-1 && limit != 1 +} + +// Bucket is a Discord bucket ratelimiter +type Bucket struct { + increaseAt time.Time + transitWaitChan chan interface{} + + // under stateLock + bucket string + remaining int64 + limit int64 + period time.Duration + resetAt time.Time + lastUpdatedAt time.Time + // under inTransitLock + inTransit int64 + + stateLock sync.Mutex + inTransitLock sync.Mutex + acquireLock sync.Mutex + + outOfSync bool + fixedWindow bool + typeChangeAllowed bool + closedChan chan struct{} +} + +func NewBucket(bucket string, remaining, limit int64, resetAt, resetAfter float64) *Bucket { + b := &Bucket{ + bucket: bucket, + remaining: remaining, + limit: limit, + outOfSync: false, + closedChan: make(chan struct{}, 1), + typeChangeAllowed: true, + lastUpdatedAt: time.Now(), + } + + if isFirstValidHeaders(remaining, limit) { + // We have the perfect condition for a sliding window, so assume that for now. + // Turning it into a fixed bucket later is preferable, as we might never get this chance again + b.period, b.increaseAt = calculateSlidingWindow(remaining, limit, resetAfter) + b.fixedWindow = false + b.resetAt = time.Now().Add(b.period * time.Duration(b.limit)) + } else { + // We can assume its a fixed bucket for now, and hope that in the future we will get + // the ideal condition + b.period, b.increaseAt = calculateFixedWindow(resetAt, resetAfter) + b.fixedWindow = true + b.resetAt = time.Unix(0, int64(resetAt*1_000_000_000)) + + if limit == 1 { + // Bucket is 100% a fixed bucket + b.typeChangeAllowed = false + } + } + + return b +} + +// Warning: this MUST be called from a locked state +// Warning: `now` must be the current time. Passing a present or past value is undefined behaviour +func (b *Bucket) isRatelimited(now time.Time) bool { + canIncrease := now.After(b.increaseAt) + canReset := now.After(b.resetAt) + + // inTransit check is performed to avoid deadlocks + if (canIncrease && !b.outOfSync && b.inTransit != 1) || canReset { + if b.fixedWindow { + // Fixed windows just reset the remaining back to the limit + b.remaining = b.limit + b.increaseAt = now.Add(b.period) + b.resetAt = b.increaseAt + b.outOfSync = true + } else if canReset { + // Sliding bucket being fully reset + b.remaining = b.limit + b.increaseAt = now.Add(b.period) + b.resetAt = now.Add(b.period * time.Duration(b.limit)) + b.outOfSync = true + } else { + // Slide window along + gain := int64(math.Floor((now.Sub(b.increaseAt).Seconds())/b.period.Seconds())) + 1 + + b.remaining = min(b.remaining+gain, b.limit) + + if b.remaining == b.limit { + // When a ratelimit resets, we will fall out of sync from the remote, so + // we want to prevent future sliding + b.increaseAt = now.Add(b.period) + b.resetAt = now.Add(b.period * time.Duration(b.limit)) + b.outOfSync = true + } else { + b.increaseAt = b.increaseAt.Add(b.period * time.Duration(gain)) + b.resetAt = b.resetAt.Add(b.period * time.Duration(gain)) + } + } + } + + return b.remaining <= 0 +} + +// Acquire will request a slot from the ratelimit or sleep until there is one available +func (b *Bucket) Acquire(ctx context.Context) error { + b.acquireLock.Lock() + defer b.acquireLock.Unlock() + if b.closedChan == nil { + return nil + } + + b.inTransitLock.Lock() + if b.inTransit >= b.limit { + // Buffer of 1 here to prevent deadlocks in a worst case scenario + b.transitWaitChan = make(chan interface{}, 1) + b.inTransitLock.Unlock() + select { + case <-b.closedChan: + break + case <-ctx.Done(): + b.inTransitLock.Lock() + if b.transitWaitChan != nil { + b.transitWaitChan = nil + } else { + // Return the slot that was given to us + b.inTransit-- + } + b.inTransitLock.Unlock() + return ctx.Err() + case <-b.transitWaitChan: + } + // We dont update inTransit because the + // slot was given to us by the goroutine + // that sent the message through transitWaitChan + } else { + b.inTransit++ + b.inTransitLock.Unlock() + } + + for { + b.stateLock.Lock() + now := time.Now() + if !b.isRatelimited(now) { + // b.lock will be unlocked after decrementing remaining + break + } + sleepDuration := b.increaseAt.Sub(now) + b.stateLock.Unlock() + + if sleepDuration > 0 { + logger.WithFields(logrus.Fields{ + "bucket": b.bucket, + "sleepDuration": sleepDuration, + }).Debug("backing off to avoid hitting ratelimits") + } else { + sleepDuration = time.Duration(0) + } + + select { + case <-b.closedChan: + break + case <-ctx.Done(): + b.Release() + return ctx.Err() + case <-time.After(sleepDuration): + } + } + + b.remaining-- + b.stateLock.Unlock() + return nil +} + +// Release returns the slot to the bucket +func (b *Bucket) Release() { + b.inTransitLock.Lock() + defer b.inTransitLock.Unlock() + + if b.transitWaitChan != nil && b.inTransit <= b.limit { + // We dont update inTransit here as we are giving + // our slot to the one that is waiting + b.transitWaitChan <- nil + b.transitWaitChan = nil + return + } + + if b.inTransit > 0 { + b.inTransit-- + } +} + +// Update updates the bucket with new ratelimit information +func (b *Bucket) Update(remaining, limit int64, resetAt, resetAfter float64, ratelimitHit bool) { + b.stateLock.Lock() + defer b.stateLock.Unlock() + + b.lastUpdatedAt = time.Now() + resetAtTime := time.Unix(0, int64(resetAt*1_000_000_000)) + + if ratelimitHit { + // During ratelimit avoidance, we will treat the bucket as fixed + // bucket and wait for it to fill up completely + b.increaseAt = resetAtTime + b.resetAt = resetAtTime + b.remaining = 0 + b.outOfSync = false + return + } + + firstValidHeaders := isFirstValidHeaders(remaining, limit) + + if b.typeChangeAllowed && !b.outOfSync && !firstValidHeaders && remaining > 0 { + b.typeChangeAllowed = false + resetAtEq := isClose(float64(b.resetAt.UnixMilli())/1_000, resetAt, 0.05) + + if resetAtEq { + logger.WithFields(logrus.Fields{ + "bucket": b.bucket, + "storedResetAt": b.resetAt, + "receivedResetAt": resetAtTime, + }).Debug("bucket detected to be a fixed bucket") + + if !b.fixedWindow { + b.fixedWindow = true + // Setting this here will have an effect below + b.outOfSync = true + } + + } else { + logger.WithFields(logrus.Fields{ + "bucket": b.bucket, + "storedResetAt": b.resetAt, + "receivedResetAt": resetAtTime, + }).Debug("bucket detected to be a sliding bucket") + + if b.fixedWindow { + b.fixedWindow = false + // Setting this here will have an effect below + b.outOfSync = true + } + } + } + + b.resetAt = resetAtTime + + if b.outOfSync || firstValidHeaders { + var period time.Duration + var increaseAt time.Time + + if b.fixedWindow { + period, increaseAt = calculateFixedWindow(resetAt, resetAfter) + } else { + period, increaseAt = calculateSlidingWindow(remaining, limit, resetAfter) + } + + b.outOfSync = false + + // Prevent both from decreasing + if b.increaseAt.Before(increaseAt) { + b.increaseAt = increaseAt + } + b.period = period + } +} + +// Close marks the bucket as closed and immediately returns from any and future Acquire calls. +// The bucket should not be used from this point onwards +func (b *Bucket) Close() { + if b.closedChan == nil { + return + } + + logger.WithFields(logrus.Fields{ + "bucket": b.bucket, + }).Debug("bucket closed") + + closedChan := b.closedChan + b.closedChan = nil + closedChan <- struct{}{} +} diff --git a/lib/bucketpath.go b/lib/bucketpath.go index 9a0062c..e02f405 100644 --- a/lib/bucketpath.go +++ b/lib/bucketpath.go @@ -8,11 +8,10 @@ import ( ) const ( - MajorUnknown = "unk" - MajorChannels = "channels" - MajorGuilds = "guilds" - MajorWebhooks = "webhooks" - MajorInvites = "invites" + MajorChannels = "channels" + MajorGuilds = "guilds" + MajorWebhooks = "webhooks" + MajorInvites = "invites" MajorInteractions = "interactions" ) @@ -39,7 +38,7 @@ func IsNumericInput(str string) bool { } func GetMetricsPath(route string) string { - route = GetOptimisticBucketPath(route, "") + route, _ = GetOptimisticBucketPath(route, "") var path = "" parts := strings.Split(route, "/") @@ -47,9 +46,11 @@ func GetMetricsPath(route string) string { return "/invite/!" } - for _, part := range parts { - if part == "" { continue } - if IsNumericInput(part) { + for idx, part := range parts { + if part == "" { + continue + } + if IsNumericInput(part) || (idx != 0 && parts[idx-1] == "activity-instances") { path += "/!" } else { path += "/" + part @@ -64,7 +65,37 @@ func GetMetricsPath(route string) string { return path } -func GetOptimisticBucketPath(url string, method string) string { +func majorParamHash(major string, parts ...string) uint64 { + hashStr := major + + for _, part := range parts { + hashStr += ":" + part + } + + return HashCRC64(hashStr) +} + +func tokenInfo(token string) string { + // aW50ZXJhY3Rpb246 is base64 for "interaction:" + if !strings.HasPrefix(token, "aW50ZXJhY3Rpb246") { + return "/!" + } + + // fix padding + if i := len(token) % 4; i != 0 { + token += strings.Repeat("=", 4-i) + } + + decodedPart, err := base64.StdEncoding.DecodeString(token) + if err != nil { + return "/unknown" + } + + interactionId := strings.Split(string(decodedPart), ":")[1] + return "/" + interactionId +} + +func GetOptimisticBucketPath(url string, method string) (string, uint64) { bucket := strings.Builder{} bucket.WriteByte('/') cleanUrl := strings.SplitN(url, "?", 1)[0] @@ -72,7 +103,7 @@ func GetOptimisticBucketPath(url string, method string) string { cleanUrl = strings.ReplaceAll(cleanUrl, "/api/v", "") l := len(cleanUrl) i := strings.Index(cleanUrl, "/") - cleanUrl = cleanUrl[i+1:l] + cleanUrl = cleanUrl[i+1 : l] } else { // Handle unversioned endpoints cleanUrl = strings.ReplaceAll(cleanUrl, "/api/", "") @@ -82,118 +113,100 @@ func GetOptimisticBucketPath(url string, method string) string { numParts := len(parts) if numParts <= 1 { - return cleanUrl + return cleanUrl, HashCRC64(cleanUrl) } - currMajor := MajorUnknown + currMajor := parts[0] + var majorParamsHash uint64 // ! stands for any replaceable id switch parts[0] { - case MajorChannels: - if numParts == 2 { - // Return the same bucket for all reqs to /channels/id - // In this case, the discord bucket is the same regardless of the id - bucket.WriteString(MajorChannels) - bucket.WriteString("/!") - return bucket.String() - } - bucket.WriteString(MajorChannels) - bucket.WriteByte('/') - bucket.WriteString(parts[1]) - currMajor = MajorChannels case MajorInvites: bucket.WriteString(MajorInvites) bucket.WriteString("/!") + currMajor = MajorInvites - case MajorGuilds: - // guilds/:guildId/channels share the same bucket for all guilds - if numParts == 3 && parts[2] == "channels" { - return "/" + MajorGuilds + "/!/channels" + majorParamsHash = majorParamHash(MajorInvites) + parts = parts[2:] + case MajorWebhooks: + bucket.WriteString(MajorWebhooks) + bucket.WriteByte('/') + bucket.WriteString(parts[1]) + + currMajor = MajorWebhooks + // Webhook tokens are optional, and they fall under different top level resources + if numParts > 2 && len(parts[2]) >= 64 { + // webhook_id + token + bucket.WriteString(tokenInfo(parts[2])) + majorParamsHash = majorParamHash(MajorWebhooks, parts[1], parts[2]) + parts = parts[3:] + } else { + // just webhook_id + majorParamsHash = majorParamHash(MajorWebhooks, parts[1]) + parts = parts[2:] } - fallthrough case MajorInteractions: if numParts == 4 && parts[3] == "callback" { - return "/" + MajorInteractions + "/" + parts[1] + "/!/callback" + majorParamsHash = majorParamHash(MajorInteractions, parts[1], parts[2]) + return "/" + MajorInteractions + "/" + parts[1] + "/!/callback", majorParamsHash } fallthrough - case MajorWebhooks: - fallthrough default: bucket.WriteString(parts[0]) bucket.WriteByte('/') bucket.WriteString(parts[1]) currMajor = parts[0] + majorParamsHash = majorParamHash(currMajor, parts[1]) + parts = parts[2:] } if numParts == 2 { - return bucket.String() + return bucket.String(), majorParamsHash } - // At this point, the major + id part is already accounted for + // At this point, the major + id part is already accounted for (and trimmed out of 'parts') // In this loop, we only need to strip all remaining snowflakes, emoji names and webhook tokens(optional) - for idx, part := range parts[2:] { + for idx, part := range parts { if IsSnowflake(part) { - // Custom rule for messages older than 14d - if currMajor == MajorChannels && parts[idx - 1] == "messages" && method == "DELETE" { + //Custom rule for message DELETES older than 14d and message PATCHES older than 1h + if currMajor == MajorChannels && idx == len(parts)-1 && parts[idx-1] == "messages" { createdAt, _ := GetSnowflakeCreatedAt(part) - if createdAt.Before(time.Now().Add(-1 * 14 * 24 * time.Hour)) { - bucket.WriteString("/!14dmsg") - } else if createdAt.After(time.Now().Add(-1 * 10 * time.Second)) { - bucket.WriteString("/!10smsg") - } - continue - } - bucket.WriteString("/!") - } else { - if currMajor == MajorChannels && part == "reactions" { - // reaction put/delete fall under a different bucket from other reaction endpoints - if method == "PUT" || method == "DELETE" { - bucket.WriteString("/reactions/!modify") - break - } - //All other reaction endpoints falls under the same bucket, so it's irrelevant if the user - //is passing userid, emoji, etc. - bucket.WriteString("/reactions/!/!") - //Reactions can only be followed by emoji/userid combo, since we don't care, break - break - } + diff := time.Since(createdAt) - // Strip webhook tokens, or extract interaction ID - if len(part) >= 64 { - // aW50ZXJhY3Rpb246 is base64 for "interaction:" - if !strings.HasPrefix(part, "aW50ZXJhY3Rpb246") { - bucket.WriteString("/!") + if method == "DELETE" && diff >= 14*24*time.Hour { + bucket.WriteString("/!14dmsg") + continue + } else if method == "PATCH" && diff >= 1*time.Hour { + bucket.WriteString("/!1hmsg") continue } - - var interactionId string - - // fix padding - if i := len(part) % 4; i != 0 { - part += strings.Repeat("=", 4-i) - } - - decodedPart, err := base64.StdEncoding.DecodeString(part) - if err != nil { - interactionId = "Unknown" - } else { - interactionId = strings.Split(string(decodedPart), ":")[1] - } - - bucket.WriteByte('/') - bucket.WriteString(interactionId) - continue } + bucket.WriteString("/!") + continue + } - // Strip webhook tokens and interaction tokens - if (currMajor == MajorWebhooks || currMajor == MajorInteractions) && len(part) >= 64 { - bucket.WriteString("/!") - continue + if currMajor == MajorChannels && part == "reactions" { + // reaction put/delete fall under a different bucket from other reaction endpoints + if method == "PUT" || method == "DELETE" { + bucket.WriteString("/reactions/!modify") + break } - bucket.WriteByte('/') - bucket.WriteString(part) + //All other reaction endpoints falls under the same bucket, so it's irrelevant if the user + //is passing userid, emoji, etc. + bucket.WriteString("/reactions/!/!") + //Reactions can only be followed by emoji/userid combo, since we don't care, break + break } + + // Strip webhook tokens and interaction tokens + if (currMajor == MajorWebhooks || currMajor == MajorInteractions) && len(part) >= 64 { + bucket.WriteString("/!") + continue + } + + bucket.WriteByte('/') + bucket.WriteString(part) } - return bucket.String() -} \ No newline at end of file + return bucket.String(), majorParamsHash +} diff --git a/lib/bucketpath_test.go b/lib/bucketpath_test.go index dee751d..f886f22 100644 --- a/lib/bucketpath_test.go +++ b/lib/bucketpath_test.go @@ -3,6 +3,8 @@ package lib import ( "fmt" "testing" + "testing/synctest" + "time" ) func TestPaths(t *testing.T) { @@ -12,7 +14,7 @@ func TestPaths(t *testing.T) { // Guild Major {"/api/v9/guilds/103039963636301824", "GET", "/guilds/103039963636301824"}, // Channel major - {"/api/v8/channels/203039963636301824", "GET", "/channels/!"}, + {"/api/v8/channels/203039963636301824", "GET", "/channels/203039963636301824"}, {"/api/v7/channels/203039963636301824/pins", "GET", "/channels/203039963636301824/pins"}, {"/api/v6/channels/872712139712913438/messages/872712150509047809/reactions/%F0%9F%98%8B", "GET", "/channels/872712139712913438/messages/!/reactions/!/!"}, {"/api/v10/channels/872712139712913438/messages/872712150509047809/reactions/PandaOhShit:863985751205085195", "GET", "/channels/872712139712913438/messages/!/reactions/!/!"}, @@ -26,32 +28,54 @@ func TestPaths(t *testing.T) { // Invites major {"/api/v9/invites/dyno", "GET", "/invites/!"}, // Interactions major - {"/api/v9/interactions/203039963636301824/aW50ZXJhY3Rpb246ODg3NTU5MDA01AY4NTUxNDU0OnZwS3QycDhvREk2aVF3U1BqN2prcXBkRmNqNlp4VEhGRjZvSVlXSGh4WG4yb3l6Z3B6NTBPNVc3OHphV05OULLMOHBMa2RTZmVKd3lzVDA2b2h3OTUxaFJ4QlN0dkxXallPcmhnSHNJb0tSV0M5ZzY1NkN4VGRvemFOSHY4b05c/callback", "GET", "/interactions/203039963636301824/!/callback"}, + {"/api/v9/interactions/203039963636301824/aW50ZXJhY3Rpb246ODg3NTU5MDA01AY4NTUxNDU0OnZwS3QycDhvREk2aVF3U1BqN2prcXBkRmNqNlp4VEhGRjZvSVlXSGh4WG4yb3l6Z3B6NTBPNVc3OHphV05OULLMOHBMa2RTZmVKd3lzVDA2b2h3OTUxaFJ4QlN0dkxXallPcmhnSHNJb0tSV0M5ZzY1NkN4VGRvemFOSHY4b05c/callback", "GET", "/interactions/!/!/callback"}, + // Make sure we dont break future fictional future /interactions + {"/api/v9/interactions/203039963636301824/get-author", "GET", "/interactions/203039963636301824/get-author"}, + {"/api/v9/interactions/203039963636301824/aW50ZXJhY3Rpb246ODg3NTU5MDA01AY4NTUxNDU0OnZwS3QycDhvREk2aVF3U1BqN2prcXBkRmNqNlp4VEhGRjZvSVlXSGh4WG4yb3l6Z3B6NTBPNVc3OHphV05OULLMOHBMa2RTZmVKd3lzVDA2b2h3OTUxaFJ4QlN0dkxXallPcmhnSHNJb0tSV0M5ZzY1NkN4VGRvemFOSHY4b05c/fictional", "GET", "/interactions/203039963636301824/!/fictional"}, // Interaction followup webhooks {"/api/v10/webhooks/203039963636301824/aW50ZXJhY3Rpb246MTEwMzA0OTQyMDkzMDU2ODMyMjpOZUllWHdNU2J4RXBFMHVYRjBpU0pHMDdEb3BhM3ZlYklBODlMUmtlUXlRbzlpZzYyTnpLU0dqdWlyVlBvZnBSUlJHbUJHYlJ0N29MbE9KQUJVTFk4bTR4UzFtZEpEeXJyY0hBUERmTEhKVE9wRkNzU1FFWUkwTnlpWFY2WHdrRg/messages/@original", "POST", "/webhooks/203039963636301824/1103049420930568322/messages/@original"}, // No known major {"/api/v9/invalid/203039963636301824", "GET", "/invalid/203039963636301824"}, {"/api/v9/invalid/203039963636301824/route/203039963636301824", "GET", "/invalid/203039963636301824/route/!"}, - //Special case for /guilds/:id/channels - {"/api/v9/guilds/203039963636301824/channels", "GET", "/guilds/!/channels"}, // Wierd routes {"/api/v9/guilds/templates/203039963636301824", "GET", "/guilds/templates/!"}, // Unversioned routes {"/api/webhooks/203039963636301824/VSOzAqY1OZFF5WJVtbIzFtmjGupk-84Hn0A_ZzToF_CHsPIeCk0Q9Uok_mjxR0dNtApI", "POST", "/webhooks/203039963636301824/!"}, - {"/api/interactions/203039963636301824/aW50ZXJhY3Rpb246ODg3NTU5MDA01AY4NTUxNDU0OnZwS3QycDhvREk2aVF3U1BqN2prcXBkRmNqNlp4VEhGRjZvSVlXSGh4WG4yb3l6Z3B6NTBPNVc3OHphV05OULLMOHBMa2RTZmVKd3lzVDA2b2h3OTUxaFJ4QlN0dkxXallPcmhnSHNJb0tSV0M5ZzY1NkN4VGRvemFOSHY4b05c/callback", "GET", "/interactions/203039963636301824/!/callback"}, + {"/api/interactions/203039963636301824/aW50ZXJhY3Rpb246ODg3NTU5MDA01AY4NTUxNDU0OnZwS3QycDhvREk2aVF3U1BqN2prcXBkRmNqNlp4VEhGRjZvSVlXSGh4WG4yb3l6Z3B6NTBPNVc3OHphV05OULLMOHBMa2RTZmVKd3lzVDA2b2h3OTUxaFJ4QlN0dkxXallPcmhnSHNJb0tSV0M5ZzY1NkN4VGRvemFOSHY4b05c/callback", "GET", "/interactions/!/!/callback"}, {"/api/channels/872712139712913438/messages/872712150509047809/reactions/PandaOhShit:863985751205085195", "GET", "/channels/872712139712913438/messages/!/reactions/!/!"}, {"/api/invites/dyno", "GET", "/invites/!"}, // Application commands {"/api/v9/applications/203039963636301824/commands", "GET", "/applications/203039963636301824/commands"}, {"/api/v9/applications/203039963636301824/commands/203039963636301824", "GET", "/applications/203039963636301824/commands/!"}, + // Message delete has multiple buckets + // 10 seconds after 2016-01-01 00:00:00 + {"/api/v9/channels/1412822759695974551/messages/132271529014534145", "DELETE", "/channels/1412822759695974551/messages/!"}, + // 1 hour before 2016-01-01 00:00:00 + {"/api/v9/channels/1412822759695974551/messages/132256471463174144", "DELETE", "/channels/1412822759695974551/messages/!"}, + // 14 days before 2016-01-01 00:00:00 + {"/api/v9/channels/1412822759695974551/messages/127198140839174145", "DELETE", "/channels/1412822759695974551/messages/!14dmsg"}, + // Message patch has multiple buckets + // 10 seconds after 2016-01-01 00:00:00 + {"/api/v9/channels/1412822759695974551/messages/132271529014534145", "PATCH", "/channels/1412822759695974551/messages/!"}, + // 1 hour before 2016-01-01 00:00:00 + {"/api/v9/channels/1412822759695974551/messages/132256471463174144", "PATCH", "/channels/1412822759695974551/messages/!1hmsg"}, + // 14 days before 2016-01-01 00:00:00 + {"/api/v9/channels/1412822759695974551/messages/127198140839174145", "PATCH", "/channels/1412822759695974551/messages/!1hmsg"}, } for _, tt := range tests { testname := fmt.Sprintf("%s-%s", tt.method, tt.path) t.Run(testname, func(t *testing.T) { - bucket := GetOptimisticBucketPath(tt.path, tt.method) - if bucket != tt.want { - t.Errorf("Expected %s but got %s", tt.want, bucket) - } + // Time will start at UTC 2000-01-01 00:00:00 + synctest.Test(t, func(t *testing.T) { + // 16 years in hours + time.Sleep(140256 * time.Hour) + + // Time will always be midnight UTC 2016-01-01 00:00:00 + bucket, _ := GetOptimisticBucketPath(tt.path, tt.method) + if bucket != tt.want { + t.Errorf("Expected %s but got %s", tt.want, bucket) + } + }) }) } } diff --git a/lib/discord.go b/lib/discord.go index cf99f86..5cb7ca2 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -1,13 +1,13 @@ package lib import ( + "bytes" "context" "crypto/tls" "encoding/json" "errors" "github.com/sirupsen/logrus" "io" - "io/ioutil" "math" "net" "net/http" @@ -23,6 +23,7 @@ var contextTimeout time.Duration var globalOverrideMap = make(map[string]uint) var disableRestLimitDetection = false +var allowConcurrentRequests = false type BotGatewayResponse struct { SessionStartLimit map[string]int `json:"session_start_limit"` @@ -88,7 +89,7 @@ func createTransport(ip string, disableHttp2 bool) http.RoundTripper { } func parseGlobalOverrides(overrides string) { - // Format: ":,: + // Format: ":,:" if overrides == "" { return @@ -111,7 +112,7 @@ func parseGlobalOverrides(overrides string) { } } -func ConfigureDiscordHTTPClient(ip string, timeout time.Duration, disableHttp2 bool, globalOverrides string, disableRestDetection bool) { +func ConfigureDiscordHTTPClient(ip string, timeout time.Duration, globalOverrides string, disableHttp2, disableRestDetection, allowConcurrent bool) { transport := createTransport(ip, disableHttp2) client = &http.Client{ Transport: transport, @@ -121,6 +122,7 @@ func ConfigureDiscordHTTPClient(ip string, timeout time.Duration, disableHttp2 b contextTimeout = timeout disableRestLimitDetection = disableRestDetection + allowConcurrentRequests = allowConcurrent parseGlobalOverrides(globalOverrides) } @@ -161,7 +163,7 @@ func GetBotGlobalLimit(token string, user *BotUserResponse) (uint, error) { return 0, errors.New("500 on gateway/bot") } - body, _ := ioutil.ReadAll(bot.Body) + body, _ := io.ReadAll(bot.Body) var s BotGatewayResponse @@ -200,7 +202,7 @@ func GetBotUser(token string) (*BotUserResponse, error) { return nil, errors.New("500 on users/@me") } - body, _ := ioutil.ReadAll(bot.Body) + body, _ := io.ReadAll(bot.Body) var s BotUserResponse @@ -212,7 +214,7 @@ func GetBotUser(token string) (*BotUserResponse, error) { return &s, nil } -func doDiscordReq(ctx context.Context, path string, method string, body io.ReadCloser, header http.Header, query string) (*http.Response, error) { +func doDiscordReq(ctx context.Context, path string, method string, body io.Reader, header http.Header, query string) (*http.Response, error) { discordReq, err := http.NewRequestWithContext(ctx, method, "https://discord.com"+path+"?"+query, body) if err != nil { return nil, err @@ -251,10 +253,10 @@ func ProcessRequest(ctx context.Context, item *QueueItem) (*http.Response, error ctx, cancel := context.WithTimeout(ctx, contextTimeout) defer cancel() - discordResp, err := doDiscordReq(ctx, req.URL.Path, req.Method, req.Body, req.Header.Clone(), req.URL.RawQuery) + discordResp, err := doDiscordReq(ctx, req.URL.Path, req.Method, bytes.NewReader(item.ReqBody), req.Header.Clone(), req.URL.RawQuery) if err != nil { - if ctx.Err() == context.DeadlineExceeded { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { res.WriteHeader(408) } else { res.WriteHeader(500) diff --git a/lib/distributed_global.go b/lib/distributed_global.go index b1610cc..d0cf56c 100644 --- a/lib/distributed_global.go +++ b/lib/distributed_global.go @@ -13,15 +13,15 @@ import ( ) type ClusterGlobalRateLimiter struct { - sync.RWMutex globalBucketsMap map[uint64]*leakybucket.Bucket - memStorage *memory.Storage + memStorage *memory.Storage + sync.RWMutex } func NewClusterGlobalRateLimiter() *ClusterGlobalRateLimiter { memStorage := memory.New() return &ClusterGlobalRateLimiter{ - memStorage: memStorage, + memStorage: memStorage, globalBucketsMap: make(map[uint64]*leakybucket.Bucket), } } @@ -53,7 +53,7 @@ func (c *ClusterGlobalRateLimiter) getOrCreate(botHash uint64, botLimit uint) *l return b } - globalBucket, _ := c.memStorage.Create(strconv.FormatUint(botHash, 10), botLimit, 1 * time.Second) + globalBucket, _ := c.memStorage.Create(strconv.FormatUint(botHash, 10), botLimit, 1*time.Second) c.globalBucketsMap[botHash] = &globalBucket c.Unlock() return &globalBucket @@ -61,10 +61,8 @@ func (c *ClusterGlobalRateLimiter) getOrCreate(botHash uint64, botLimit uint) *l return b } } - - func (c *ClusterGlobalRateLimiter) FireGlobalRequest(ctx context.Context, addr string, botHash uint64, botLimit uint) error { - globalReq, err := http.NewRequestWithContext(ctx, "GET", "http://" + addr + "/nirn/global", nil) + globalReq, err := http.NewRequestWithContext(ctx, "GET", "http://"+addr+"/nirn/global", nil) if err != nil { return err } @@ -85,4 +83,4 @@ func (c *ClusterGlobalRateLimiter) FireGlobalRequest(ctx context.Context, addr s } return nil -} \ No newline at end of file +} diff --git a/lib/http.go b/lib/http.go index eaa1ecd..939f884 100644 --- a/lib/http.go +++ b/lib/http.go @@ -1,7 +1,7 @@ package lib import ( - "io/ioutil" + "io" "net/http" "strings" ) @@ -20,7 +20,7 @@ func copyHeader(dst, src http.Header) { func CopyResponseToResponseWriter(resp *http.Response, respWriter *http.ResponseWriter) error { writer := *respWriter - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { writer.WriteHeader(500) _, _ = writer.Write([]byte(err.Error())) @@ -35,4 +35,4 @@ func CopyResponseToResponseWriter(resp *http.Response, respWriter *http.Response return err } return nil -} \ No newline at end of file +} diff --git a/lib/queue.go b/lib/queue.go index 203383a..a9f1b15 100644 --- a/lib/queue.go +++ b/lib/queue.go @@ -3,46 +3,103 @@ package lib import ( "context" "errors" - "github.com/Clever/leakybucket" - "github.com/Clever/leakybucket/memory" - "github.com/sirupsen/logrus" + "io" "net/http" + "slices" "strconv" "strings" "sync" "sync/atomic" "time" + + "github.com/Clever/leakybucket" + "github.com/Clever/leakybucket/memory" + "github.com/sirupsen/logrus" ) +// A pool of bucketsContextManager +var bucketsContextManagerPool = sync.Pool{ + New: func() interface{} { + return &bucketsContextManager{ + buckets: make([]*Bucket, 0, 1), + } + }, +} + +type bucketsContextManager struct { + buckets []*Bucket +} + +func (b *bucketsContextManager) Acquire(ctx context.Context) error { + // We count till what position we reach instead of using a slice to prevent allocations + var acquiredBucketsCount int + var err error + + for _, bucket := range b.buckets { + err = bucket.Acquire(ctx) + if err != nil { + break + } + + acquiredBucketsCount++ + } + + if err != nil { + // Make sure we release all the buckets we have already acquired before the error + for idx, bucket := range b.buckets { + if idx >= acquiredBucketsCount { + break + } + + bucket.Release() + } + } + + return err +} + +func (b *bucketsContextManager) Release() { + for _, bucket := range b.buckets { + bucket.Release() + } +} + +type ItemProcessFunction func(ctx context.Context, item *QueueItem) (*http.Response, error) + type QueueItem struct { Req *http.Request Res *http.ResponseWriter doneChan chan *http.Response errChan chan error + ReqBody []byte } type QueueChannel struct { - ch chan *QueueItem - lastUsed time.Time + lastUsed time.Time + ch chan *QueueItem + lockerFun func(item *QueueItem) + buckets []string + sync.Mutex } type RequestQueue struct { - sync.RWMutex + globalBucket leakybucket.Bucket globalLockedUntil *int64 // bucket path hash as key - queues map[uint64]*QueueChannel - processor func(ctx context.Context, item *QueueItem) (*http.Response, error) - globalBucket leakybucket.Bucket - // bufferSize Defines the size of the request channel buffer for each bucket - bufferSize int + queues map[uint64]*QueueChannel + buckets map[string]*Bucket + processor ItemProcessFunction user *BotUserResponse - identifier string isTokenInvalid *int64 - botLimit uint - queueType QueueType + identifier string + // bufferSize Defines the size of the request channel buffer for each bucket + bufferSize int + botLimit uint + queueType QueueType + sync.Mutex } -func NewRequestQueue(processor func(ctx context.Context, item *QueueItem) (*http.Response, error), token string, bufferSize int) (*RequestQueue, error) { +func NewRequestQueue(processor ItemProcessFunction, token string, bufferSize int) (*RequestQueue, error) { queueType := NoAuth var user *BotUserResponse var err error @@ -65,6 +122,7 @@ func NewRequestQueue(processor func(ctx context.Context, item *QueueItem) (*http *invalid = 999 return &RequestQueue{ queues: make(map[uint64]*QueueChannel), + buckets: make(map[string]*Bucket), processor: processor, globalBucket: globalBucket, globalLockedUntil: new(int64), @@ -91,6 +149,7 @@ func NewRequestQueue(processor func(ctx context.Context, item *QueueItem) (*http ret := &RequestQueue{ queues: make(map[uint64]*QueueChannel), + buckets: make(map[string]*Bucket), processor: processor, globalBucket: globalBucket, globalLockedUntil: new(int64), @@ -122,10 +181,10 @@ func (q *RequestQueue) destroy() { } } -func (q *RequestQueue) sweep() { +func (q *RequestQueue) sweepQueues() { q.Lock() defer q.Unlock() - logger.Info("Sweep start") + logger.Info("Queues sweep start") sweptEntries := 0 for key, val := range q.queues { if time.Since(val.lastUsed) > 10*time.Minute { @@ -134,14 +193,37 @@ func (q *RequestQueue) sweep() { sweptEntries++ } } - logger.WithFields(logrus.Fields{"sweptEntries": sweptEntries}).Info("Finished sweep") + logger.WithFields(logrus.Fields{"sweptEntries": sweptEntries}).Info("Finished queues sweep") +} + +func (q *RequestQueue) sweepBuckets() { + q.Lock() + defer q.Unlock() + logger.Debug("Buckets sweep start") + sweptEntries := 0 + for key, val := range q.buckets { + // This is technically a data race, but we are looking for buckets that are insanely + // unused, so we can afford the data race + if time.Since(val.lastUpdatedAt) > 1*time.Minute { + delete(q.buckets, key) + val.Close() + sweptEntries++ + } + } + logger.WithFields(logrus.Fields{"sweptEntries": sweptEntries}).Debug("Finished buckets sweep") } func (q *RequestQueue) tickSweep() { t := time.NewTicker(5 * time.Minute) - - for range t.C { - q.sweep() + t2 := time.NewTicker(30 * time.Second) + + for { + select { + case <-t.C: + q.sweepQueues() + case <-t2.C: + q.sweepBuckets() + } } } @@ -156,19 +238,19 @@ func safeSend(queue *QueueChannel, value *QueueItem) { queue.ch <- value } -func (q *RequestQueue) Queue(req *http.Request, res *http.ResponseWriter, path string, pathHash uint64) error { +func (q *RequestQueue) Queue(req *http.Request, res *http.ResponseWriter, path string, pathHash, majorBucketHash uint64) error { logger.WithFields(logrus.Fields{ "bucket": path, "path": req.URL.Path, "method": req.Method, }).Trace("Inbound request") - ch := q.getQueueChannel(path, pathHash) + ch := q.getQueueChannel(path, pathHash, majorBucketHash) doneChan := make(chan *http.Response) errChan := make(chan error) - safeSend(ch, &QueueItem{req, res, doneChan, errChan}) + safeSend(ch, &QueueItem{Req: req, Res: res, errChan: errChan, doneChan: doneChan}) select { case <-doneChan: @@ -178,7 +260,7 @@ func (q *RequestQueue) Queue(req *http.Request, res *http.ResponseWriter, path s } } -func (q *RequestQueue) getQueueChannel(path string, pathHash uint64) *QueueChannel { +func (q *RequestQueue) getQueueChannel(path string, pathHash, majorBucketHash uint64) *QueueChannel { t := time.Now() q.Lock() defer q.Unlock() @@ -186,65 +268,75 @@ func (q *RequestQueue) getQueueChannel(path string, pathHash uint64) *QueueChann if !ok { ch = &QueueChannel{ ch: make(chan *QueueItem, q.bufferSize), + buckets: make([]string, 0, 1), lastUsed: t, } q.queues[pathHash] = ch // It's important that we only have 1 goroutine per channel - go q.subscribe(ch, path, pathHash) + go q.subscribe(ch, path, pathHash, majorBucketHash) } else { ch.lastUsed = t } return ch } -func parseHeaders(headers *http.Header, preferRetryAfter bool) (int64, int64, time.Duration, bool, error) { +func parseHeaders(headers *http.Header) (string, int64, int64, float64, float64, string, error) { if headers == nil { - return 0, 0, 0, false, errors.New("null headers") + return "", 0, 0, 0, 0, "", errors.New("null headers") } + bucket := headers.Get("x-ratelimit-bucket") limit := headers.Get("x-ratelimit-limit") remaining := headers.Get("x-ratelimit-remaining") + resetAt := headers.Get("x-ratelimit-reset") resetAfter := headers.Get("x-ratelimit-reset-after") retryAfter := headers.Get("retry-after") - if resetAfter == "" || (preferRetryAfter && retryAfter != "") { + scope := headers.Get("x-ratelimit-scope") + + if scope == "" { + scope = "user" + } + + if resetAfter == "" || (scope != "user" && retryAfter != "") { // Globals return no x-ratelimit-reset-after headers, shared ratelimits have a wrong reset-after // this is the best option without parsing the body resetAfter = headers.Get("retry-after") } - isGlobal := headers.Get("x-ratelimit-global") == "true" - var resetParsed float64 - var reset time.Duration = 0 var err error + + var resetAfterParsed float64 = 0 if resetAfter != "" { - resetParsed, err = strconv.ParseFloat(resetAfter, 64) + resetAfterParsed, err = strconv.ParseFloat(resetAfter, 64) if err != nil { - return 0, 0, 0, false, err + return "", 0, 0, 0, 0, "", err } - - // Convert to MS instead of seconds to preserve decimal precision - reset = time.Duration(int(resetParsed*1000)) * time.Millisecond } - if isGlobal { - return 0, 0, reset, isGlobal, nil + if scope == "global" { + return bucket, 0, 0, resetAfterParsed, 0, scope, nil } if limit == "" { - return 0, 0, reset, false, nil + return "", 0, 0, resetAfterParsed, 0, scope, nil } limitParsed, err := strconv.ParseInt(limit, 10, 32) if err != nil { - return 0, 0, 0, false, err + return "", 0, 0, 0, 0, "", err } remainingParsed, err := strconv.ParseInt(remaining, 10, 32) if err != nil { - return 0, 0, 0, false, err + return "", 0, 0, 0, 0, "", err } - return limitParsed, remainingParsed, reset, isGlobal, nil + resetAtParsed, err := strconv.ParseFloat(resetAt, 64) + if err != nil { + return "", 0, 0, 0, 0, "", err + } + + return bucket, remainingParsed, limitParsed, resetAfterParsed, resetAtParsed, scope, nil } func return404webhook(item *QueueItem) { @@ -254,10 +346,9 @@ func return404webhook(item *QueueItem) { _, err := res.Write([]byte(body)) if err != nil { item.errChan <- err - return + } else { + item.doneChan <- nil } - item.doneChan <- nil - } func return401(item *QueueItem) { @@ -282,106 +373,236 @@ func isInteraction(url string) bool { return false } -func (q *RequestQueue) subscribe(ch *QueueChannel, path string, pathHash uint64) { - // This function has 1 goroutine for each bucket path - // Locking here is not needed +func (q *RequestQueue) getBucketsContextManager(ch *QueueChannel) *bucketsContextManager { + q.Lock() + defer q.Unlock() + ch.Lock() + defer ch.Unlock() - //Only used for logging - var prevRem int64 = 0 - var prevReset time.Duration = 0 + if len(ch.buckets) == 0 { + return nil + } - // Fail fast path for webhook 404s - var ret404 = false - for item := range ch.ch { - ctx := context.WithValue(item.Req.Context(), "identifier", q.identifier) - if ret404 { - return404webhook(item) - continue - } + contextManager := bucketsContextManagerPool.Get().(*bucketsContextManager) + contextManager.buckets = contextManager.buckets[:0] - if atomic.LoadInt64(q.isTokenInvalid) > 0 { - return401(item) + for idx := 0; idx < len(ch.buckets); { + bucket, ok := q.buckets[ch.buckets[idx]] + if ok { + contextManager.buckets = append(contextManager.buckets, bucket) + idx++ continue } - resp, err := q.processor(ctx, item) - if err != nil { - item.errChan <- err - continue - } + // The bucket no longer exists, so remove it from the channel slice + ch.buckets = append(ch.buckets[:idx], ch.buckets[idx+1:]...) + } - scope := resp.Header.Get("x-ratelimit-scope") + if len(contextManager.buckets) == 0 { + bucketsContextManagerPool.Put(contextManager) + return nil + } - _, remaining, resetAfter, isGlobal, err := parseHeaders(&resp.Header, scope != "user") + return contextManager +} - if isGlobal { - //Lock global - sw := atomic.CompareAndSwapInt64(q.globalLockedUntil, 0, time.Now().Add(resetAfter).UnixNano()) - if sw { - logger.WithFields(logrus.Fields{ - "until": time.Now().Add(resetAfter), - "resetAfter": resetAfter, - }).Warn("Global reached, locking") - } - } +func (q *RequestQueue) doRequest(ctx context.Context, item *QueueItem, ch *QueueChannel, buckets *bucketsContextManager, path, pathHash, topBucketHash string) { + if buckets != nil { + defer func() { + buckets.Release() + bucketsContextManagerPool.Put(buckets) + }() + } - if err != nil { - item.errChan <- err - continue - } - item.doneChan <- resp + resp, err := q.processor(ctx, item) + if err != nil { + item.errChan <- err + return + } + + bucketHash, remaining, limit, resetAfter, resetAt, scope, err := parseHeaders(&resp.Header) + if err != nil { + item.errChan <- err + return + } + + item.doneChan <- resp - if resp.StatusCode == 429 && scope != "shared" { + if scope == "global" { + // Lock global + resetAfterDuration := time.Duration(resetAfter*1_000) * time.Millisecond + sw := atomic.CompareAndSwapInt64(q.globalLockedUntil, 0, time.Now().Add(resetAfterDuration).UnixNano()) + if sw { logger.WithFields(logrus.Fields{ - "prevRemaining": prevRem, - "prevResetAfter": prevReset, - "remaining": remaining, - "resetAfter": resetAfter, - "bucket": path, - "route": item.Req.URL.String(), - "method": item.Req.Method, - "isGlobal": isGlobal, - "pathHash": pathHash, - // TODO: Remove this when 429s are not a problem anymore - "discordBucket": resp.Header.Get("x-ratelimit-bucket"), - "ratelimitScope": resp.Header.Get("x-ratelimit-scope"), - }).Warn("Unexpected 429") + "until": time.Now().Add(resetAfterDuration), + "resetAfter": resetAfterDuration, + }).Warn("Global reached, locking") } + return + } - if resp.StatusCode == 404 && strings.HasPrefix(path, "/webhooks/") && !isInteraction(item.Req.URL.String()) { - logger.WithFields(logrus.Fields{ - "bucket": path, - "route": item.Req.URL.String(), - "method": item.Req.Method, - }).Info("Setting fail fast 404 for webhook") - ret404 = true + ratelimitHit := resp.StatusCode == 429 + multiBucket := false + + if bucketHash != "" || ratelimitHit { + if bucketHash == "" { + // We might have hit a Cloudflare 429, so we create a special bucket for that + bucketHash = "route:" + pathHash + } else { + bucketHash = bucketHash + ":" + topBucketHash + ":" + scope } - if resp.StatusCode == 401 && !isInteraction(item.Req.URL.String()) && q.queueType != NoAuth { - // Permanently lock this queue + q.Lock() + bucket, ok := q.buckets[bucketHash] + if !ok { logger.WithFields(logrus.Fields{ - "bucket": path, - "route": item.Req.URL.String(), + "bucket": bucketHash, + "remaining": remaining, + "limit": limit, + "resetAt": resetAt, + "resetAfter": resetAfter, + "identifier": q.identifier, + "path": path, "method": item.Req.Method, + }).Debug("creating new bucket") + + q.buckets[bucketHash] = NewBucket(bucketHash, remaining, limit, resetAt, resetAfter) + } else { + logger.WithFields(logrus.Fields{ + "bucket": bucketHash, + "remaining": remaining, + "limit": limit, + "resetAt": resetAt, + "resetAfter": resetAfter, "identifier": q.identifier, - "status": resp.StatusCode, - }).Error("Received 401 during normal operation, assuming token is invalidated, locking bucket permanently") + "path": path, + "method": item.Req.Method, + }).Debug("updating existing bucket") + + bucket.Update(remaining, limit, resetAt, resetAfter, ratelimitHit) + } + q.Unlock() + + ch.Lock() + if !slices.Contains(ch.buckets, bucketHash) { + logger.WithFields(logrus.Fields{ + "bucket": bucketHash, + "identifier": q.identifier, + "path": path, + "method": item.Req.Method, + "additionalBuckets": ch.buckets, + }).Debug("linking new bucket to route") + + ch.buckets = append(ch.buckets, bucketHash) + } + multiBucket = len(ch.buckets) > 1 + ch.Unlock() - if EnvGet("DISABLE_401_LOCK", "false") != "true" { - atomic.StoreInt64(q.isTokenInvalid, 999) + } + + if ratelimitHit && scope != "shared" { + logger.WithFields(logrus.Fields{ + "remaining": remaining, + "resetAfter": resetAfter, + "identifier": q.identifier, + "route": item.Req.URL.String(), + "method": item.Req.Method, + "path": path, + "discordBucket": bucketHash, + "ratelimitScope": scope, + "multipleBucketsInPath": multiBucket, + }).Warn("Unexpected 429") + return + } + + if resp.StatusCode == 404 && strings.HasPrefix(path, "/webhooks/") && !isInteraction(item.Req.URL.String()) { + logger.WithFields(logrus.Fields{ + "route": item.Req.URL.String(), + "method": item.Req.Method, + }).Info("Setting fail fast 404 for webhook") + + ch.Lock() + ch.lockerFun = return404webhook + ch.Unlock() + return + } + + if resp.StatusCode == 401 && !isInteraction(item.Req.URL.String()) && q.queueType != NoAuth { + // Permanently lock this queue + logger.WithFields(logrus.Fields{ + "route": item.Req.URL.String(), + "method": item.Req.Method, + "identifier": q.identifier, + "status": resp.StatusCode, + }).Error("Received 401 during normal operation, assuming token is invalidated, locking bucket permanently") + + if EnvGet("DISABLE_401_LOCK", "false") != "true" { + atomic.StoreInt64(q.isTokenInvalid, 999) + } + return + } +} + +func (q *RequestQueue) subscribe(ch *QueueChannel, path string, pathHashInt, majorBucketHashInt uint64) { + // This function has 1 goroutine for each bucket path + // Locking here is not needed + + majorBucketHash := strconv.FormatUint(majorBucketHashInt, 10) + pathHash := strconv.FormatUint(pathHashInt, 10) + + for item := range ch.ch { + ctx := context.WithValue(item.Req.Context(), "identifier", q.identifier) + + if atomic.LoadInt64(q.isTokenInvalid) > 0 { + return401(item) + continue + } + + if globalUnlockedUntil := atomic.LoadInt64(q.globalLockedUntil); globalUnlockedUntil > 0 { + if d := time.Until(time.Unix(0, globalUnlockedUntil)); d > 0 { + time.Sleep(d) } + _ = atomic.CompareAndSwapInt64(q.globalLockedUntil, globalUnlockedUntil, 0) } - // Prevent reaction bucket from being stuck - if resp.StatusCode == 429 && scope == "shared" && (path == "/channels/!/messages/!/reactions/!modify" || path == "/channels/!/messages/!/reactions/!/!") { - prevRem, prevReset = remaining, resetAfter + ch.Lock() + if ch.lockerFun != nil { + ch.lockerFun(item) + ch.Unlock() continue } + ch.Unlock() + + // This is unfortunate, but we need to read the body here so that the ctx gets closed properly + // when the client disconnects, which is very useful for cancelling `ratelimit.Acquire` early + // see: https://github.com/golang/go/issues/23262 + var err error + item.ReqBody, err = io.ReadAll(item.Req.Body) + if err != nil { + _ = item.Req.Body.Close() + item.errChan <- err + continue + } + _ = item.Req.Body.Close() + + buckets := q.getBucketsContextManager(ch) + + if buckets != nil { + if err = buckets.Acquire(ctx); err != nil { + bucketsContextManagerPool.Put(buckets) + item.errChan <- err + continue + } + } - if remaining == 0 || resp.StatusCode == 429 { - duration := time.Until(time.Now().Add(resetAfter)) - time.Sleep(duration) + // We don't have the initial headers, so we do the requests sequentially, which should + // create and populate the bucket when it's known, of it thats what the user wants + // If this is a route with no ratelimits, then we will simply execute them all sequentially, + // which should be fine + if buckets == nil || !allowConcurrentRequests { + q.doRequest(ctx, item, ch, buckets, path, pathHash, majorBucketHash) + } else { + go q.doRequest(ctx, item, ch, buckets, path, pathHash, majorBucketHash) } - prevRem, prevReset = remaining, resetAfter } } diff --git a/lib/queue_manager.go b/lib/queue_manager.go index f0f1ce4..ca10fe4 100644 --- a/lib/queue_manager.go +++ b/lib/queue_manager.go @@ -3,15 +3,16 @@ package lib import ( "context" "errors" - lru "github.com/hashicorp/golang-lru" - "github.com/hashicorp/memberlist" - "github.com/sirupsen/logrus" "net/http" "sort" "strconv" "strings" "sync" "time" + + lru "github.com/hashicorp/golang-lru" + "github.com/hashicorp/memberlist" + "github.com/sirupsen/logrus" ) type QueueType int64 @@ -23,25 +24,27 @@ const ( ) // Some routes that have @me on the path don't really spread out through the cluster, causing issues -// and exacerbating tail latency hits from Discord. Only routes with no ratelimit headers should be put here +// and exacerbating tail latency hits from Discord. Same goes for interaction callbacks. +// Only routes with no ratelimit headers should be put here var pathsToRouteLocally = map[uint64]struct{}{ - HashCRC64("/users/@me/channels"): {}, - HashCRC64("/users/@me"): {}, + HashCRC64("/users/@me/channels"): {}, + HashCRC64("/users/@me"): {}, + HashCRC64("/interactions/!/!/callback"): {}, } type QueueManager struct { - sync.RWMutex queues map[string]*RequestQueue bearerQueues *lru.Cache - bearerMu sync.RWMutex - bufferSize int cluster *memberlist.Memberlist clusterGlobalRateLimiter *ClusterGlobalRateLimiter - orderedClusterMembers []string nameToAddressMap map[string]string localNodeName string localNodeIP string localNodeProxyListenAddr string + orderedClusterMembers []string + bufferSize int + sync.RWMutex + bearerMu sync.RWMutex } func onEvictLruItem(key interface{}, value interface{}) { @@ -246,20 +249,39 @@ func (m *QueueManager) getOrCreateBearerQueue(token string) (*RequestQueue, erro return q.(*RequestQueue), nil } +func isPathTraversal(path string) bool { + segments := strings.Split(path, "/") + for _, segment := range segments { + if segment == ".." { + return true + } + } + return false +} + func (m *QueueManager) DiscordRequestHandler(resp http.ResponseWriter, req *http.Request) { reqStart := time.Now() + + if isPathTraversal(req.URL.Path) { + logger.WithFields(logrus.Fields{"method": req.Method, "url": req.URL.RawPath}).Warn("path traversal detected, dropping request") + resp.Header().Set("generated-by-proxy", "true") + resp.Header().Set("reason", "path traversal") + resp.WriteHeader(422) + return + } + metricsPath := GetMetricsPath(req.URL.Path) ConnectionsOpen.With(map[string]string{"route": metricsPath, "method": req.Method}).Inc() defer ConnectionsOpen.With(map[string]string{"route": metricsPath, "method": req.Method}).Dec() token := req.Header.Get("Authorization") - routingHash, path, queueType := m.GetRequestRoutingInfo(req, token) + routingHash, majorBucketHash, path, queueType := m.GetRequestRoutingInfo(req, token) - m.fulfillRequest(&resp, req, queueType, path, routingHash, token, reqStart) + m.fulfillRequest(&resp, req, queueType, path, routingHash, majorBucketHash, token, reqStart) } -func (m *QueueManager) GetRequestRoutingInfo(req *http.Request, token string) (routingHash uint64, path string, queueType QueueType) { - path = GetOptimisticBucketPath(req.URL.Path, req.Method) +func (m *QueueManager) GetRequestRoutingInfo(req *http.Request, token string) (routingHash, majorBucketHash uint64, path string, queueType QueueType) { + path, majorBucketHash = GetOptimisticBucketPath(req.URL.Path, req.Method) queueType = NoAuth if strings.HasPrefix(token, "Bearer") { queueType = Bearer @@ -271,7 +293,7 @@ func (m *QueueManager) GetRequestRoutingInfo(req *http.Request, token string) (r return } -func (m *QueueManager) fulfillRequest(resp *http.ResponseWriter, req *http.Request, queueType QueueType, path string, pathHash uint64, token string, reqStart time.Time) { +func (m *QueueManager) fulfillRequest(resp *http.ResponseWriter, req *http.Request, queueType QueueType, path string, pathHash, majorBucketHash uint64, token string, reqStart time.Time) { logEntry := logger.WithField("clientIp", req.RemoteAddr) forwdFor := req.Header.Get("X-Forwarded-For") if forwdFor != "" { @@ -330,11 +352,15 @@ func (m *QueueManager) fulfillRequest(resp *http.ResponseWriter, req *http.Reque } } } - err = q.Queue(req, resp, path, pathHash) + err = q.Queue(req, resp, path, pathHash, majorBucketHash) if err != nil { log := logEntry.WithField("function", "Queue") if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { - log.WithField("waitedFor", time.Since(reqStart)).Warn(err) + log.WithFields(logrus.Fields{ + "waitedFor": time.Since(reqStart), + "method": req.Method, + "route": req.URL.Path, + }).Warn(err) } else { log.Error(err) } diff --git a/main.go b/main.go index 2288d8b..f3ede1e 100644 --- a/main.go +++ b/main.go @@ -2,10 +2,7 @@ package main import ( "context" - "github.com/germanoeich/nirn-proxy/lib" - "github.com/hashicorp/memberlist" - _ "github.com/joho/godotenv/autoload" - "github.com/sirupsen/logrus" + "errors" "net" "net/http" "os" @@ -13,6 +10,12 @@ import ( "strings" "syscall" "time" + + "github.com/hashicorp/memberlist" + _ "github.com/joho/godotenv/autoload" + "github.com/sirupsen/logrus" + + "github.com/germanoeich/nirn-proxy/lib" ) var logger = logrus.New() @@ -22,13 +25,18 @@ var bufferSize = 50 func setupLogger() { logLevel := lib.EnvGet("LOG_LEVEL", "info") - lvl, err := logrus.ParseLevel(logLevel) + lvl, err := logrus.ParseLevel(logLevel) if err != nil { panic("Failed to parse log level") } + logger.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + TimestampFormat: time.RFC3339Nano, + }) logger.SetLevel(lvl) + lib.SetLogger(logger) } @@ -75,8 +83,9 @@ func main() { globalOverrides := lib.EnvGet("BOT_RATELIMIT_OVERRIDES", "") disableGlobalRatelimitDetection := lib.EnvGetBool("DISABLE_GLOBAL_RATELIMIT_DETECTION", false) + allowConcurrentRequests := lib.EnvGetBool("ALLOW_CONCURRENT_REQUESTS", true) - lib.ConfigureDiscordHTTPClient(outboundIp, time.Duration(timeout)*time.Millisecond, disableHttp2, globalOverrides, disableGlobalRatelimitDetection) + lib.ConfigureDiscordHTTPClient(outboundIp, time.Duration(timeout)*time.Millisecond, globalOverrides, disableHttp2, disableGlobalRatelimitDetection, allowConcurrentRequests) port := lib.EnvGet("PORT", "8080") bindIp := lib.EnvGet("BIND_IP", "0.0.0.0") @@ -108,11 +117,11 @@ func main() { go lib.StartMetrics(bindIp + ":" + port) } - done := make(chan os.Signal, 1) + done := make(chan os.Signal) signal.Notify(done, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) go func() { - if err := s.ListenAndServe(); err != nil && err != http.ErrServerClosed { + if err := s.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { logger.WithFields(logrus.Fields{"function": "http.ListenAndServe"}).Panic(err) } }()