From 648587c8b05d473acb5fe8765c7da5db5dfd9937 Mon Sep 17 00:00:00 2001 From: Andrey Maltsev Date: Fri, 31 Jul 2026 14:53:41 +0300 Subject: [PATCH] fix: pin consistent-hash clients across weighted backendRefs When a route splits traffic across multiple weighted backendRefs and uses a ConsistentHash load balancer, Envoy Gateway rendered the split as a weighted_clusters route action whose cluster selection is random per request. The route hash policy only pinned endpoint selection within a cluster, so a client was not pinned to a single backend across the split. Set WeightedCluster.use_hash_policy on the generated weighted clusters when a hash policy is present, so Envoy selects the weighted cluster deterministically from the request's hash policy instead of at random. Gated on a hash policy being configured, so non-ConsistentHash weighted routes are unaffected. Fixes #9626 Signed-off-by: Andrey Maltsev --- internal/xds/translator/route.go | 12 +++++ ...oute-weighted-backend-consistent-hash.yaml | 49 +++++++++++++++++++ ...hted-backend-consistent-hash.clusters.yaml | 42 ++++++++++++++++ ...ted-backend-consistent-hash.endpoints.yaml | 24 +++++++++ ...ted-backend-consistent-hash.listeners.yaml | 35 +++++++++++++ ...ighted-backend-consistent-hash.routes.yaml | 34 +++++++++++++ .../http-route-weighted-zones.routes.yaml | 1 + .../9626-consistent-hash-weighted-backends.md | 1 + 8 files changed, 198 insertions(+) create mode 100644 internal/xds/translator/testdata/in/xds-ir/http-route-weighted-backend-consistent-hash.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.endpoints.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.routes.yaml create mode 100644 release-notes/current/bug_fixes/9626-consistent-hash-weighted-backends.md diff --git a/internal/xds/translator/route.go b/internal/xds/translator/route.go index 53578096a2..64aa32699a 100644 --- a/internal/xds/translator/route.go +++ b/internal/xds/translator/route.go @@ -101,6 +101,18 @@ func buildXdsRoute(httpRoute *ir.HTTPRoute, httpListener *ir.HTTPListener, backe // Hash Policy if router.GetRoute() != nil { router.GetRoute().HashPolicy = buildHashPolicy(httpRoute) + + // When a route splits traffic across multiple weighted backendRefs and uses a + // ConsistentHash load balancer, enable use_hash_policy so Envoy selects the weighted + // cluster deterministically from the request's hash policy instead of at random. + // Without this, the consistent hash only pins endpoint selection within a cluster, + // while the choice among the weighted backends stays random per request, so a client + // is not pinned to a single backend across the split. + if wc := router.GetRoute().GetWeightedClusters(); wc != nil && len(router.GetRoute().GetHashPolicy()) > 0 { + wc.RandomValueSpecifier = &routev3.WeightedCluster_UseHashPolicy{ + UseHashPolicy: wrapperspb.Bool(true), + } + } } // Timeouts diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route-weighted-backend-consistent-hash.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route-weighted-backend-consistent-hash.yaml new file mode 100644 index 0000000000..80218ff1f4 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/http-route-weighted-backend-consistent-hash.yaml @@ -0,0 +1,49 @@ +http: +- name: "first-listener" + address: "::" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + # A weighted split across two distinct backend clusters (cluster-per-setting is + # triggered by the per-backendRef filter) combined with a ConsistentHash load + # balancer. use_hash_policy must be set on the weighted clusters so the cluster + # selection is driven by the request hash policy instead of being random per + # request, pinning a client to a single backend across the split. + - name: "first-route" + hostname: "*" + traffic: + loadBalancer: + consistentHash: + headers: + - name: x-user-id + destination: + name: "first-route-dest" + settings: + - addressType: IP + endpoints: + - host: "1.1.1.1" + port: 50001 + weight: 80 + name: "first-route-dest/backend/0" + filters: + addRequestHeaders: + - name: x-backend + value: + - primary + append: false + - addressType: IP + endpoints: + - host: "2.2.2.2" + port: 50002 + weight: 20 + name: "first-route-dest/backend/1" + filters: + addRequestHeaders: + - name: x-backend + value: + - backup + append: false diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.clusters.yaml new file mode 100644 index 0000000000..69b8ef91be --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.clusters.yaml @@ -0,0 +1,42 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: first-route-dest/backend/0 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.maglev + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.maglev.v3.Maglev + name: first-route-dest/backend/0 + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: first-route-dest/backend/1 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.maglev + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.maglev.v3.Maglev + name: first-route-dest/backend/1 + perConnectionBufferLimitBytes: 32768 + type: EDS diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.endpoints.yaml new file mode 100644 index 0000000000..464a32d6c6 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.endpoints.yaml @@ -0,0 +1,24 @@ +- clusterName: first-route-dest/backend/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.1.1.1 + portValue: 50001 + loadBalancingWeight: 1 + loadBalancingWeight: 80 + locality: + region: first-route-dest/backend/0 +- clusterName: first-route-dest/backend/1 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 2.2.2.2 + portValue: 50002 + loadBalancingWeight: 1 + loadBalancingWeight: 20 + locality: + region: first-route-dest/backend/1 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.listeners.yaml new file mode 100644 index 0000000000..5dd5e46e3c --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.listeners.yaml @@ -0,0 +1,35 @@ +- address: + socketAddress: + address: '::' + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10080 + useRemoteAddress: true + name: first-listener + maxConnectionsToAcceptPerSocketEvent: 1 + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.routes.yaml new file mode 100644 index 0000000000..47faef0704 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-backend-consistent-hash.routes.yaml @@ -0,0 +1,34 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - match: + prefix: / + name: first-route + route: + clusterNotFoundResponseCode: INTERNAL_SERVER_ERROR + hashPolicy: + - header: + headerName: x-user-id + upgradeConfigs: + - upgradeType: websocket + weightedClusters: + clusters: + - name: first-route-dest/backend/0 + requestHeadersToAdd: + - appendAction: OVERWRITE_IF_EXISTS_OR_ADD + header: + key: x-backend + value: primary + weight: 80 + - name: first-route-dest/backend/1 + requestHeadersToAdd: + - appendAction: OVERWRITE_IF_EXISTS_OR_ADD + header: + key: x-backend + value: backup + weight: 20 + useHashPolicy: true diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-zones.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-zones.routes.yaml index daae320889..cd09bea851 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-zones.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-zones.routes.yaml @@ -30,6 +30,7 @@ clusters: - name: route-with-weighted-zones-maglev-dest/backend/0 weight: 1 + useHashPolicy: true - match: prefix: / name: route-with-weighted-zones-backend-utilization diff --git a/release-notes/current/bug_fixes/9626-consistent-hash-weighted-backends.md b/release-notes/current/bug_fixes/9626-consistent-hash-weighted-backends.md new file mode 100644 index 0000000000..fd329176ec --- /dev/null +++ b/release-notes/current/bug_fixes/9626-consistent-hash-weighted-backends.md @@ -0,0 +1 @@ +Fixed ConsistentHash load balancing not pinning a client to a single backend when a route splits traffic across multiple weighted backendRefs. Envoy Gateway now sets `use_hash_policy` on the generated weighted clusters when a ConsistentHash load balancer is configured, so the request's hash policy selects the weighted cluster deterministically instead of at random per request.