From d213744a5a599b24bc405552243aa63a5f728dc1 Mon Sep 17 00:00:00 2001
From: Ramon Smits <ramon.smits@gmail.com>
Date: Wed, 15 Jul 2026 10:12:12 +0200
Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Detect=20and=20back=20off=20Azure?=
 =?UTF-8?q?=20Service=20Bus=20queue-length=20management=20throttling?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Azure Service Bus throttles management operations (GetQueuesRuntimeProperties)
used to read queue lengths. By default the client retries the HTTP 429 responses
internally and usually succeeds, so the throttling never surfaced as an exception
and was invisible in the logs while still showing on the namespace dashboard.

Disable the internal retries for the queue-length management client so a throttled
query fails fast and throws ServiceBusException(ServiceBusy). The polling loop
catches that and reactively backs off the query interval (exponential up to 1 min,
recovering gradually) and logs a warning once per throttling episode, until the
throttling clears. Disabling retries also removes the per-request retry
amplification (each 429 retried up to 3x).
---
 .../QueueLengthProviderBackoffTests.cs        | 46 +++++++++++++
 .../AuthenticationMethod.cs                   |  2 +-
 .../QueueLengthProvider.cs                    | 67 ++++++++++++++++++-
 .../ServiceControl.Transports.ASBS.csproj     |  4 ++
 .../SharedAccessSignatureAuthentication.cs    |  6 +-
 .../TokenCredentialAuthentication.cs          |  5 +-
 6 files changed, 123 insertions(+), 7 deletions(-)
 create mode 100644 src/ServiceControl.Transports.ASBS.Tests/QueueLengthProviderBackoffTests.cs

diff --git a/src/ServiceControl.Transports.ASBS.Tests/QueueLengthProviderBackoffTests.cs b/src/ServiceControl.Transports.ASBS.Tests/QueueLengthProviderBackoffTests.cs
new file mode 100644
index 0000000000..bdf16e5035
--- /dev/null
+++ b/src/ServiceControl.Transports.ASBS.Tests/QueueLengthProviderBackoffTests.cs
@@ -0,0 +1,46 @@
+namespace ServiceControl.Transports.UnitTests.ASBS
+{
+    using System;
+    using NUnit.Framework;
+    using ServiceControl.Transports.ASBS;
+
+    [TestFixture]
+    class QueueLengthProviderBackoffTests
+    {
+        // Base is the configured QueueLengthQueryDelayInterval (default). Max is the internal reactive-backoff cap.
+        static readonly TimeSpan Base = TimeSpan.FromMilliseconds(500);
+        static readonly TimeSpan Max = TimeSpan.FromSeconds(60);
+
+        [Test]
+        public void Doubles_the_delay_when_throttled()
+        {
+            // A throttled cycle backs off so the provider stops making the throttling worse.
+            Assert.That(QueueLengthProvider.NextDelay(Base, Base, Max, throttled: true),
+                Is.EqualTo(TimeSpan.FromSeconds(1)));
+        }
+
+        [Test]
+        public void Caps_the_delay_at_max_when_throttled()
+        {
+            // 40s doubled would be 80s; the cap keeps it bounded.
+            Assert.That(QueueLengthProvider.NextDelay(TimeSpan.FromSeconds(40), Base, Max, throttled: true),
+                Is.EqualTo(Max));
+        }
+
+        [Test]
+        public void Halves_the_delay_on_success_while_still_backed_off()
+        {
+            // Success steps the cadence back down gradually rather than snapping to base, avoiding
+            // throttle -> reset -> throttle oscillation.
+            Assert.That(QueueLengthProvider.NextDelay(TimeSpan.FromSeconds(2), Base, Max, throttled: false),
+                Is.EqualTo(TimeSpan.FromSeconds(1)));
+        }
+
+        [Test]
+        public void Never_drops_below_base_on_success()
+        {
+            Assert.That(QueueLengthProvider.NextDelay(Base, Base, Max, throttled: false),
+                Is.EqualTo(Base));
+        }
+    }
+}
diff --git a/src/ServiceControl.Transports.ASBS/AuthenticationMethod.cs b/src/ServiceControl.Transports.ASBS/AuthenticationMethod.cs
index 966b2f3c3d..50fd649115 100644
--- a/src/ServiceControl.Transports.ASBS/AuthenticationMethod.cs
+++ b/src/ServiceControl.Transports.ASBS/AuthenticationMethod.cs
@@ -5,7 +5,7 @@
 
     public abstract class AuthenticationMethod
     {
-        public abstract ServiceBusAdministrationClient BuildManagementClient();
+        public abstract ServiceBusAdministrationClient BuildManagementClient(ServiceBusAdministrationClientOptions options = null);
         public abstract AzureServiceBusTransport CreateTransportDefinition(ConnectionSettings connectionSettings, TopicTopology topology);
     }
 }
\ No newline at end of file
diff --git a/src/ServiceControl.Transports.ASBS/QueueLengthProvider.cs b/src/ServiceControl.Transports.ASBS/QueueLengthProvider.cs
index 8978c03fed..bf6cddf5e0 100644
--- a/src/ServiceControl.Transports.ASBS/QueueLengthProvider.cs
+++ b/src/ServiceControl.Transports.ASBS/QueueLengthProvider.cs
@@ -5,6 +5,7 @@ namespace ServiceControl.Transports.ASBS
     using System.Collections.Generic;
     using System.Threading;
     using System.Threading.Tasks;
+    using Azure.Messaging.ServiceBus;
     using Azure.Messaging.ServiceBus.Administration;
     using Microsoft.Extensions.Logging;
 
@@ -16,7 +17,11 @@ public QueueLengthProvider(TransportSettings settings, Action<QueueLengthEntry[]
 
             queryDelayInterval = connectionSettings.QueryDelayInterval ?? TimeSpan.FromMilliseconds(500);
 
-            managementClient = connectionSettings.AuthenticationMethod.BuildManagementClient();
+            // Disable the client's internal retries so throttled (429) queries fail fast and surface as an
+            // exception (see ExecuteAsync) instead of being silently retried away, and drop the retry amplification.
+            var clientOptions = new ServiceBusAdministrationClientOptions();
+            clientOptions.Retry.MaxRetries = 0;
+            managementClient = connectionSettings.AuthenticationMethod.BuildManagementClient(clientOptions);
             this.logger = logger;
         }
 
@@ -29,12 +34,18 @@ public override void TrackEndpointInputQueue(EndpointToQueueMapping queueToTrack
 
         protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
+            // currentDelay backs off while throttled and recovers to the base once clear; throttled latches the log.
+            var currentDelay = queryDelayInterval;
+            var throttled = false;
+
             while (!stoppingToken.IsCancellationRequested)
             {
+                bool throttledThisCycle;
+
                 try
                 {
                     logger.LogDebug("Waiting for next interval");
-                    await Task.Delay(queryDelayInterval, stoppingToken);
+                    await Task.Delay(currentDelay, stoppingToken);
 
                     logger.LogDebug("Querying management client");
 
@@ -43,18 +54,65 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                     logger.LogDebug("Retrieved details of {QueueCount} queues", queueRuntimeInfos.Count);
 
                     UpdateAllQueueLengths(queueRuntimeInfos);
+
+                    throttledThisCycle = false;
                 }
                 catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
                 {
-                    // no-op
+                    break;
+                }
+                catch (ServiceBusException e) when (e.Reason == ServiceBusFailureReason.ServiceBusy)
+                {
+                    // With retries disabled, throttling surfaces here immediately and drives the back-off.
+                    throttledThisCycle = true;
                 }
                 catch (Exception e)
                 {
+                    // Unrelated error: log and keep the current cadence.
                     logger.LogError(e, "Error querying Azure Service Bus queue sizes");
+                    continue;
+                }
+
+                // Store nothing on a throttled cycle; the last values stand. Ideally we'd record an explicit
+                // "no value" (null/-1) so the graph shows a gap, but Value is a non-nullable long and the
+                // API/ServicePulse coerce gaps to 0 - that needs a new API contract + a ServicePulse using it.
+                currentDelay = NextDelay(currentDelay, queryDelayInterval, MaxBackoffInterval, throttledThisCycle);
+
+                if (throttledThisCycle)
+                {
+                    if (!throttled)
+                    {
+                        throttled = true;
+                        logger.LogWarning("Azure Service Bus is throttling the management operations used to read queue lengths. Backing off to a {CurrentDelay} query interval. This is expected on busy or large namespaces; increase the 'QueueLengthQueryDelayInterval' connection string setting to reduce it. Queue length metrics may be delayed until the throttling clears.", currentDelay);
+                    }
+                    else
+                    {
+                        logger.LogDebug("Still throttled by Azure Service Bus; backing off query interval to {CurrentDelay}", currentDelay);
+                    }
+                }
+                else if (throttled && currentDelay == queryDelayInterval)
+                {
+                    // Recover only once fully back at the base, so a slow ramp-down doesn't re-arm the warning.
+                    throttled = false;
+                    logger.LogInformation("Azure Service Bus management throttling has cleared; resumed querying queue lengths every {QueryDelayInterval}", queryDelayInterval);
                 }
             }
         }
 
+        // Pure back-off policy (unit-tested). Throttled -> exponential up to the cap; success -> halve back toward
+        // the base so it settles near the sustainable rate instead of oscillating.
+        internal static TimeSpan NextDelay(TimeSpan current, TimeSpan baseDelay, TimeSpan maxDelay, bool throttled)
+        {
+            if (throttled)
+            {
+                var doubled = TimeSpan.FromTicks(current.Ticks * 2);
+                return doubled > maxDelay ? maxDelay : doubled;
+            }
+
+            var halved = TimeSpan.FromTicks(current.Ticks / 2);
+            return halved < baseDelay ? baseDelay : halved;
+        }
+
         async Task<IReadOnlyDictionary<string, QueueRuntimeProperties>> GetQueueList(CancellationToken cancellationToken)
         {
             var queuePathToRuntimeInfo = new Dictionary<string, QueueRuntimeProperties>(StringComparer.InvariantCultureIgnoreCase);
@@ -111,5 +169,8 @@ void UpdateQueueLength(KeyValuePair<string, string> monitoredEndpoint, IReadOnly
         readonly ServiceBusAdministrationClient managementClient;
         readonly TimeSpan queryDelayInterval;
         readonly ILogger<QueueLengthProvider> logger;
+
+        // Upper bound for the reactive back-off when Azure Service Bus is throttling management operations.
+        static readonly TimeSpan MaxBackoffInterval = TimeSpan.FromMinutes(1);
     }
 }
\ No newline at end of file
diff --git a/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj b/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj
index bd61cf9af1..916b33ee96 100644
--- a/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj
+++ b/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj
@@ -10,6 +10,10 @@
     <ProjectReference Include="..\ServiceControl.Transports\ServiceControl.Transports.csproj" Private="false" ExcludeAssets="runtime" />
   </ItemGroup>
 
+  <ItemGroup>
+    <InternalsVisibleTo Include="ServiceControl.Transports.ASBS.Tests" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Azure.Identity" />
     <PackageReference Include="Azure.ResourceManager.Monitor" />
diff --git a/src/ServiceControl.Transports.ASBS/SharedAccessSignatureAuthentication.cs b/src/ServiceControl.Transports.ASBS/SharedAccessSignatureAuthentication.cs
index 5368353371..3ad4a0f2a0 100644
--- a/src/ServiceControl.Transports.ASBS/SharedAccessSignatureAuthentication.cs
+++ b/src/ServiceControl.Transports.ASBS/SharedAccessSignatureAuthentication.cs
@@ -9,8 +9,10 @@ public class SharedAccessSignatureAuthentication : AuthenticationMethod
 
         public string ConnectionString { get; }
 
-        public override ServiceBusAdministrationClient BuildManagementClient()
-            => new ServiceBusAdministrationClient(ConnectionString);
+        public override ServiceBusAdministrationClient BuildManagementClient(ServiceBusAdministrationClientOptions options = null)
+            => options is null
+                ? new ServiceBusAdministrationClient(ConnectionString)
+                : new ServiceBusAdministrationClient(ConnectionString, options);
 
         public override AzureServiceBusTransport CreateTransportDefinition(ConnectionSettings connectionSettings, TopicTopology topology)
             => new AzureServiceBusTransport(ConnectionString, topology);
diff --git a/src/ServiceControl.Transports.ASBS/TokenCredentialAuthentication.cs b/src/ServiceControl.Transports.ASBS/TokenCredentialAuthentication.cs
index d00e277891..27c5223cdc 100644
--- a/src/ServiceControl.Transports.ASBS/TokenCredentialAuthentication.cs
+++ b/src/ServiceControl.Transports.ASBS/TokenCredentialAuthentication.cs
@@ -27,7 +27,10 @@ public TokenCredentialAuthentication(string fullyQualifiedNamespace, string? cli
 
         public string? ClientId { get; }
 
-        public override ServiceBusAdministrationClient BuildManagementClient() => new(FullyQualifiedNamespace, Credential);
+        public override ServiceBusAdministrationClient BuildManagementClient(ServiceBusAdministrationClientOptions? options = null)
+            => options is null
+                ? new(FullyQualifiedNamespace, Credential)
+                : new(FullyQualifiedNamespace, Credential, options);
 
         public override AzureServiceBusTransport CreateTransportDefinition(ConnectionSettings connectionSettings, TopicTopology topology)
         {