diff --git a/src/ServiceControl.Transports.ASBS.Tests/QueueLengthProviderBackoffTests.cs b/src/ServiceControl.Transports.ASBS.Tests/QueueLengthProviderBackoffTests.cs new file mode 100644 index 0000000000..bdf16e5035 --- /dev/null +++ b/src/ServiceControl.Transports.ASBS.Tests/QueueLengthProviderBackoffTests.cs @@ -0,0 +1,46 @@ +namespace ServiceControl.Transports.UnitTests.ASBS +{ + using System; + using NUnit.Framework; + using ServiceControl.Transports.ASBS; + + [TestFixture] + class QueueLengthProviderBackoffTests + { + // Base is the configured QueueLengthQueryDelayInterval (default). Max is the internal reactive-backoff cap. + static readonly TimeSpan Base = TimeSpan.FromMilliseconds(500); + static readonly TimeSpan Max = TimeSpan.FromSeconds(60); + + [Test] + public void Doubles_the_delay_when_throttled() + { + // A throttled cycle backs off so the provider stops making the throttling worse. + Assert.That(QueueLengthProvider.NextDelay(Base, Base, Max, throttled: true), + Is.EqualTo(TimeSpan.FromSeconds(1))); + } + + [Test] + public void Caps_the_delay_at_max_when_throttled() + { + // 40s doubled would be 80s; the cap keeps it bounded. + Assert.That(QueueLengthProvider.NextDelay(TimeSpan.FromSeconds(40), Base, Max, throttled: true), + Is.EqualTo(Max)); + } + + [Test] + public void Halves_the_delay_on_success_while_still_backed_off() + { + // Success steps the cadence back down gradually rather than snapping to base, avoiding + // throttle -> reset -> throttle oscillation. + Assert.That(QueueLengthProvider.NextDelay(TimeSpan.FromSeconds(2), Base, Max, throttled: false), + Is.EqualTo(TimeSpan.FromSeconds(1))); + } + + [Test] + public void Never_drops_below_base_on_success() + { + Assert.That(QueueLengthProvider.NextDelay(Base, Base, Max, throttled: false), + Is.EqualTo(Base)); + } + } +} diff --git a/src/ServiceControl.Transports.ASBS/AuthenticationMethod.cs b/src/ServiceControl.Transports.ASBS/AuthenticationMethod.cs index 966b2f3c3d..50fd649115 100644 --- a/src/ServiceControl.Transports.ASBS/AuthenticationMethod.cs +++ b/src/ServiceControl.Transports.ASBS/AuthenticationMethod.cs @@ -5,7 +5,7 @@ public abstract class AuthenticationMethod { - public abstract ServiceBusAdministrationClient BuildManagementClient(); + public abstract ServiceBusAdministrationClient BuildManagementClient(ServiceBusAdministrationClientOptions options = null); public abstract AzureServiceBusTransport CreateTransportDefinition(ConnectionSettings connectionSettings, TopicTopology topology); } } \ No newline at end of file diff --git a/src/ServiceControl.Transports.ASBS/QueueLengthProvider.cs b/src/ServiceControl.Transports.ASBS/QueueLengthProvider.cs index 8978c03fed..bf6cddf5e0 100644 --- a/src/ServiceControl.Transports.ASBS/QueueLengthProvider.cs +++ b/src/ServiceControl.Transports.ASBS/QueueLengthProvider.cs @@ -5,6 +5,7 @@ namespace ServiceControl.Transports.ASBS using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; + using Azure.Messaging.ServiceBus; using Azure.Messaging.ServiceBus.Administration; using Microsoft.Extensions.Logging; @@ -16,7 +17,11 @@ public QueueLengthProvider(TransportSettings settings, Action exponential up to the cap; success -> halve back toward + // the base so it settles near the sustainable rate instead of oscillating. + internal static TimeSpan NextDelay(TimeSpan current, TimeSpan baseDelay, TimeSpan maxDelay, bool throttled) + { + if (throttled) + { + var doubled = TimeSpan.FromTicks(current.Ticks * 2); + return doubled > maxDelay ? maxDelay : doubled; + } + + var halved = TimeSpan.FromTicks(current.Ticks / 2); + return halved < baseDelay ? baseDelay : halved; + } + async Task> GetQueueList(CancellationToken cancellationToken) { var queuePathToRuntimeInfo = new Dictionary(StringComparer.InvariantCultureIgnoreCase); @@ -111,5 +169,8 @@ void UpdateQueueLength(KeyValuePair monitoredEndpoint, IReadOnly readonly ServiceBusAdministrationClient managementClient; readonly TimeSpan queryDelayInterval; readonly ILogger logger; + + // Upper bound for the reactive back-off when Azure Service Bus is throttling management operations. + static readonly TimeSpan MaxBackoffInterval = TimeSpan.FromMinutes(1); } } \ No newline at end of file diff --git a/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj b/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj index bd61cf9af1..916b33ee96 100644 --- a/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj +++ b/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj @@ -10,6 +10,10 @@ + + + + diff --git a/src/ServiceControl.Transports.ASBS/SharedAccessSignatureAuthentication.cs b/src/ServiceControl.Transports.ASBS/SharedAccessSignatureAuthentication.cs index 5368353371..3ad4a0f2a0 100644 --- a/src/ServiceControl.Transports.ASBS/SharedAccessSignatureAuthentication.cs +++ b/src/ServiceControl.Transports.ASBS/SharedAccessSignatureAuthentication.cs @@ -9,8 +9,10 @@ public class SharedAccessSignatureAuthentication : AuthenticationMethod public string ConnectionString { get; } - public override ServiceBusAdministrationClient BuildManagementClient() - => new ServiceBusAdministrationClient(ConnectionString); + public override ServiceBusAdministrationClient BuildManagementClient(ServiceBusAdministrationClientOptions options = null) + => options is null + ? new ServiceBusAdministrationClient(ConnectionString) + : new ServiceBusAdministrationClient(ConnectionString, options); public override AzureServiceBusTransport CreateTransportDefinition(ConnectionSettings connectionSettings, TopicTopology topology) => new AzureServiceBusTransport(ConnectionString, topology); diff --git a/src/ServiceControl.Transports.ASBS/TokenCredentialAuthentication.cs b/src/ServiceControl.Transports.ASBS/TokenCredentialAuthentication.cs index d00e277891..27c5223cdc 100644 --- a/src/ServiceControl.Transports.ASBS/TokenCredentialAuthentication.cs +++ b/src/ServiceControl.Transports.ASBS/TokenCredentialAuthentication.cs @@ -27,7 +27,10 @@ public TokenCredentialAuthentication(string fullyQualifiedNamespace, string? cli public string? ClientId { get; } - public override ServiceBusAdministrationClient BuildManagementClient() => new(FullyQualifiedNamespace, Credential); + public override ServiceBusAdministrationClient BuildManagementClient(ServiceBusAdministrationClientOptions? options = null) + => options is null + ? new(FullyQualifiedNamespace, Credential) + : new(FullyQualifiedNamespace, Credential, options); public override AzureServiceBusTransport CreateTransportDefinition(ConnectionSettings connectionSettings, TopicTopology topology) {