Hamza/feature/retry budget alerts

SW.Bitween.Api/Data/BitweenDbContext.cs

@@ -228,6 +228,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 b.Property(p => p.Id).ValueGeneratedOnAdd();
                 b.Property(p => p.Name).IsRequired().HasMaxLength(200);
                 b.Property(p => p.Groups).StoreAsJson();
+                b.Property(p => p.AlertHandlerId).HasMaxLength(200).IsUnicode(false);
+                b.Property(p => p.AlertHandlerProperties).StoreAsJson();
             });
 
             modelBuilder.Entity<DelayedRetry>(b =>
@@ -245,6 +247,16 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 b.HasKey(p => new { p.SubscriptionId, p.GroupId });
                 b.Property(p => p.AttemptsUsed);
                 b.Property(p => p.LastAttemptOn);
+                b.Property(p => p.ExhaustedNotifiedOn);
+            });
+
+            modelBuilder.Entity<RetryAlertOverride>(b =>
+            {
+                b.ToTable("RetryAlertOverrides");
+                b.HasKey(p => new { p.SubscriptionId, p.GroupId });
+                b.Property(p => p.AlertMode).HasConversion<byte>();
+                b.Property(p => p.AlertHandlerId).HasMaxLength(200).IsUnicode(false);
+                b.Property(p => p.AlertHandlerProperties).StoreAsJson();
             });
 
             modelBuilder.Entity<Xchange>(b =>
@@ -290,6 +302,9 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 b.Property(p => p.ResponseContentType).IsUnicode(false).HasMaxLength(200);
                 b.Property(p => p.OutputContentType).IsUnicode(false).HasMaxLength(200);
                 b.Property(p => p.RetryBlockedReason).HasMaxLength(500);
+                b.Property(p => p.RetryGroupId);
+                b.Property(p => p.AttemptNumber);
+                b.HasIndex(p => p.RetryGroupId);
 
 
                 b.HasOne<Xchange>().WithOne().HasForeignKey<XchangeResult>(p => p.Id).OnDelete(DeleteBehavior.Cascade);

SW.Bitween.Api/Domain/RetryAlertOverride.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using SW.Bitween.Model;
+
+namespace SW.Bitween.Domain;
+
+/// <summary>
+/// The most specific level of the retry-alert hierarchy: where one subscription's failures in one
+/// retry group should be alerted, overriding whatever the group or the policy says.
+/// </summary>
+/// <remarks>
+/// Deliberately its own table rather than columns on <see cref="RetryGroupUsage"/>. Usage rows are
+/// deleted by <c>RetryPolicies/resetusage</c>, so config stored there would be silently discarded
+/// every time someone cleared a spent budget.
+/// </remarks>
+public class RetryAlertOverride
+{
+    /// <summary>The subscription this override applies to.</summary>
+    public int SubscriptionId { get; set; }
+
+    /// <summary><c>RetryGroup.Id</c>, which survives policy edits, so the override does too.</summary>
+    public Guid GroupId { get; set; }
+
+    /// <summary>
+    /// Whether this level sends, stays silent, or defers upward. A row whose mode is
+    /// <see cref="RetryAlertMode.Inherit"/> is equivalent to having no row at all.
+    /// </summary>
+    public RetryAlertMode AlertMode { get; set; }
+
+    /// <summary>Adapter that delivers the alert. Required when <see cref="AlertMode"/> is Send.</summary>
+    public string AlertHandlerId { get; set; }
+
+    /// <summary>That adapter's own settings — api key, recipients, subject.</summary>
+    public IReadOnlyDictionary<string, string> AlertHandlerProperties { get; set; }
+}

SW.Bitween.Api/Domain/RetryBudgetExhaustedEvent.cs

@@ -0,0 +1,42 @@
+using System;
+using SW.PrimitiveTypes;
+
+namespace SW.Bitween.Domain;
+
+/// <summary>
+/// Raised the one time a retry group's <c>MaxAttemptsTotal</c> runs out for a subscription, so the
+/// configured alert handler can be told that failures matching that group have stopped being retried.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Deliberately not an <c>IHasWorkGroup</c> event: it publishes under its own type name and is picked
+/// up by a dedicated <c>IConsume&lt;RetryBudgetExhaustedEvent&gt;</c> consumer with its own queue. A
+/// slow or broken alert handler therefore cannot delay or fail the ordinary notifier path, which
+/// shares the work group's result queue.
+/// </para>
+/// <para>
+/// Carried on <see cref="XchangeResult"/> rather than published directly, so it only reaches the bus
+/// once the failure it describes has actually been committed.
+/// </para>
+/// </remarks>
+public class RetryBudgetExhaustedEvent : BaseDomainEvent
+{
+    /// <summary>The failure that found the budget empty.</summary>
+    public string XchangeId { get; set; }
+
+    public int SubscriptionId { get; set; }
+
+    public Guid GroupId { get; set; }
+
+    /// <summary>The group's name as it was when the budget ran out, in case it is later renamed.</summary>
+    public string GroupName { get; set; }
+
+    /// <summary>
+    /// The ceiling that was reached. Not paired with an "used" count, because at exhaustion the two
+    /// are the same number — except when the ceiling was lowered below what had already been spent,
+    /// where the ceiling is still the meaningful figure.
+    /// </summary>
+    public int MaxAttemptsTotal { get; set; }
+
+    public DateTime OccurredOn { get; set; }
+}

SW.Bitween.Api/Domain/RetryGroupUsage.cs

@@ -24,4 +24,15 @@ public class RetryGroupUsage
 
     /// <summary>When the last attempt was claimed — the only clue left once a group is exhausted.</summary>
     public DateTime LastAttemptOn { get; set; }
+
+    /// <summary>
+    /// When the exhaustion alert for this integration and group was claimed, or <c>null</c> while
+    /// the budget still has room.
+    /// </summary>
+    /// <remarks>
+    /// Claiming this is what makes the alert fire exactly once: every failure after the budget runs
+    /// out would otherwise raise another one. Reset deletes the whole row, which re-arms the alert
+    /// along with the budget.
+    /// </remarks>
+    public DateTime? ExhaustedNotifiedOn { get; set; }
 }

SW.Bitween.Api/Domain/RetryPolicy.cs

@@ -9,6 +9,15 @@ public class RetryPolicy : BaseEntity, IAudited, IRetryPolicy
 {
     public string Name { get; set; }
     public List<RetryGroup> Groups { get; set; } = [];
+
+    /// <summary>
+    /// Default destination for "retry budget exhausted" alerts, used by every group that does not
+    /// override it. Null means no alert unless a group or a subscription+group override defines one.
+    /// </summary>
+    public string AlertHandlerId { get; set; }
+
+    /// <summary>That adapter's own settings — api key, recipients, subject.</summary>
+    public IReadOnlyDictionary<string, string> AlertHandlerProperties { get; set; }
     public DateTime CreatedOn { get; set; }
     public string CreatedBy { get; set; }
     public DateTime? ModifiedOn { get; set; }

SW.Bitween.Api/Domain/XchangeNotification.cs

@@ -5,9 +5,12 @@ namespace SW.Bitween.Domain
 {
     public class XchangeNotification:BaseEntity
     {
+        /// <summary>Name recorded for rows written by the retry-budget alert rather than a notifier.</summary>
+        public const string RetryBudgetAlertName = "Retry budget alert";
+
         private XchangeNotification(){}
 
-        public XchangeNotification(string xchangeId, int notifierId, string notifierName, string exception = null)
+        public XchangeNotification(string xchangeId, int? notifierId, string notifierName, string exception = null)
         {
             XchangeId = xchangeId;
             FinishedOn = DateTime.UtcNow;
@@ -16,12 +19,21 @@ public XchangeNotification(string xchangeId, int notifierId, string notifierName
             NotifierId = notifierId;
             NotifierName = notifierName;
         }
-
+
+        /// <summary>
+        /// Logs an attempt to deliver a "retry budget exhausted" alert. These rows have no
+        /// <see cref="NotifierId"/> because the alert is configured on the retry policy rather than
+        /// on a notifier — which is also how the send is recognised as already done on a redelivery.
+        /// </summary>
+        public static XchangeNotification ForRetryBudgetAlert(string xchangeId, string exception = null) =>
+            new(xchangeId, null, RetryBudgetAlertName, exception);
+
 
         public string XchangeId { get; private set; }
         public bool Success { get; set; }
 
-        public int NotifierId { get; set; }
+        /// <summary>The notifier that produced this row, or <c>null</c> for a retry-budget alert.</summary>
+        public int? NotifierId { get; set; }
         public string NotifierName { get; set; }
 
 

SW.Bitween.Api/Domain/XchangeResult/XchangeResult.cs

@@ -72,7 +72,43 @@ public XchangeResult(string xchangeId,WorkGroup workGroup, XchangeFile outputFil
         /// <summary>Records the policy's refusal so it can be shown alongside the failure.</summary>
         public void SetRetryBlocked(string reason) => RetryBlockedReason = reason;
 
+        /// <summary>
+        /// The retry group that matched this failure, or <c>null</c> when no policy applied or none
+        /// matched. The evaluator works this out and would otherwise discard it, leaving no way to
+        /// ask which failures a group is responsible for.
+        /// </summary>
+        public Guid? RetryGroupId { get; private set; }
 
+        /// <summary>
+        /// How many times this message had already been attempted when the policy evaluated it
+        /// (0 on the original run). Stored because deriving it means walking the whole
+        /// <c>Xchange.RetryFor</c> chain one query at a time.
+        /// </summary>
+        public int? AttemptNumber { get; private set; }
+
+        /// <summary>Records which group owned this failure, and how far into its retries it was.</summary>
+        public void SetRetryEvaluation(Guid groupId, int attemptNumber)
+        {
+            RetryGroupId = groupId;
+            AttemptNumber = attemptNumber;
+        }
 
+        /// <summary>
+        /// Announces that this failure was the one that emptied the group's shared budget. Only ever
+        /// called by the caller that won the claim, so the event is raised once per exhaustion.
+        /// </summary>
+        public void RaiseBudgetExhausted(int subscriptionId, Guid groupId, string groupName,
+            int maxAttemptsTotal)
+        {
+            Events.Add(new RetryBudgetExhaustedEvent
+            {
+                XchangeId = Id,
+                SubscriptionId = subscriptionId,
+                GroupId = groupId,
+                GroupName = groupName,
+                MaxAttemptsTotal = maxAttemptsTotal,
+                OccurredOn = DateTime.UtcNow
+            });
+        }
     }
 }

SW.Bitween.Api/Resources/RetryPolicies/Attempts.cs

@@ -0,0 +1,97 @@
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using SW.Bitween.Domain;
+using SW.Bitween.Domain.Accounts;
+using SW.Bitween.Model;
+using SW.PrimitiveTypes;
+
+namespace SW.Bitween.Resources.RetryPolicies;
+
+/// <summary>
+/// Lists the failures one group caught for one subscription — what a row of <see cref="Usage"/>
+/// spent its budget on.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Separate from <see cref="Usage"/> and asked for one pair at a time, because a policy with fifty
+/// subscriptions would otherwise pay for fifty of these joins to answer a question about one row.
+/// </para>
+/// <para>
+/// Only failures carrying a group id appear, so nothing recorded before the group was stamped onto
+/// results is listed. A pair whose counter is well spent can therefore come back empty, which is
+/// why <see cref="RetryGroupAttempts.Total"/> is the count of what is listable rather than the
+/// counter's own value.
+/// </para>
+/// </remarks>
+[HandlerName("attempts")]
+public class Attempts : ICommandHandler<int, RetryGroupAttemptsRequest, object>
+{
+    /// <summary>
+    /// Enough to show what keeps failing without turning one table row into a page. The caller is
+    /// told the total, so a short list never reads as the whole story.
+    /// </summary>
+    private const int Limit = 10;
+
+    private readonly BitweenDbContext _dbContext;
+    private readonly RequestContext _requestContext;
+
+    public Attempts(BitweenDbContext dbContext, RequestContext requestContext)
+    {
+        _dbContext = dbContext;
+        _requestContext = requestContext;
+    }
+
+    public async Task<object> Handle(int key, RetryGroupAttemptsRequest request)
+    {
+        _requestContext.EnsureAccess(AccountRole.Admin, AccountRole.Member);
+
+        // Both halves of the pair have to belong to the policy in the route. For the subscription
+        // that keeps this from becoming a way to read any subscription's failures through any
+        // policy id; for the group it is about the answer being readable — an unknown group would
+        // otherwise report zero failures, which is indistinguishable from a group that genuinely
+        // has none.
+        var policy = await _dbContext.Set<RetryPolicy>().AsNoTracking()
+            .FirstOrDefaultAsync(p => p.Id == key);
+        if (policy == null) throw new SWNotFoundException(key.ToString());
+
+        if (policy.Groups.All(g => g.Id != request.GroupId))
+            throw new SWNotFoundException($"{key}/{request.GroupId}");
+
+        var belongs = await _dbContext.Set<Subscription>().AsNoTracking()
+            .AnyAsync(s => s.Id == request.SubscriptionId && s.RetryPolicyId == key);
+        if (!belongs) throw new SWNotFoundException($"{key}/{request.SubscriptionId}");
+
+        var query = from result in _dbContext.Set<XchangeResult>()
+            join xchange in _dbContext.Set<Xchange>() on result.Id equals xchange.Id
+            join pending in _dbContext.Set<DelayedRetry>() on result.Id equals pending.Id into scheduled
+            from pending in scheduled.DefaultIfEmpty()
+            where xchange.SubscriptionId == request.SubscriptionId
+                  && result.RetryGroupId == request.GroupId
+            select new RetryGroupAttemptRow
+            {
+                XchangeId = result.Id,
+                AttemptNumber = result.AttemptNumber,
+                FailedOn = result.FinishedOn,
+                Exception = result.Exception,
+                // A row survives here only until its retry runs, which is what separates a failure
+                // still being worked on from one that has been given up.
+                RetryPending = pending != null,
+                RetryBlockedReason = result.RetryBlockedReason
+            };
+
+        query = query.AsNoTracking();
+
+        return new RetryGroupAttempts
+        {
+            Total = await query.CountAsync(),
+            // Pending first, so the ones still moving cannot be pushed out of the list by a long
+            // history of failures that are already over.
+            Attempts = await query
+                .OrderByDescending(r => r.RetryPending)
+                .ThenByDescending(r => r.FailedOn)
+                .Take(Limit)
+                .ToListAsync()
+        };
+    }
+}

SW.Bitween.Api/Resources/RetryPolicies/Create.cs

@@ -21,11 +21,20 @@ public async Task<object> Handle(RetryPolicyCreate model)
     {
         _requestContext.EnsureAccess(AccountRole.Admin, AccountRole.Member);
         RetryGroupValidation.EnsureCanFire(model.Groups);
+        RetryGroupValidation.EnsureAlertTransportIsSecure(
+            model.AlertHandlerId, model.AlertHandlerProperties);
+
+        // A new policy has nothing stored behind a sentinel, so any that arrives — from a policy
+        // copied out of Get, say — is dropped rather than saved as the literal password.
+        foreach (var group in model.Groups ?? [])
+            AdapterSecretProperties.MergeInPlace(null, group.AlertHandlerProperties);
 
         var entity = new RetryPolicy
         {
             Name = model.Name,
-            Groups = model.Groups ?? []
+            Groups = model.Groups ?? [],
+            AlertHandlerId = model.AlertHandlerId,
+            AlertHandlerProperties = AdapterSecretProperties.Merge(null, model.AlertHandlerProperties)
         };
         _dbContext.Add(entity);
         await _dbContext.SaveChangesAsync();

SW.Bitween.Api/Resources/RetryPolicies/Delete.cs

@@ -33,13 +33,24 @@ public async Task<object> Handle(int key)
         var policy = await _dbContext.FindAsync<RetryPolicy>(key);
         var groupIds = policy.Groups.Select(g => g.Id).ToList();
 
+        // One change, one commit — same reasoning as Update: a half-done delete leaves rows keyed
+        // by groups that no longer exist anywhere, which nothing can then reach.
+        await using var transaction = await _dbContext.Database.BeginTransactionAsync();
+
         await _dbContext.DeleteByKeyAsync<RetryPolicy>(key);
 
         if (groupIds.Count > 0)
+        {
             await _dbContext.Set<RetryGroupUsage>()
                 .Where(u => groupIds.Contains(u.GroupId))
                 .ExecuteDeleteAsync();
 
+            await _dbContext.Set<RetryAlertOverride>()
+                .Where(o => groupIds.Contains(o.GroupId))
+                .ExecuteDeleteAsync();
+        }
+
+        await transaction.CommitAsync();
         return null;
     }
 }