From f64e59f1d709773216cc8b7e0adeb1200fe0e4ec Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 26 May 2026 11:05:39 +0300 Subject: [PATCH 01/33] little fix --- src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs | 3 +++ src/Sa.Outbox.PostgreSql/Services/OutboxTaskLoader.cs | 4 +++- .../Services/Plug/OutboxDeliveryManager.cs | 1 + src/Sa.Outbox/Delivery/DeliveryTenant.cs | 2 +- src/Sa.Schedule/IJobScheduler.cs | 2 +- 5 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs b/src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs index 3f069bf0..2a99fe9e 100644 --- a/src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs +++ b/src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs @@ -11,6 +11,9 @@ public sealed record LoadGroupResult(int CopiedRows, Guid NewOffset) internal interface IOutboxTaskLoader { + /// + /// Подгружаеи новые задания для консьюмера из таблицы вх. сообщений _msg$ + /// Task LoadNewTasks( OutboxMessageFilter filter, int batchSize, CancellationToken cancellationToken = default); } diff --git a/src/Sa.Outbox.PostgreSql/Services/OutboxTaskLoader.cs b/src/Sa.Outbox.PostgreSql/Services/OutboxTaskLoader.cs index 65796e74..df747b26 100644 --- a/src/Sa.Outbox.PostgreSql/Services/OutboxTaskLoader.cs +++ b/src/Sa.Outbox.PostgreSql/Services/OutboxTaskLoader.cs @@ -10,7 +10,9 @@ namespace Sa.Outbox.PostgreSql.Services; - +/// +/// Подгружаеи новые задания для консьюмера из таблицы вх. сообщений _msg$ +/// internal sealed partial class OutboxTaskLoader( IPgDataSource pg, SqlOutboxBuilder sql, diff --git a/src/Sa.Outbox.PostgreSql/Services/Plug/OutboxDeliveryManager.cs b/src/Sa.Outbox.PostgreSql/Services/Plug/OutboxDeliveryManager.cs index b7005e9c..b4f0b1b3 100644 --- a/src/Sa.Outbox.PostgreSql/Services/Plug/OutboxDeliveryManager.cs +++ b/src/Sa.Outbox.PostgreSql/Services/Plug/OutboxDeliveryManager.cs @@ -30,6 +30,7 @@ public async Task RentDelivery( var _ = await loader.LoadNewTasks(filter, batchSize, cancellationToken); + // новых заданий может и не быть... продолжаем обработку старых return await startCmd.ExecuteFill(writeBuffer, lockDuration, filter, cancellationToken); } diff --git a/src/Sa.Outbox/Delivery/DeliveryTenant.cs b/src/Sa.Outbox/Delivery/DeliveryTenant.cs index 4356996c..87d524b0 100644 --- a/src/Sa.Outbox/Delivery/DeliveryTenant.cs +++ b/src/Sa.Outbox/Delivery/DeliveryTenant.cs @@ -70,7 +70,7 @@ private async Task CalculateBatchSizeAsync( filter, cancellationToken); - return Math.Min(consumeSettings.MaxBatchSize, calculatedSize); + return Math.Clamp(calculatedSize, 0, consumeSettings.MaxBatchSize); } private async Task>> AcquireMessagesAsync( diff --git a/src/Sa.Schedule/IJobScheduler.cs b/src/Sa.Schedule/IJobScheduler.cs index b9842a97..78fce5bd 100644 --- a/src/Sa.Schedule/IJobScheduler.cs +++ b/src/Sa.Schedule/IJobScheduler.cs @@ -23,7 +23,7 @@ public interface IJobScheduler: IDisposable, IAsyncDisposable int ActiveTasks { get; } /// - /// + /// Consume instance count /// int ConcurrencyLimit { get; set; } From c35ad106325f5118a75de023fd30ac9327cc8b1e Mon Sep 17 00:00:00 2001 From: dundich Date: Thu, 28 May 2026 22:55:36 +0300 Subject: [PATCH 02/33] ~ Signed-off-by: dundich --- src/Sa.Outbox/Delivery/DelivarySnapshot.cs | 2 +- src/Sa.Outbox/Delivery/Job/DeliveryJob.cs | 2 +- src/Sa.Outbox/Delivery/Job/IDeliveryScheduleProvider.cs | 6 +++--- src/Sa.Outbox/Delivery/Job/JobPropertiesExtension.cs | 9 +++++++++ 4 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 src/Sa.Outbox/Delivery/Job/JobPropertiesExtension.cs diff --git a/src/Sa.Outbox/Delivery/DelivarySnapshot.cs b/src/Sa.Outbox/Delivery/DelivarySnapshot.cs index 033dab87..1a85db7a 100644 --- a/src/Sa.Outbox/Delivery/DelivarySnapshot.cs +++ b/src/Sa.Outbox/Delivery/DelivarySnapshot.cs @@ -28,7 +28,7 @@ internal sealed class DelivarySnapshot( private readonly Lazy _lazyDeliveries = new(() => { ConsumerGroupSettings[] settings = [.. scheduleSettings.GetJobSettings() - .Select(c => c.Properties.Tag as ConsumerGroupSettings) + .Select(c => c.Properties.GetConsumerGroupSettings()) .Where(mt => mt != null) .Cast()]; diff --git a/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs b/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs index 76397cab..56f5fa0f 100644 --- a/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs +++ b/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs @@ -10,7 +10,7 @@ internal sealed class DeliveryJob(IDeliveryProcessor processor) : IDel { public async Task Execute(IJobContext context, CancellationToken cancellationToken) { - ConsumerGroupSettings settings = context.Settings.Properties.Tag as ConsumerGroupSettings + ConsumerGroupSettings settings = context.Settings.Properties.GetConsumerGroupSettings() ?? throw new NotImplementedException("tag"); await processor.ProcessMessages(settings, cancellationToken); diff --git a/src/Sa.Outbox/Delivery/Job/IDeliveryScheduleProvider.cs b/src/Sa.Outbox/Delivery/Job/IDeliveryScheduleProvider.cs index 28b27938..9bf8d8d7 100644 --- a/src/Sa.Outbox/Delivery/Job/IDeliveryScheduleProvider.cs +++ b/src/Sa.Outbox/Delivery/Job/IDeliveryScheduleProvider.cs @@ -2,10 +2,10 @@ namespace Sa.Outbox.Delivery.Job; +/// +/// Manages job scheduling +/// public interface IDeliveryScheduleProvider { IJobScheduler GetJob(Guid jobId); - - int GetInstanceCount(Guid jobId) => GetJob(jobId).ConcurrencyLimit; - void SetInstanceCount(Guid jobId, int count) => GetJob(jobId).ConcurrencyLimit = count; } diff --git a/src/Sa.Outbox/Delivery/Job/JobPropertiesExtension.cs b/src/Sa.Outbox/Delivery/Job/JobPropertiesExtension.cs new file mode 100644 index 00000000..1540c98b --- /dev/null +++ b/src/Sa.Outbox/Delivery/Job/JobPropertiesExtension.cs @@ -0,0 +1,9 @@ +using Sa.Schedule; + +namespace Sa.Outbox.Delivery.Job; + +internal static class JobPropertiesExtension +{ + public static ConsumerGroupSettings? GetConsumerGroupSettings(this IJobProperties properties) + => properties?.Tag as ConsumerGroupSettings; +} From add525c15a7e9549f492e7092151e7dcd72c2493 Mon Sep 17 00:00:00 2001 From: dundich Date: Thu, 25 Jun 2026 14:09:25 +0300 Subject: [PATCH 03/33] refactor classes Signed-off-by: dundich --- .gitignore | 3 +- src/Sa.Outbox/Delivery/DeliveryTenant.cs | 4 +- src/Sa/Classes/AsyncManualResetEvent.cs | 66 ---- src/Sa/Classes/Levenshtein.cs | 11 +- src/Sa/Classes/LockRenewer.cs | 24 +- src/Sa/Classes/MimeTypeMap.cs | 11 +- src/Sa/Classes/Retry.cs | 2 +- src/Sa/Extensions/DateTimeExtensions.cs | 20 +- src/Sa/Extensions/EnumerableExtensions.cs | 36 ++- src/Sa/Extensions/ExceptionExtensions.cs | 9 +- src/Sa/Extensions/JsonExtensions.cs | 1 - src/Sa/Extensions/SpanExtensions.cs | 20 +- src/Sa/Extensions/StrToExtensions.cs | 71 +++-- src/Sa/Extensions/StringExtensions.cs | 128 ++++++-- .../Classes/AsyncManualResetEventTests.cs | 283 ------------------ src/Tests/SaTests/Classes/LockRenewerTests.cs | 10 +- src/Tests/SaTests/Classes/ResetLazyTests.cs | 2 +- 17 files changed, 251 insertions(+), 450 deletions(-) delete mode 100644 src/Sa/Classes/AsyncManualResetEvent.cs delete mode 100644 src/Tests/SaTests/Classes/AsyncManualResetEventTests.cs diff --git a/.gitignore b/.gitignore index bc12bcb8..005d817d 100644 --- a/.gitignore +++ b/.gitignore @@ -801,4 +801,5 @@ src/.vscode/ /src/Sa.Media.FFmpeg/build/artifacts/ /src/Sa.Media.FFmpeg/build/build.*/ -*.lscache \ No newline at end of file +*.lscache +.qwen/settings.json diff --git a/src/Sa.Outbox/Delivery/DeliveryTenant.cs b/src/Sa.Outbox/Delivery/DeliveryTenant.cs index 87d524b0..c9082c94 100644 --- a/src/Sa.Outbox/Delivery/DeliveryTenant.cs +++ b/src/Sa.Outbox/Delivery/DeliveryTenant.cs @@ -39,7 +39,7 @@ public async Task ProcessInTenant( if (messages.IsEmpty) return 0; - using IDisposable locker = RenewerLocker(settings.ConsumeSettings, filter, cancellationToken); + await using IAsyncDisposable locker = RenewerLocker(settings.ConsumeSettings, filter, cancellationToken); var successfulDeliveries = await deliveryCourier.Deliver(settings, filter, messages, cancellationToken); @@ -100,7 +100,7 @@ private Task ReleaseMessagesAsync( cancellationToken); } - private IDisposable RenewerLocker( + private IAsyncDisposable RenewerLocker( ConsumeSettings settings, OutboxMessageFilter filter, CancellationToken cancellationToken) diff --git a/src/Sa/Classes/AsyncManualResetEvent.cs b/src/Sa/Classes/AsyncManualResetEvent.cs deleted file mode 100644 index 397aae6a..00000000 --- a/src/Sa/Classes/AsyncManualResetEvent.cs +++ /dev/null @@ -1,66 +0,0 @@ -namespace Sa.Classes; - - -internal sealed class AsyncManualResetEvent -{ - private readonly Lock _syncRoot = new(); - private TaskCompletionSource _tcs = new(TaskCreationOptions.RunContinuationsAsynchronously); - private bool _isSignaled; - - public AsyncManualResetEvent(bool initialSet = false) - { - _isSignaled = initialSet; - if (initialSet) - { - _tcs.TrySetResult(); - } - } - - public Task WaitAsync(CancellationToken cancellationToken = default) - { - lock (_syncRoot) - { - if (_isSignaled) - { - return Task.CompletedTask; - } - - return _tcs.Task.WaitAsync(cancellationToken); - } - } - - public void Set() - { - lock (_syncRoot) - { - if (!_isSignaled) - { - _isSignaled = true; - _tcs.TrySetResult(); - } - } - } - - public void Reset() - { - lock (_syncRoot) - { - if (_isSignaled) - { - _isSignaled = false; - _tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - } - } - } - - public bool IsSet - { - get - { - lock (_syncRoot) - { - return _isSignaled; - } - } - } -} diff --git a/src/Sa/Classes/Levenshtein.cs b/src/Sa/Classes/Levenshtein.cs index 03f996ac..6fd0d64b 100644 --- a/src/Sa/Classes/Levenshtein.cs +++ b/src/Sa/Classes/Levenshtein.cs @@ -12,7 +12,7 @@ namespace Sa.Classes; internal static class Levenshtein { /// - /// Compares the two values to find the minimum Damerau-Levenshtein distance. + /// Compares the two values to find the minimum Damerau-Levenshtein distance. /// Thread safe and memory efficient. /// /// First string to compare @@ -141,9 +141,8 @@ public static IEnumerable> FindMatches( double similarityThreshold = 0.8, bool isNormalize = true) { - - string? sourceString = isNormalize && source is not null - ? source.Trim().NormalizeWhiteSpace().ToLower() + string? sourceNormalized = isNormalize && source is not null + ? source.NormalizeWhiteSpace(isTrimmed: true).ToLowerInvariant() : source; foreach (T? target in targetObjects) @@ -151,9 +150,9 @@ public static IEnumerable> FindMatches( string? targetString = targetStringSelector(target); if (isNormalize && targetString is not null) - targetString = targetString.Trim().NormalizeWhiteSpace().ToLower(); + targetString = targetString.NormalizeWhiteSpace(isTrimmed: true).ToLowerInvariant(); - var similarity = GetSimilarity(sourceString, targetString); + var similarity = GetSimilarity(sourceNormalized, targetString); if (similarity >= similarityThreshold) { diff --git a/src/Sa/Classes/LockRenewer.cs b/src/Sa/Classes/LockRenewer.cs index c601e0da..a005c92c 100644 --- a/src/Sa/Classes/LockRenewer.cs +++ b/src/Sa/Classes/LockRenewer.cs @@ -4,7 +4,7 @@ namespace Sa.Classes; internal static class LockRenewer { - public static IDisposable KeepLocked( + public static IAsyncDisposable KeepLocked( TimeSpan lockExpiration, Func extendLocked, bool blockImmediately = false, @@ -31,17 +31,22 @@ public static IDisposable KeepLocked( } }, cancellationToken); - IDisposable keeper = new DisposableTimer(timer, task); - - return keeper; + return new DisposableTimer(timer, task); } - private sealed class DisposableTimer(PeriodicTimer timer, Task task) : IDisposable + private sealed class DisposableTimer(PeriodicTimer Timer, Task Task) : IDisposable, IAsyncDisposable { public void Dispose() { - timer.Dispose(); - task.Wait(); // Ожидание завершения задачи перед освобождением ресурсов + Timer.Dispose(); + // Fire-and-forget wait — avoids blocking the caller on task completion + _ = Task.ContinueWith(_ => { }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + } + + public async ValueTask DisposeAsync() + { + Timer.Dispose(); + await Task; } } @@ -54,6 +59,7 @@ public static async Task WaitForConditionAsync( { var interval = pollInterval ?? TimeSpan.FromMilliseconds(10); var sw = Stopwatch.StartNew(); + var timer = new PeriodicTimer(interval); try { @@ -65,10 +71,10 @@ public static async Task WaitForConditionAsync( return true; } - await Task.Delay(interval, cancellationToken); + await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false); } } - catch (OperationCanceledException ex) when (ex.CancellationToken == cancellationToken) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { /* ignore */ } diff --git a/src/Sa/Classes/MimeTypeMap.cs b/src/Sa/Classes/MimeTypeMap.cs index ecbdc289..00ae9a62 100644 --- a/src/Sa/Classes/MimeTypeMap.cs +++ b/src/Sa/Classes/MimeTypeMap.cs @@ -9,14 +9,15 @@ internal static class MimeTypeMap private const string Dot = "."; private const string QuestionMark = "?"; private const string DefaultMimeType = "application/octet-stream"; - private static readonly Lazy> _mappings = new(BuildMappings); + + private static readonly Dictionary _mappings = BuildMappings(); private static Dictionary BuildMappings() { var mappings = new Dictionary(StringComparer.OrdinalIgnoreCase) { #region Big freaking list of mime types - + // maps both ways, // extension -> mime type // and @@ -26,7 +27,7 @@ private static Dictionary BuildMappings() // some mime types can map to multiple extensions, so to get a deterministic mapping, // add those to the dictionary specifically // - // combination of values from Windows 7 Registry and + // combination of values from Windows 7 Registry and // from C:\Windows\System32\inetsrv\config\applicationHost.config // some added, including .7z and .dat // @@ -781,7 +782,7 @@ public static bool TryGetMimeType(string str, out string? mimeType) str = Dot + str; } - return _mappings.Value.TryGetValue(str, out mimeType); + return _mappings.TryGetValue(str, out mimeType); } /// @@ -812,7 +813,7 @@ public static string GetExtension(string mimeType, bool throwErrorIfNotFound = t throw new ArgumentException("Requested mime type is not valid: " + mimeType); } - if (_mappings.Value.TryGetValue(mimeType, out string? extension)) + if (_mappings.TryGetValue(mimeType, out string? extension)) { return extension; } diff --git a/src/Sa/Classes/Retry.cs b/src/Sa/Classes/Retry.cs index 191efddd..7e98413c 100644 --- a/src/Sa/Classes/Retry.cs +++ b/src/Sa/Classes/Retry.cs @@ -220,7 +220,7 @@ private static async Task Wait(TimeSpan delay, CancellationToken cancellationTok } catch (TaskCanceledException) { - // ignore + // ignore } } diff --git a/src/Sa/Extensions/DateTimeExtensions.cs b/src/Sa/Extensions/DateTimeExtensions.cs index d84ed771..6eb9805a 100644 --- a/src/Sa/Extensions/DateTimeExtensions.cs +++ b/src/Sa/Extensions/DateTimeExtensions.cs @@ -1,29 +1,31 @@ using System.Diagnostics; +using System.Runtime.CompilerServices; namespace Sa.Extensions; internal static class DateTimeExtensions { /// - /// Unix timestamp + /// Unix timestamp. Skips when the value is already UTC. /// - [DebuggerStepThrough] + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] public static long ToUnixTimestamp(this DateTime dateTime, bool isInMilliseconds = false) { - TimeSpan ts = dateTime.ToUniversalTime().Subtract(DateTime.UnixEpoch); + var dt = dateTime.Kind == DateTimeKind.Utc ? dateTime : dateTime.ToUniversalTime(); + var ts = dt.Subtract(DateTime.UnixEpoch); return isInMilliseconds ? (long)ts.TotalMilliseconds : (long)ts.TotalSeconds; } - [DebuggerStepThrough] + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] public static DateTimeOffset StartOfDay(this DateTimeOffset dateTime) => new(dateTime.Year, dateTime.Month, dateTime.Day, 0, 0, 0, 0, dateTime.Offset); - [DebuggerStepThrough] + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] public static DateTimeOffset EndOfDay(this DateTimeOffset dateTime) => dateTime.StartOfDay().AddDays(1); - [DebuggerStepThrough] + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] public static DateTimeOffset StartOfMonth(this DateTimeOffset dateTime) => new(dateTime.Year, dateTime.Month, 1, 0, 0, 0, 0, dateTime.Offset); - [DebuggerStepThrough] + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] public static DateTimeOffset EndOfMonth(this DateTimeOffset dateTime) => dateTime.StartOfMonth().AddMonths(1); - [DebuggerStepThrough] + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] public static DateTimeOffset StartOfYear(this DateTimeOffset dateTime) => new(dateTime.Year, 1, 1, 0, 0, 0, 0, dateTime.Offset); - [DebuggerStepThrough] + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] public static DateTimeOffset EndOfYear(this DateTimeOffset dateTime) => dateTime.StartOfYear().AddYears(1); } diff --git a/src/Sa/Extensions/EnumerableExtensions.cs b/src/Sa/Extensions/EnumerableExtensions.cs index 25d1bb6b..1d72c881 100644 --- a/src/Sa/Extensions/EnumerableExtensions.cs +++ b/src/Sa/Extensions/EnumerableExtensions.cs @@ -1,27 +1,61 @@ using System.Diagnostics; +using System.Runtime.CompilerServices; namespace Sa.Extensions; internal static class EnumerableExtensions { - [DebuggerStepThrough] + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] public static string JoinByString(this IEnumerable source, string? joinWith = null) { if (source == null) return default!; return string.Join(joinWith, source); } + /// + /// Maps and joins elements with minimal allocations. Uses when available + /// to pre-allocate, otherwise falls back to string.Join. + /// [DebuggerStepThrough] public static string JoinByString(this IEnumerable source, Func map, string? joinWith = null) { if (source == null) return default!; + + // Fast path: if source is also ICollection, use the count hint + if (source is ICollection coll) + { + var arr = new T[coll.Count]; + coll.CopyTo(arr, 0); + int i = 0; + foreach (var item in arr) + { + arr[i++] = map(item); + } + return string.Join(joinWith, arr); + } + return string.Join(joinWith, source.Select(map)); } + /// + /// Maps with index and joins elements. Uses when available. + /// [DebuggerStepThrough] public static string JoinByString(this IEnumerable source, Func map, string? joinWith = null) { if (source == null) return default!; + + if (source is ICollection coll) + { + var arr = new T[coll.Count]; + coll.CopyTo(arr, 0); + for (int i = 0; i < arr.Length; i++) + { + arr[i] = map(arr[i], i); + } + return string.Join(joinWith, arr); + } + return string.Join(joinWith, source.Select(map)); } } diff --git a/src/Sa/Extensions/ExceptionExtensions.cs b/src/Sa/Extensions/ExceptionExtensions.cs index 63c41f06..3b0c7372 100644 --- a/src/Sa/Extensions/ExceptionExtensions.cs +++ b/src/Sa/Extensions/ExceptionExtensions.cs @@ -22,11 +22,12 @@ public static bool IsCritical(this Exception ex) [DebuggerStepThrough] public static string GetErrorMessages(this Exception exception) { - StringBuilder sb = new(); - sb.AppendLine(exception.Message); - if (exception.InnerException != null) + var sb = new StringBuilder(exception.Message.Length + 64); + var current = exception; + while (current != null) { - sb.AppendLine(GetErrorMessages(exception.InnerException)); + sb.AppendLine(current.Message); + current = current.InnerException; } return sb.ToString(); } diff --git a/src/Sa/Extensions/JsonExtensions.cs b/src/Sa/Extensions/JsonExtensions.cs index dbef7d60..3a852885 100644 --- a/src/Sa/Extensions/JsonExtensions.cs +++ b/src/Sa/Extensions/JsonExtensions.cs @@ -14,7 +14,6 @@ public static string ToJson(this T value, JsonSerializerOptions? options = nu return JsonSerializer.Serialize(value, options); } - [DebuggerStepThrough] [RequiresDynamicCode(JsonHttpResultTrimmerWarning.SerializationRequiresDynamicCodeMessage)] [RequiresUnreferencedCode(JsonHttpResultTrimmerWarning.SerializationUnreferencedCodeMessage)] diff --git a/src/Sa/Extensions/SpanExtensions.cs b/src/Sa/Extensions/SpanExtensions.cs index c3b2bcd1..c41c987c 100644 --- a/src/Sa/Extensions/SpanExtensions.cs +++ b/src/Sa/Extensions/SpanExtensions.cs @@ -9,17 +9,33 @@ public static IEnumerable> GetChunks(this Memory arr, int chunkS { for (int i = 0; i < arr.Length; i += chunkSize) { - // by slice Memory chunk = arr[i..Math.Min(i + chunkSize, arr.Length)]; yield return chunk; } } + /// + /// Same as but returns a materialized array with pre-allocated capacity. + /// + [DebuggerStepThrough] + public static Memory[] GetChunksArray(this Memory arr, int chunkSize) + { + int count = (arr.Length + chunkSize - 1) / chunkSize; + var result = new Memory[count]; + int idx = 0; + for (int i = 0; i < arr.Length; i += chunkSize) + { + int len = Math.Min(chunkSize, arr.Length - i); + result[idx++] = arr.Slice(i, len); + } + return result; + } + /// /// Combines Select and Where with indexes into a single call for optimal /// performance. /// - /// + /// /// The input sequence to filter and select /// The transformation with index to apply before filtering. /// The predicate with index with which to filter result. diff --git a/src/Sa/Extensions/StrToExtensions.cs b/src/Sa/Extensions/StrToExtensions.cs index b8e81b4c..93586f14 100644 --- a/src/Sa/Extensions/StrToExtensions.cs +++ b/src/Sa/Extensions/StrToExtensions.cs @@ -1,74 +1,80 @@ using System.Diagnostics; using System.Globalization; +using System.Runtime.CompilerServices; using System.Text; namespace Sa.Extensions; internal static class StrToExtensions { + // ── bool ─────────────────────────────────────────────────────────────── + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool? StrToBool(this string? str) => str is not null && bool.TryParse(str.AsSpan(), out var r) ? r : null; [DebuggerStepThrough] - public static bool? StrToBool(this string? str) => bool.TryParse(str, out bool result) ? result : null; - - [DebuggerStepThrough] - public static bool? StrToBool(this ReadOnlySpan str) => bool.TryParse(str, out bool result) ? result : null; + public static bool? StrToBool(this ReadOnlySpan str) => bool.TryParse(str, out var r) ? r : null; + // ── int ──────────────────────────────────────────────────────────────── + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int? StrToInt(this string? str) => str is not null && int.TryParse(str.AsSpan(), CultureInfo.InvariantCulture, out var r) ? r : null; [DebuggerStepThrough] - public static int? StrToInt(this string? str) => int.TryParse(str, CultureInfo.InvariantCulture, out int result) ? result : null; - [DebuggerStepThrough] - public static int? StrToInt(this ReadOnlySpan str) => int.TryParse(str, CultureInfo.InvariantCulture, out int result) ? result : null; + public static int? StrToInt(this ReadOnlySpan str) => int.TryParse(str, CultureInfo.InvariantCulture, out var r) ? r : null; + // ── short ────────────────────────────────────────────────────────────── + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] + public static short? StrToShort(this string? str) => str is not null && short.TryParse(str.AsSpan(), CultureInfo.InvariantCulture, out var r) ? r : null; [DebuggerStepThrough] - public static short? StrToShort(this string? str) => short.TryParse(str, CultureInfo.InvariantCulture, out short result) ? result : null; - [DebuggerStepThrough] - public static short? StrToShort(this ReadOnlySpan str) => short.TryParse(str, CultureInfo.InvariantCulture, out short result) ? result : null; + public static short? StrToShort(this ReadOnlySpan str) => short.TryParse(str, CultureInfo.InvariantCulture, out var r) ? r : null; + // ── ushort ───────────────────────────────────────────────────────────── + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ushort? StrToUShort(this string? str) => str is not null && ushort.TryParse(str.AsSpan(), CultureInfo.InvariantCulture, out var r) ? r : null; [DebuggerStepThrough] - public static ushort? StrToUShort(this string? str) => ushort.TryParse(str, CultureInfo.InvariantCulture, out ushort result) ? result : null; - - [DebuggerStepThrough] - public static ushort? StrToUShort(this ReadOnlySpan str) => ushort.TryParse(str, CultureInfo.InvariantCulture, out ushort result) ? result : null; + public static ushort? StrToUShort(this ReadOnlySpan str) => ushort.TryParse(str, CultureInfo.InvariantCulture, out var r) ? r : null; + // ── long ─────────────────────────────────────────────────────────────── + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] + public static long? StrToLong(this string? str) => str is not null && long.TryParse(str.AsSpan(), CultureInfo.InvariantCulture, out var r) ? r : null; [DebuggerStepThrough] - public static long? StrToLong(this string? str) => long.TryParse(str, CultureInfo.InvariantCulture, out long result) ? result : null; - [DebuggerStepThrough] - public static long? StrToLong(this ReadOnlySpan str) => long.TryParse(str, CultureInfo.InvariantCulture, out long result) ? result : null; - + public static long? StrToLong(this ReadOnlySpan str) => long.TryParse(str, CultureInfo.InvariantCulture, out var r) ? r : null; - [DebuggerStepThrough] - public static ulong? StrToULong(this string? str) => ulong.TryParse(str, CultureInfo.InvariantCulture, out ulong result) ? result : null; + // ── ulong ────────────────────────────────────────────────────────────── + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ulong? StrToULong(this string? str) => str is not null && ulong.TryParse(str.AsSpan(), CultureInfo.InvariantCulture, out var r) ? r : null; [DebuggerStepThrough] - public static ulong? StrToULong(this ReadOnlySpan str) => ulong.TryParse(str, CultureInfo.InvariantCulture, out ulong result) ? result : null; + public static ulong? StrToULong(this ReadOnlySpan str) => ulong.TryParse(str, CultureInfo.InvariantCulture, out var r) ? r : null; - [DebuggerStepThrough] - public static double? StrToDouble(this string? str) => double.TryParse(str, CultureInfo.InvariantCulture, out double result) ? result : null; + // ── double ───────────────────────────────────────────────────────────── + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] + public static double? StrToDouble(this string? str) => str is not null && double.TryParse(str.AsSpan(), CultureInfo.InvariantCulture, out var r) ? r : null; [DebuggerStepThrough] - public static double? StrToDouble(this ReadOnlySpan str) => double.TryParse(str, CultureInfo.InvariantCulture, out double result) ? result : null; - + public static double? StrToDouble(this ReadOnlySpan str) => double.TryParse(str, CultureInfo.InvariantCulture, out var r) ? r : null; + // ── byte[] ───────────────────────────────────────────────────────────── [DebuggerStepThrough] public static byte[] StrToBytes(this string str, Encoding? encoding = null) => (encoding ?? Encoding.UTF8).GetBytes(str); + // ── Guid ─────────────────────────────────────────────────────────────── + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Guid? StrToGuid(this string? str) => str is not null && Guid.TryParse(str.AsSpan(), CultureInfo.InvariantCulture, out var r) ? r : null; [DebuggerStepThrough] - public static Guid? StrToGuid(this string? str) => Guid.TryParse(str, CultureInfo.InvariantCulture, out Guid result) ? result : null; - [DebuggerStepThrough] - public static Guid? StrToGuid(this ReadOnlySpan str) => Guid.TryParse(str, CultureInfo.InvariantCulture, out Guid result) ? result : null; - + public static Guid? StrToGuid(this ReadOnlySpan str) => Guid.TryParse(str, CultureInfo.InvariantCulture, out var r) ? r : null; + // ── Enum ─────────────────────────────────────────────────────────────── [DebuggerStepThrough] public static T StrToEnum(this string? str, T defaultValue) where T : struct => (Enum.TryParse(str, true, out T result)) ? result : defaultValue; - - [DebuggerStepThrough] + // ── DateTime ─────────────────────────────────────────────────────────── + [DebuggerStepThrough,MethodImpl(MethodImplOptions.AggressiveInlining)] public static DateTime? StrToDate(this string? str, IFormatProvider? provider = null, DateTimeStyles style = DateTimeStyles.None) - => DateTime.TryParseExact(str, DateFmt.Formats, provider ?? CultureInfo.InvariantCulture, style, out DateTime result) + => str is not null && DateTime.TryParseExact(str.AsSpan(), DateFmt.Formats, provider ?? CultureInfo.InvariantCulture, style, out DateTime result) ? result : null; @@ -77,7 +83,6 @@ internal static class StrToExtensions => DateTime.TryParseExact(str, DateFmt.Formats, provider ?? CultureInfo.InvariantCulture, style, out DateTime result) ? result : null; - } #region Date Fmts diff --git a/src/Sa/Extensions/StringExtensions.cs b/src/Sa/Extensions/StringExtensions.cs index 0fa0f005..7c333be7 100644 --- a/src/Sa/Extensions/StringExtensions.cs +++ b/src/Sa/Extensions/StringExtensions.cs @@ -1,54 +1,138 @@ -using System.Diagnostics; +using Sa.Classes; +using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Text; -using Sa.Classes; namespace Sa.Extensions; internal static partial class StringExtensions { - [DebuggerStepThrough] + /// + /// Returns unless it is null, empty, or consists entirely of whitespace — in which cases returns null. + /// + [DebuggerStepThrough, MethodImpl(MethodImplOptions.AggressiveInlining)] public static string? NullIfEmpty(this string? str) - => string.IsNullOrWhiteSpace(str) ? default : str; + { + if (string.IsNullOrEmpty(str)) return null; + // Single-pass whitespace check — avoids a separate IsOnlyWhitespace call +#pragma warning disable S3267 + foreach (char c in str) + { + if (!char.IsWhiteSpace(c)) return str; + } +#pragma warning restore S3267 + return null; + } + /// + /// Replaces all consecutive white-space characters with a single space. Zero-allocation variant available via . + /// [DebuggerStepThrough] - public static uint GetMurmurHash3(this string str, uint seed = 0) - => MurmurHash3.Hash32(Encoding.UTF8.GetBytes(str), seed); +#pragma warning disable S3776 + public static string NormalizeWhiteSpace(this string? str, bool isTrimmed = true) +#pragma warning restore S3776 + { + if (string.IsNullOrEmpty(str)) return String.Empty; + + bool slowPath = false; + + int len = str.Length; + for (int i = 0; i < len; i++) + { + char c = str[i]; + if (char.IsWhiteSpace(c) || char.IsSeparator(c) || char.IsControl(c)) + { + slowPath = true; + break; + } + } + + if (!slowPath) + { + // No whitespace found — just trim and return + return isTrimmed ? str.Trim() : str; + } + + // Slow path: span-based normalization — allocates one new string, avoids StringBuilder heap churn + Span dest = stackalloc char[len]; + int w = 0; + bool prevWhite = false; + for (int i = 0; i < len; i++) + { + char c = str[i]; + if (char.IsWhiteSpace(c) || char.IsSeparator(c) || char.IsControl(c)) + { + if (!prevWhite) + { + dest[w++] = ' '; + prevWhite = true; + } + } + else + { + dest[w++] = c; + prevWhite = false; + } + } + string result = dest[..w].ToString(); + return isTrimmed ? result.Trim() : result; + } + /// + /// Span-based zero-allocation normalization that writes into . + /// Returns the number of characters written. + /// [DebuggerStepThrough] - public static string NormalizeWhiteSpace(this string? str, bool isTrimmed = true) + public static int NormalizeWhiteSpaceSpan(ReadOnlySpan str, Span dest, bool isTrimmed = true) { - if (string.IsNullOrEmpty(str)) return String.Empty; + if (str.IsEmpty) return 0; if (isTrimmed) { str = str.Trim(); - if (str.Length == 0) - return string.Empty; + if (str.IsEmpty) return 0; } - var sb = new StringBuilder(str.Length); - bool previousWasWhiteSpace = false; - - foreach (char c in str) + int len = str.Length; + int w = 0; + bool prevWhite = false; + for (int i = 0; i < len; i++) { - if (char.IsWhiteSpace(c)) + char c = str[i]; + if (char.IsWhiteSpace(c) || char.IsSeparator(c) || char.IsControl(c)) { - if (!previousWasWhiteSpace) + if (!prevWhite) { - sb.Append(' '); - previousWasWhiteSpace = true; + dest[w++] = ' '; + prevWhite = true; } } else { - sb.Append(c); - previousWasWhiteSpace = false; + dest[w++] = c; + prevWhite = false; } } - string result = sb.ToString(); - return isTrimmed ? result.Trim() : result; + if (isTrimmed && w > 0 && dest[w - 1] == ' ') + w--; + + return w; + } + + /// + /// Computes MurmurHash3 for the UTF-8 encoding of without allocating a byte array. + /// + [DebuggerStepThrough] + public static uint GetMurmurHash3(this string str, uint seed = 0) + { + // Estimate UTF-8 byte length (upper bound: 3 bytes per char for BMP Latin, up to 4 for emoji) + int estimatedLen = str.Length * 3; + Span buf = estimatedLen <= 512 ? stackalloc byte[estimatedLen] : new byte[estimatedLen]; + + int actualLen = Encoding.UTF8.GetBytes(str, buf); + return MurmurHash3.Hash32(buf[..actualLen], seed); } } diff --git a/src/Tests/SaTests/Classes/AsyncManualResetEventTests.cs b/src/Tests/SaTests/Classes/AsyncManualResetEventTests.cs deleted file mode 100644 index 57d39c94..00000000 --- a/src/Tests/SaTests/Classes/AsyncManualResetEventTests.cs +++ /dev/null @@ -1,283 +0,0 @@ -using Sa.Classes; - -namespace SaTests.Classes; - -public class AsyncManualResetEventTests -{ - static CancellationToken TestToken => TestContext.Current.CancellationToken; - - [Fact] - public void Constructor_WithInitialSetTrue_ShouldBeSet() - { - // Arrange & Act - var resetEvent = new AsyncManualResetEvent(true); - - // Assert - Assert.True(resetEvent.IsSet); - } - - [Fact] - public void Constructor_WithInitialSetFalse_ShouldNotBeSet() - { - // Arrange & Act - var resetEvent = new AsyncManualResetEvent(false); - - // Assert - Assert.False(resetEvent.IsSet); - } - - [Fact] - public void Constructor_Default_ShouldNotBeSet() - { - // Arrange & Act - var resetEvent = new AsyncManualResetEvent(); - - // Assert - Assert.False(resetEvent.IsSet); - } - - [Fact] - public async Task WaitAsync_WhenSet_ReturnsCompletedTask() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(true); - - // Act - var task = resetEvent.WaitAsync(TestToken); - - // Assert - Assert.True(task.IsCompleted); - await task; // Should not throw - } - - [Fact] - public async Task WaitAsync_WhenNotSet_ReturnsIncompleteTask() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(false); - - // Act - var task = resetEvent.WaitAsync(TestToken); - - // Assert - Assert.False(task.IsCompleted); - - // Cleanup - resetEvent.Set(); - await task; - } - - [Fact] - public async Task WaitAsync_AfterSet_ReturnsCompletedTask() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(false); - - // Act - resetEvent.Set(); - var task = resetEvent.WaitAsync(TestToken); - - // Assert - Assert.True(task.IsCompleted); - await task; - } - - [Fact] - public async Task WaitAsync_MultipleWaiters_AllCompleteWhenSet() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(false); - const int waiterCount = 10; - var tasks = new Task[waiterCount]; - - // Act - for (int i = 0; i < waiterCount; i++) - { - tasks[i] = resetEvent.WaitAsync(TestToken); - } - - // Assert - all tasks should be incomplete - foreach (var task in tasks) - { - Assert.False(task.IsCompleted); - } - - // Act - set the event - resetEvent.Set(); - - // Assert - all tasks should complete - await Task.WhenAll(tasks); - foreach (var task in tasks) - { - Assert.True(task.IsCompleted); - } - } - - [Fact] - public async Task Set_WhenAlreadySet_DoesNothing() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(true); - - // Act - should not throw - resetEvent.Set(); - resetEvent.Set(); - - // Assert - Assert.True(resetEvent.IsSet); - await resetEvent.WaitAsync(TestToken); // Should complete immediately - } - - [Fact] - public void Reset_WhenSet_ClearsTheSignal() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(true); - - // Act - resetEvent.Reset(); - - // Assert - Assert.False(resetEvent.IsSet); - } - - [Fact] - public void Reset_WhenNotSet_RemainsNotSet() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(false); - - // Act - resetEvent.Reset(); - - // Assert - Assert.False(resetEvent.IsSet); - } - - [Fact] - public async Task WaitAsync_WithCancellationToken_CanBeCancelled() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(false); - using var cts = new CancellationTokenSource(); - - // Act - var task = resetEvent.WaitAsync(cts.Token); - - // Assert - task should be waiting - Assert.False(task.IsCompleted); - - // Act - cancel - await cts.CancelAsync(); - - // Assert - task should be cancelled - await Assert.ThrowsAsync(() => task); - } - - [Fact] - public async Task WaitAsync_AfterReset_ShouldWaitAgain() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(true); - - // Act - first wait should complete immediately - await resetEvent.WaitAsync(TestToken); - - // Reset and wait again - resetEvent.Reset(); - var secondWaitTask = resetEvent.WaitAsync(TestToken); - - // Assert - second wait should not complete - Assert.False(secondWaitTask.IsCompleted); - - // Cleanup - resetEvent.Set(); - await secondWaitTask; - } - - [Fact] - public async Task StressTest_MultipleSetResetCycles() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(false); - const int cycles = 100; - - for (int i = 0; i < cycles; i++) - { - // Act - set and verify - resetEvent.Set(); - await resetEvent.WaitAsync(TestToken); // Should complete immediately - Assert.True(resetEvent.IsSet); - - // Act - reset and verify - resetEvent.Reset(); - Assert.False(resetEvent.IsSet); - } - } - - [Fact] - public async Task ConcurrentAccess_MultipleThreads() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(false); - const int threadCount = 10; - var tasks = new Task[threadCount * 2]; // half waiters, half setters - var completedWaiters = 0; - - // Act - start multiple threads that wait and set - for (int i = 0; i < threadCount; i++) - { - // Waiter task - tasks[i] = Task.Run(async () => - { - await resetEvent.WaitAsync(TestToken); - Interlocked.Increment(ref completedWaiters); - }, TestToken); - - // Setter task (start slightly later) - tasks[i + threadCount] = Task.Run(async () => - { - await Task.Delay(10); - resetEvent.Set(); - }, TestToken); - } - - // Wait for all tasks - await Task.WhenAll(tasks); - - // Assert - all waiters should have completed - Assert.Equal(threadCount, completedWaiters); - Assert.True(resetEvent.IsSet); - } - - [Fact] - public async Task WaitAsync_WithAlreadyCancelledToken_ThrowsImmediately() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(false); - using var cts = new CancellationTokenSource(); - await cts.CancelAsync(); - - // Act & Assert - await Assert.ThrowsAsync(() => resetEvent.WaitAsync(cts.Token)); - } - - [Fact] - public async Task WaitAsync_AfterResetButBeforeSet_ShouldWait() - { - // Arrange - var resetEvent = new AsyncManualResetEvent(true); - - // Reset and start waiting - resetEvent.Reset(); - var waitTask = resetEvent.WaitAsync(TestToken); - - // Assert - should be waiting - Assert.False(waitTask.IsCompleted); - - // Act - set and verify completion - resetEvent.Set(); - await waitTask; - Assert.True(waitTask.IsCompleted); - } -} diff --git a/src/Tests/SaTests/Classes/LockRenewerTests.cs b/src/Tests/SaTests/Classes/LockRenewerTests.cs index 286621a3..c6fe8178 100644 --- a/src/Tests/SaTests/Classes/LockRenewerTests.cs +++ b/src/Tests/SaTests/Classes/LockRenewerTests.cs @@ -21,7 +21,8 @@ async Task extendLocked(CancellationToken token) } // Act - using (var locker = LockRenewer.KeepLocked(lockExpiration, extendLocked, cancellationToken: cancellationToken)) + await using (var locker = LockRenewer.KeepLocked( + lockExpiration, extendLocked, cancellationToken: cancellationToken)) { await Task.Delay(200, TestContext.Current.CancellationToken); // Give it some time to run await cancellationTokenSource.CancelAsync(); @@ -47,7 +48,8 @@ async Task extendLocked(CancellationToken token) } // Act - using (var locker = LockRenewer.KeepLocked(lockExpiration, extendLocked, blockImmediately: true, cancellationToken: cancellationToken)) + await using (var locker = LockRenewer.KeepLocked( + lockExpiration, extendLocked, blockImmediately: true, cancellationToken: cancellationToken)) { await Task.Delay(100, TestContext.Current.CancellationToken); // Give it some time to run await cancellationTokenSource.CancelAsync(); @@ -69,7 +71,7 @@ public async Task Dispose_ReleasesResources() async Task extendLocked(CancellationToken token) { await Task.Delay(TimeSpan.FromMilliseconds(10), token); // Simulate some work - extensionCount++; + Interlocked.Increment(ref extensionCount); } // Act @@ -77,7 +79,7 @@ async Task extendLocked(CancellationToken token) await Task.Delay(100, TestContext.Current.CancellationToken); - locker.Dispose(); + await locker.DisposeAsync(); var expected = extensionCount; diff --git a/src/Tests/SaTests/Classes/ResetLazyTests.cs b/src/Tests/SaTests/Classes/ResetLazyTests.cs index cbe77930..8565cfb1 100644 --- a/src/Tests/SaTests/Classes/ResetLazyTests.cs +++ b/src/Tests/SaTests/Classes/ResetLazyTests.cs @@ -37,7 +37,7 @@ public void Reset_ShouldClearValue_AndCallCleanup() ); // Act - var firstList = lazy.Value; + _ = lazy.Value; lazy.Reset(); // Assert From 806e3fd57a6c5c3839dc02cc72e0ab0ce0d036dd Mon Sep 17 00:00:00 2001 From: dundich Date: Thu, 25 Jun 2026 17:01:23 +0300 Subject: [PATCH 04/33] improve configuration Signed-off-by: dundich --- .gitignore | 6 +- QWEN.md | 140 +++++++++ .../DatabaseConfigurationProvider.cs | 11 +- src/Sa.Configuration.PostgreSql/Readme.md | 109 +++++-- src/Sa.Configuration/Readme.md | 286 ++++++++++++------ .../SecretStore/Engine/SecretService.cs | 2 +- .../DatabaseConfigurationExtensionsTests.cs | 2 +- .../DatabaseConfigurationProviderTests.cs | 119 ++++++++ .../ArgumentsConfigurationProviderTests.cs | 74 +++++ .../ChainedSecretStoreTests.cs | 124 ++++++++ .../CommandLineArgsSecretStoreTests.cs | 75 +++++ .../EnvironmentVariableSecretStoreTests.cs | 91 ++++++ 12 files changed, 920 insertions(+), 119 deletions(-) create mode 100644 QWEN.md create mode 100644 src/Tests/Sa.Configuration.PostgreSqlTests/DatabaseConfigurationProviderTests.cs create mode 100644 src/Tests/Sa.ConfigurationTests/ArgumentsConfigurationProviderTests.cs create mode 100644 src/Tests/Sa.ConfigurationTests/ChainedSecretStoreTests.cs create mode 100644 src/Tests/Sa.ConfigurationTests/CommandLineArgsSecretStoreTests.cs create mode 100644 src/Tests/Sa.ConfigurationTests/EnvironmentVariableSecretStoreTests.cs diff --git a/.gitignore b/.gitignore index 005d817d..5c33783c 100644 --- a/.gitignore +++ b/.gitignore @@ -802,4 +802,8 @@ src/.vscode/ /src/Sa.Media.FFmpeg/build/build.*/ *.lscache -.qwen/settings.json + +# ai +.qwen/ +.agents/ +/.github/skills diff --git a/QWEN.md b/QWEN.md new file mode 100644 index 00000000..b41f744b --- /dev/null +++ b/QWEN.md @@ -0,0 +1,140 @@ +# Sa — .NET 10 Experimental AOT Library Suite + +## Project Overview + +**Sa** is a collection of reusable .NET 10 libraries focused on infrastructure patterns for distributed systems. It targets **.NET 10.0**, uses **Native AOT**, and follows the **Central Package Management (CPM)** pattern via `Directory.Packages.props`. + +### Libraries + +| Library | Purpose | +|---|---| +| **Sa** | Shared utility classes (LockRenewer, MurmurHash3, Retry, extensions) consumed by other libs via `` | +| **Sa.Configuration** | Command-line argument parsing (`Arguments`) and secure secrets management from files/env vars/host key files | +| **Sa.Configuration.PostgreSql** | PostgreSQL-backed dynamic configuration source — changes in DB reflect in-app without redeploy | +| **Sa.Data.PostgreSql** | Lightweight Npgsql client wrapper | +| **Sa.Data.S3** | S3 data client (Minio-compatible) | +| **Sa.HybridFileStorage** | Hybrid file storage abstraction with automatic provider failover (FileSystem ↔ S3 ↔ Postgres) | +| **Sa.HybridFileStorage.FileSystem** | FileSystem provider implementation | +| **Sa.HybridFileStorage.S3** | S3 provider implementation | +| **Sa.HybridFileStorage.Postgres** | PostgreSQL provider implementation | +| **Sa.Media** | Async, memory-efficient WAV file reader (`AsyncWavReader`) | +| **Sa.Media.FFmpeg** | FFmpeg .NET wrapper with built-in binaries (Win x64 / Linux), audio conversion, metadata extraction, channel split/join, DI support | +| **Sa.Outbox** | Base Outbox pattern infrastructure for reliable message publishing | +| **Sa.Outbox.PostgreSql** | PostgreSQL Outbox implementation — parallel processing, tenant support, scheduled data cleanup | +| **Sa.Partitional.PostgreSql** | Declarative PostgreSQL table partitioning (time: day/month/year; list; range) with migration/deletion schedules | +| **Sa.Schedule** | Scheduled task executor with failure strategies (close app, stop job, stop all jobs, ignore) | +| **Sa.Utils.WorkQueue** | Async queue with concurrency limiting, built on `System.Threading.Channels` | + +### Samples + +Located in `src/Samples/`: Configuration.Web, FFMpeg.Console, HybridFileStorage.Console, Partitional.ConsoleApp, PgOutbox.ConsoleApp, Schedule.Console, Storage.Tests. + +### Tests + +Located in `src/Tests/`: 15 test projects using **xunit v3**, **Testcontainers** (PostgreSQL + Minio) for integration tests. Test fixtures in `src/Tests/Fixtures/`. + +--- + +## Building and Running + +### Prerequisites + +- .NET 10 SDK +- PowerShell (for local build scripts) + +### Build Commands + +```powershell +# Full build (clean + restore + build) +.\build\do_build.ps1 + +# Run all tests +.\build\do_test.ps1 + +# Package NuGet packages (produces .nupkg + .snupkg in dist/) +.\build\do_package.ps1 + +# Push to local registry +.\build\do_push_local.ps1 + +# Push to prod (nuget.org) +.\build\do_push_prod.ps1 +``` + +### Direct dotnet commands + +```powershell +# Restore +dotnet restore src/Sa.slnx -c Release + +# Build +dotnet build src/Sa.slnx -c Release -v n + +# Test (all) +dotnet test src/Sa.slnx -v n + +# Test CI (skip tests requiring local Docker infrastructure) +dotnet test src/Sa.slnx --filter "Category!=Local" +``` + +### GitHub Actions + +Workflow in `.github/workflows/` — builds on `main` branch push/PR. Uses `dotnet 9.x` runtime in CI (despite targeting net10.0). Note: tests are commented out in CI. + +--- + +## Architecture Notes + +### Shared Code Pattern + +Common utilities live in `src/Sa/` and are linked into consuming projects via MSBuild ``. This avoids duplication while keeping projects independently buildable. Linked classes include: + +- `Classes/`: LockRenewer, MurmurHash3, Retry, ResetLazy, Section, MimeTypeMap, IArrayPool, LockRenewer +- `Extensions/`: DateTimeExtensions, EnumerableExtensions, ExceptionExtensions, SpanExtensions, StringExtensions, NumericExtensions, StrToExtensions, GuidExtensions + +### Project Dependencies + +``` +Sa.Utils.WorkQueue → (none) +Sa.Schedule → Sa.Utils.WorkQueue +Sa.Outbox → Sa.Schedule +Sa.Partitional.PostgreSql → Sa.Schedule + Sa.Data.PostgreSql +Sa.Outbox.PostgreSql → Sa.Outbox + Sa.Partitional.PostgreSql (+ object pool, recycler mem stream) +Sa.Data.S3 → (none, just Npgsql indirectly) +Sa.HybridFileStorage → (base abstraction) +Sa.HybridFileStorage.S3 → Sa.Data.S3 + Sa.HybridFileStorage +Sa.Configuration → Microsoft.Extensions.Hosting +Sa.Configuration.PostgreSql → (standalone) +``` + +### Common Properties (inherited by all packages) + +From `Common.Properties.xml`: +- Target: `net10.0` +- AOT: `PublishAot=true`, `IsAotCompatible=true` +- Nullable: enabled +- Analyzers: enabled +- TrimmerSingleWarn: false +- Symbols: included +- License: MIT + +From `Common.NuGet.Properties.xml`: additional shared package references (logging, DI, SourceLink). + +### Testing Conventions + +- Framework: **xunit v3** (not classic xunit) +- Integration tests use **Testcontainers** (PostgreSQL + Minio) +- Test projects import `Host.Test.Properties.xml` for common test config +- Local-dependent tests are tagged `Category!=Local` for CI + +--- + +## Development Conventions + +- **ImplicitUsings** and **Nullable** enabled across all projects +- **Central Package Management** — all versions in `Directory.Packages.props` +- **SourceLink** enabled for debug symbol linking to GitHub +- **InternalsVisibleTo** used for test project access to internal members +- No `_editorconfig` rules beyond standard .NET conventions +- All projects use SDK-style csproj format +- Solution managed via `.slnx` (new solution format) diff --git a/src/Sa.Configuration.PostgreSql/DatabaseConfigurationProvider.cs b/src/Sa.Configuration.PostgreSql/DatabaseConfigurationProvider.cs index a033bf0c..b2f5ffb3 100644 --- a/src/Sa.Configuration.PostgreSql/DatabaseConfigurationProvider.cs +++ b/src/Sa.Configuration.PostgreSql/DatabaseConfigurationProvider.cs @@ -2,12 +2,13 @@ using Microsoft.Extensions.Configuration; using Sa.Data.PostgreSql; -using System.Threading; + /// /// Configuration provider that loads settings from a PostgreSQL database. /// -public sealed class DatabaseConfigurationProvider(PostgreSqlConfigurationOptions options) : ConfigurationProvider +public sealed class DatabaseConfigurationProvider(PostgreSqlConfigurationOptions options) + : ConfigurationProvider { /// /// Loads configuration from PostgreSQL database. @@ -15,16 +16,16 @@ public sealed class DatabaseConfigurationProvider(PostgreSqlConfigurationOptions public override void Load() { PgRetryStrategy - .ExecuteWithRetry(async _ => await LoadAsync(options)) + .ExecuteWithRetry(async _ => await LoadAsync()) .AsTask() .GetAwaiter() .GetResult(); } /// - /// Loads configuration from PostgreSQL database asynchronously. + /// Asynchronously loads configuration from PostgreSQL database. /// - private async Task LoadAsync(PostgreSqlConfigurationOptions options) + private async Task LoadAsync() { try { diff --git a/src/Sa.Configuration.PostgreSql/Readme.md b/src/Sa.Configuration.PostgreSql/Readme.md index f96a96f6..00c1586a 100644 --- a/src/Sa.Configuration.PostgreSql/Readme.md +++ b/src/Sa.Configuration.PostgreSql/Readme.md @@ -1,32 +1,103 @@ # Sa.Configuration.PostgreSql -The `AddPostgreSqlConfiguration` extension method allows you to add a PostgreSQL-based configuration source to an IConfigurationBuilder. This enables your application to load configuration settings directly from a PostgreSQL database. +A dynamic configuration source for .NET that loads settings from PostgreSQL. Changes in the database are applied to the running application without restart — just call `Reload()` on `IConfigurationRoot`. -## Key Components -- PostgreSqlConfigurationOptions: A record that holds the connection string, SQL query, and optional parameters for querying the database. -- DatabaseConfigurationSource: Implements IConfigurationSource and creates a DatabaseConfigurationProvider to fetch configuration data. -- DatabaseConfigurationProvider: Inherits from ConfigurationProvider and overrides the Load method to execute the SQL query and populate the configuration data. +## Features + +- **Live configuration**: values are stored in the database and can be changed at runtime +- **Parameterized SQL queries**: supports `@named_parameters` via `NpgsqlParameter` +- **Automatic retry**: built-in retry strategy (PgRetryStrategy) with detection of Npgsql transaction errors +- **Key/value trimming**: whitespace is automatically trimmed from both keys and values +- **Safe NULL handling**: `NULL` in DB → `null` in config; empty string → `string.Empty` + +## Public API + +| Type | Purpose | +|------|---------| +| `PostgreSqlConfigurationOptions` | Immutable record: `ConnectionString`, `SelectSql`, `Parameters` | +| `Setup.AddSaPostgreSqlConfiguration()` | Extension method for `IConfigurationBuilder` | + +## Quick Start -## Example Usage ```csharp -using Microsoft.Extensions.Configuration; using Sa.Configuration.PostgreSql; -var builder = new ConfigurationBuilder(); +var builder = WebApplication.CreateBuilder(args); + +builder.Configuration.AddSaPostgreSqlConfiguration(new PostgreSqlConfigurationOptions( + ConnectionString: "Host=localhost;Database=myapp;Username=app;Password=secret", + SelectSql: "SELECT key, value FROM app_settings" +)); + +var app = builder.Build(); + +// Reading settings +var theme = app.Configuration["theme"]; // → "dark" +var lang = app.Configuration["language"]; // → "en" +``` + +## Parameterized Queries + +Use `@parameters` for filtering by client/tenant: + +```csharp +builder.Configuration.AddSaPostgreSqlConfiguration(new PostgreSqlConfigurationOptions( + ConnectionString: "...", + SelectSql: "SELECT key, value FROM client_settings WHERE client_id = @client_id", + Parameters: [new NpgsqlParameter("client_id", "acme-corp")] +)); +``` + +## Live Configuration Updates -// Define PostgreSqlConfigurationOptions -var options = new PostgreSqlConfigurationOptions( - "Host=my_host;Database=my_db;Username=my_user;Password=my_pw", - "SELECT key, value FROM configuration" +When rows in the `app_settings` table change, the application can pick up new values: + +```csharp +// After modifying rows in the database: +((IConfigurationRoot)app.Configuration).Reload(); + +// Or manually: +provider.Reload(); // DatabaseConfigurationProvider implements IConfigurationProvider +``` + +## Load Behavior + +| Scenario | Result | +|----------|--------| +| Key is empty or whitespace only | Skipped | +| Value is `NULL` in DB | Stored as `null` | +| Value is an empty string in DB | Stored as `string.Empty` | +| Connection error | `InvalidOperationException` with the original exception as `InnerException` | + +## Table Schema + +Minimum table required for the provider: + +```sql +CREATE TABLE app_settings ( + key VARCHAR PRIMARY KEY, + value TEXT ); -// Add PostgreSQL configuration to the builder -builder.AddSaPostgreSqlConfiguration(options); +-- Sample data +INSERT INTO app_settings (key, value) VALUES + ('theme', 'dark'), + ('language', 'en'), + ('debug_mode', ''); -- empty string +``` + +## Dependencies + +- `Microsoft.Extensions.Configuration` +- `Sa.Data.PostgreSql` (Npgsql wrapper with PgRetryStrategy and IPgDataSource) -// Build the configuration -var configuration = builder.Build(); +## Project Layout -// Access configuration values -string setting1 = configuration["Setting1"]; -Console.WriteLine($"Setting1: {setting1}"); +``` +src/Sa.Configuration.PostgreSql/ +├── PostgreSqlConfigurationOptions.cs # Options record +├── DatabaseConfigurationSource.cs # IConfigurationSource +├── DatabaseConfigurationProvider.cs # ConfigurationProvider + retry +├── Setup.cs # Extension method AddSaPostgreSqlConfiguration() +└── Readme.md # ← you are here ``` diff --git a/src/Sa.Configuration/Readme.md b/src/Sa.Configuration/Readme.md index 880ac51d..5f5ea7b2 100644 --- a/src/Sa.Configuration/Readme.md +++ b/src/Sa.Configuration/Readme.md @@ -1,168 +1,270 @@ -# Working with Secrets via Configuration +# Sa.Configuration -The `Sa.Configuration` library provides **secure and transparent integration of secrets** into the standard .NET configuration system. All sensitive data is automatically substituted during configuration loading — without manual processing in application code. +Secure secrets management and command-line argument parsing within the .NET `Microsoft.Extensions.Configuration` ecosystem. Secrets are automatically substituted into configuration without manual application code. ---- +## Features -## How It Works - -1. **Load secrets** from secure sources (files, environment variables) -2. **Automatic substitution** of values in configuration during loading -3. **Transparent usage** via standard `IConfiguration` +- **Automatic secret substitution**: `{{key}}` placeholders are replaced with real values from files, environment variables, or command-line arguments +- **Cycle protection**: built-in guard against infinite recursion during placeholder resolution +- **Optional placeholders**: `{{?key}}` — if the secret is not found, returns `null` instead of throwing +- **Chained Stores**: multiple secret sources with priority ordering +- **Argument parser**: supports `--key value`, `--key=value`, `-flag` formats +- **Environments**: automatic loading of `secrets.{Environment}.txt` (Development/Staging/Production) ---- +## Quick Start -## Setup - -### 1. Registration in `Program.cs` +### 1. Register in `Program.cs` ```csharp using Sa.Configuration; var builder = WebApplication.CreateBuilder(args); +// Connects arguments + secrets from files/env vars/command line builder.Configuration.AddSaConfiguration(); var app = builder.Build(); ``` -### 2. Secret Sources - -Secrets are loaded from the following sources (in priority order): - -| Source | Description | Example | -|--------|-------------|---------| -| **Secrets file** | Text file with `key=value` pairs | `secrets.txt` | -| **Environment variables** | System environment variables | `SA_PG_PASSWORD=myPass` | -| **Command-line arguments** | Application startup parameters | `--sa_pg_port=5432` | - ---- - -## Secrets File Format (`secrets.txt`) +### 2. Secrets File (`secrets.txt`) ```ini # Postgres sa_pg_host=localhost sa_pg_user=postgres sa_pg_port=5432 -sa_pg_database=postgres +sa_pg_database=myapp sa_pg_schema=public sa_pg_password=superSecret123 -# Other secrets -sa_secret=TOP SECRET! +# API keys api_key=abc123xyz +jwt_secret=h8k2m9p0 ``` -> ⚠️ **Important**: The `secrets.txt` file must be excluded from version control (.gitignore) +> ⚠️ Add `secrets*.txt` to `.gitignore`! ---- - -## Usage in `appsettings.json` - -Specify **placeholders** in the format `{{secret_key}}`: +### 3. Placeholders in `appsettings.json` ```json { "secret": "{{sa_secret}}", - + "sa": { "pg": { "connection": "User ID={{sa_pg_user}};Password={{sa_pg_password}};Host={{sa_pg_host}};Port={{sa_pg_port}};Database={{sa_pg_database}};Pooling=true;SearchPath={{sa_pg_schema}};Command Timeout=180;" } }, - + "ExternalApi": { "ApiKey": "{{api_key}}" } } ``` ---- +### 4. Reading Configuration + +```csharp +var pgConn = app.Configuration["sa:pg:connection"]; +// → "User ID=postgres;Password=superSecret123;Host=localhost;..." +``` + +## Secret Priority Order + +Secrets are looked up in descending priority order: + +| # | Source | Example File | +|---|--------|-------------| +| 1 | Base secrets file | `secrets.txt` | +| 2 | Environment-specific file | `secrets.Development.txt` | +| 3 | Environment variables | `SA_PG_PASSWORD=...` | +| 4 | Command-line arguments | `--sa_pg_password=...` | + +The first source that has a value wins. This allows overriding secrets per environment. + +## Optional Placeholders + +Use `{{?key}}` instead of `{{key}}` to avoid an error when a secret is missing: + +```json +{ + "optional_feature": "{{?feature_flag}}" +} +``` -## Code Example +If `feature_flag` is not found in any store, `null` is returned. -### Retrieving values via `IConfiguration` +## Usage with Sa.Configuration.PostgreSql ```csharp -var todosApi = app.MapGroup("/settings"); +using Sa.Configuration; +using Sa.Configuration.PostgreSql; + +var builder = WebApplication.CreateBuilder(args); -todosApi.MapGet("/", (IConfiguration configuration) => new Settings[] { - new (Key: "pg_connection", Value: configuration["sa:pg:connection"]), - new (Key: "theme", Value: configuration["theme"]), - new (Key: "secret", Value: configuration["secret"]) -}).WithName("GetSettings"); +// First, standard sources (appsettings.json, secrets.txt) +builder.Configuration.AddSaConfiguration(); + +// Then, dynamic settings from the database +builder.Configuration.AddSaPostgreSqlConfiguration(new PostgreSqlConfigurationOptions( + ConnectionString: "...", + SelectSql: "SELECT key, value FROM app_settings" +)); + +var app = builder.Build(); ``` +## Arguments — Command-Line Argument Parser + +```csharp +using Sa.Configuration.CommandLine; + +// some.exe --config_db /share/data.db --debug +var args = new Arguments(args); ---- +string? configDb = args["config_db"]; // → "/share/data.db" +bool? debug = args.GetBool("debug"); // → true +int? port = args.GetInt("port"); // → null +TimeSpan? timeout = args.GetTimeSpan("timeout"); +``` -## What Happens Under the Hood +Supported formats: ``` -┌─────────────────────────────────────────────────────────┐ -│ 1. appsettings.json contains: │ -│ "connection": "Host={{sa_pg_host}};Password={{...}}" │ -├─────────────────────────────────────────────────────────┤ -│ 2. secrets.txt contains: │ -│ sa_pg_host=localhost │ -│ sa_pg_password=superSecret123 │ -├─────────────────────────────────────────────────────────┤ -│ 3. IConfiguration["sa:pg:connection"] returns: │ -│ "Host=localhost;Password=superSecret123;..." │ -└─────────────────────────────────────────────────────────┘ +--key value +--key=value +-key value +-key=value +-flag → flag=true (boolean flag) ``` ---- +Typed methods return `null` when the parameter is absent or invalid: -## Advantages +| Method | Return Type | Conversion | +|--------|------------|------------| +| `GetBool()` | `bool?` | `"true"/"1"/"yes"/"on"` → `true` | +| `GetInt()` | `int?` | `int.TryParse(..., InvariantCulture)` | +| `GetFloat()` | `float?` | same as above | +| `GetLong()` | `long?` | same as above | +| `GetTimeSpan()` | `TimeSpan?` | `TimeSpan.TryParse(..., InvariantCulture)` | -- **Security**: secrets are not stored in code or configuration files -- **Flexibility**: supports multiple secret sources -- **Simplicity**: transparent operation through standard `IConfiguration` -- **Debugging**: easy to switch secrets via environment variables or arguments +## Secrets — Secrets Management ---- +### Creating Defaults -## Tips +```csharp +using Sa.Configuration.SecretStore; -- For local development, create `secrets.Development.txt`; for production — use environment variables -- Never commit secret files to the repository -- Use different secret files for different environments (dev, staging, prod) +// Standard chain: File → File.Env → EnvVar → CommandLine +var secrets = Secrets.CreateDefault(); +``` ---- +### Custom Chain -# Core Classes +```csharp +var secrets = new Secrets( + new FileSecretStore("my-secrets.txt"), + new EnvironmentVariableSecretStore(), + new InMemorySecretStore(new Dictionary { + { "override_key", "override_value" } + }) +); +``` -## Arguments Class +### Placeholder Substitution -The `Arguments` class is designed to parse command-line arguments passed to a C# application. It provides a dictionary-like interface for easy parameter retrieval and supports both single-value and multi-value parameters. +```csharp +string template = "Server={{host}};Password={{password}}"; +string result = secrets.PopulateSecrets(template); +// → "Server=localhost;Password=s3cret!" +``` -**Key Features:** -- **Parameter Parsing**: Splits command-line arguments into key-value pairs -- **Easy Retrieval**: Access parameter values using an indexer -- **Default Handling**: Automatically assigns default values for boolean flags +### Getting a Single Secret -**Example:** ```csharp -// some.exe --config_db /share/data.db -var arguments = new Arguments(args); -string? configDb = arguments["config_db"]; +string? password = secrets.GetSecret("sa_pg_password"); ``` ---- +## Public API -## Secrets Class +### Namespace `Sa.Configuration` -The `Secrets` class provides a secure mechanism for managing sensitive information such as API keys and database passwords from various sources. It supports loading secrets from files, environment variables, and dynamically generated host key files. +| Type | Purpose | +|------|---------| +| `Setup.AddSaConfiguration()` | Main entry-point: connects arguments + secret processing | -**Key Features:** -- **Chained Secret Stores**: Combines multiple sources for retrieving secrets -- **Dynamic Loading**: Supports environment-specific configurations -- **Placeholder Replacement**: Easily populates strings with secret values +### Namespace `Sa.Configuration.CommandLine` -**Example:** -```csharp -string input = "Database password: {{db_password}}"; -string? populatedString = service.PopulateSecrets(input); +| Type | Purpose | +|------|---------| +| `Arguments` | Command-line argument parser | +| `Arguments.CreateDefault()` | Creates from `Environment.GetCommandLineArgs()` | +| `Setup.AddSaCommandLine()` | Extension method for `IConfigurationBuilder` | + +### Namespace `Sa.Configuration.SecretStore` + +| Type | Purpose | +|------|---------| +| `Secrets` | Main secrets management class, implements `ISecretService` | +| `Secrets.CreateDefault()` | Standard store chain | +| `Secrets.GetEnvironmentName()` | Resolves environment (`DOTNET_ENVIRONMENT` / `ASPNETCORE_ENVIRONMENT`) | +| `SecretOptions` | Options for `CreateDefault()`: `FileName`, `Args`, `EnvironmentName` | +| `ISecretService` | Interface: `PopulateSecrets()` + `GetSecret()` | +| `ISecretStore` | Interface: `GetSecret(string key)` | +| `Setup.AddSaPostSecretProcessing()` | Extension method: applies `ISecretService` to config AFTER other sources are loaded | + +### Secret Stores (`Sa.Configuration.SecretStore.Stories`) + +| Class | Description | +|-------|-------------| +| `FileSecretStore` | Loads `key=value` from a text file (skips `#` comments) | +| `EnvironmentVariableSecretStore` | Reads from `Environment.GetEnvironmentVariable()` | +| `CommandLineArgsSecretStore` | Pulls secrets from `Arguments` | +| `InMemorySecretStore` | Dictionary in memory, fluent `.AddSecret()` | + +## How It Works + +``` +┌──────────────────────────────────────────────────────┐ +│ 1. appsettings.json contains: │ +│ "connection": "Host={{sa_pg_host}};Password={{...}}"│ +├──────────────────────────────────────────────────────┤ +│ 2. secrets.txt contains: │ +│ sa_pg_host=localhost │ +│ sa_pg_password=s3cret! │ +├──────────────────────────────────────────────────────┤ +│ 3. AddSaPostSecretProcessing substitutes placeholders:│ +│ IConfiguration["sa:pg:connection"] │ +│ → "Host=localhost;Password=s3cret!;..." │ +└──────────────────────────────────────────────────────┘ +``` + +## Project Layout + +``` +src/Sa.Configuration/ +├── Setup.cs # AddSaConfiguration() +├── CommandLine/ +│ ├── Arguments.cs # Argument parser +│ ├── Arguments.partial.cs # Typed GetXxx() methods +│ ├── ArgumentsConfigurationProvider.cs # IConfigurationProvider +│ └── Setup.cs # AddSaCommandLine() +├── SecretStore/ +│ ├── Secrets.cs # Secrets management +│ ├── SecretOptions.cs # CreateDefault() options +│ ├── ISecretService.cs # Service interface +│ ├── ISecretStore.cs # Store interface +│ ├── Engine/ +│ │ ├── ChainedSecretStore.cs # Store stacking +│ │ ├── ChainedSecrets.cs # Chained + Service +│ │ └── SecretService.cs # Placeholder substitution +│ ├── Stories/ +│ │ ├── FileSecretStore.cs # Text file +│ │ ├── EnvironmentVariableSecretStore.cs # ENV vars +│ │ ├── CommandLineArgsSecretStore.cs # Args parser +│ │ └── InMemorySecretStore.cs # Dictionary in memory +│ ├── PostSecretProcessingConfigurationProvider.cs # IConfigurationProvider +│ ├── PostSecretProcessingConfigurationSource.cs # IConfigurationSource +│ └── Setup.cs # AddSaPostSecretProcessing() +└── Readme.md # ← you are here ``` diff --git a/src/Sa.Configuration/SecretStore/Engine/SecretService.cs b/src/Sa.Configuration/SecretStore/Engine/SecretService.cs index db545376..058e3459 100644 --- a/src/Sa.Configuration/SecretStore/Engine/SecretService.cs +++ b/src/Sa.Configuration/SecretStore/Engine/SecretService.cs @@ -69,7 +69,7 @@ internal sealed partial class SecretService(ISecretStore secretStore) : ISecretS } private static bool IsSearchPositionValid(string inputString, int currentPosition) - => inputString.Length >= currentPosition; + => currentPosition < inputString.Length; private static string NormalizeValue(string secretValue) { diff --git a/src/Tests/Sa.Configuration.PostgreSqlTests/DatabaseConfigurationExtensionsTests.cs b/src/Tests/Sa.Configuration.PostgreSqlTests/DatabaseConfigurationExtensionsTests.cs index 92efd0f0..de575169 100644 --- a/src/Tests/Sa.Configuration.PostgreSqlTests/DatabaseConfigurationExtensionsTests.cs +++ b/src/Tests/Sa.Configuration.PostgreSqlTests/DatabaseConfigurationExtensionsTests.cs @@ -81,7 +81,7 @@ public void AddPostgreSqlConfiguration_ShouldAddDatabaseConfigurationSource() var optionsEx = new PostgreSqlConfigurationOptions( fixture.ConnectionString, "SELECT key, value FROM configuration_ex where client_id = @client_id", - [new("client_id", 1)]); + new NpgsqlParameter("client_id", 1)); builder.AddSaPostgreSqlConfiguration(optionsEx); diff --git a/src/Tests/Sa.Configuration.PostgreSqlTests/DatabaseConfigurationProviderTests.cs b/src/Tests/Sa.Configuration.PostgreSqlTests/DatabaseConfigurationProviderTests.cs new file mode 100644 index 00000000..c84933c6 --- /dev/null +++ b/src/Tests/Sa.Configuration.PostgreSqlTests/DatabaseConfigurationProviderTests.cs @@ -0,0 +1,119 @@ +using Microsoft.Extensions.Configuration; +using Npgsql; +using Sa.Configuration.PostgreSql; +using Sa.Data.PostgreSql.Fixture; + +namespace Sa.Configuration.PostgreSqlTests; + + +public sealed class DatabaseConfigurationProviderTests(DatabaseConfigurationProviderTests.Fixture fixture) + : IClassFixture +{ + public sealed class Fixture : PgDataSourceFixture + { + public override async ValueTask InitializeAsync() + { + await base.InitializeAsync(); + + using var connection = new NpgsqlConnection(ConnectionString); + await connection.OpenAsync(); + + // Основная таблица + using (var createTableCommand = new NpgsqlCommand(@" + CREATE TABLE IF NOT EXISTS config_provider_test ( + key TEXT PRIMARY KEY, + value TEXT + );", connection)) + { + await createTableCommand.ExecuteNonQueryAsync(); + } + + // Вставка данных + using var insertCommand = new NpgsqlCommand(@" + INSERT INTO config_provider_test (key, value) VALUES + ('NormalKey', 'normal_value'), + ('TrimmedKey ', ' trimmed_value '), + ('WhitespaceKey', ' spaced '), + ('EmptyValueKey', ''), + ('NullValueKey', null), + (' LeadingSpaceKey', 'leading_space_value') + ON CONFLICT (key) DO NOTHING;", connection); + + await insertCommand.ExecuteNonQueryAsync(); + } + } + + [Fact] + public void Load_HandlesEmptyResult_WhenNoRowsMatch() + { + // Arrange + var builder = new ConfigurationBuilder(); + var options = new PostgreSqlConfigurationOptions( + fixture.ConnectionString, + "SELECT key, value FROM config_provider_test WHERE key = 'nonexistent_row'"); + + builder.AddSaPostgreSqlConfiguration(options); + + // Act + var configuration = builder.Build(); + + // Assert + Assert.Null(configuration["nonexistent_row"]); + } + + [Fact] + public void Load_TrimsKeyAndValue_WhenTheyHaveWhitespace() + { + // Arrange + var builder = new ConfigurationBuilder(); + var options = new PostgreSqlConfigurationOptions( + fixture.ConnectionString, + "SELECT key, value FROM config_provider_test"); + + builder.AddSaPostgreSqlConfiguration(options); + + // Act + var configuration = builder.Build(); + + // Assert + Assert.Equal("trimmed_value", configuration["TrimmedKey"]); + Assert.Equal("spaced", configuration["WhitespaceKey"]); + Assert.Equal("leading_space_value", configuration["LeadingSpaceKey"]); + } + + [Fact] + public void Load_HandlesEmptyStringValue() + { + // Arrange + var builder = new ConfigurationBuilder(); + var options = new PostgreSqlConfigurationOptions( + fixture.ConnectionString, + "SELECT key, value FROM config_provider_test WHERE key = 'EmptyValueKey'"); + + builder.AddSaPostgreSqlConfiguration(options); + + // Act + var configuration = builder.Build(); + + // Assert + Assert.Null(configuration["EmptyValueKey"]); + } + + [Fact] + public void Load_HandlesNullValue_ResultSetIsNull() + { + // Arrange + var builder = new ConfigurationBuilder(); + var options = new PostgreSqlConfigurationOptions( + fixture.ConnectionString, + "SELECT key, value FROM config_provider_test WHERE key = 'NullValueKey'"); + + builder.AddSaPostgreSqlConfiguration(options); + + // Act + var configuration = builder.Build(); + + // Assert + Assert.Null(configuration["NullValueKey"]); + } +} diff --git a/src/Tests/Sa.ConfigurationTests/ArgumentsConfigurationProviderTests.cs b/src/Tests/Sa.ConfigurationTests/ArgumentsConfigurationProviderTests.cs new file mode 100644 index 00000000..e3cf48e6 --- /dev/null +++ b/src/Tests/Sa.ConfigurationTests/ArgumentsConfigurationProviderTests.cs @@ -0,0 +1,74 @@ +using Sa.Configuration.CommandLine; + +namespace Sa.ConfigurationTests; + +public class ArgumentsConfigurationProviderTests +{ + [Fact] + public void Load_PopulatesData_FromArguments() + { + // Arrange + var source = new ArgumentsConfigurationSource() + { + Args = ["--host", "localhost", "--port", "5432", "--debug"] + }; + var provider = new ArgumentsConfigurationProvider(source); + + // Act + provider.Load(); + + // Assert + Assert.True(provider.TryGet("host", out var l)); + Assert.True(provider.TryGet("port", out var p)); + Assert.True(provider.TryGet("debug", out var d)); + + Assert.Equal("localhost", l); + Assert.Equal("5432", p); + Assert.Equal("true", d); + } + + + [Fact] + public void Load_HandlesEqualsSeparator() + { + // Arrange + var source = new ArgumentsConfigurationSource + { + Args = ["--connection_string=Host=db;Port=5432"] + }; + var provider = new ArgumentsConfigurationProvider(source); + + // Act + provider.Load(); + + Assert.True(provider.TryGet("connection_string", out var r)); + // Assert + Assert.Equal("Host=db;Port=5432", r); + } + + [Fact] + public void Load_HandlesMixedSeparators() + { + // Arrange + var source = new ArgumentsConfigurationSource + { + Args = ["--host=localhost", "-port", "5432", "--debug"] + }; + var provider = new ArgumentsConfigurationProvider(source); + + // Act + provider.Load(); + + + + // Assert + Assert.True(provider.TryGet("host", out var r1)); + Assert.True(provider.TryGet("port", out var r2)); + Assert.True(provider.TryGet("debug", out var r3)); + + + Assert.Equal("localhost", r1); + Assert.Equal("5432", r2); + Assert.Equal("true", r3); + } +} diff --git a/src/Tests/Sa.ConfigurationTests/ChainedSecretStoreTests.cs b/src/Tests/Sa.ConfigurationTests/ChainedSecretStoreTests.cs new file mode 100644 index 00000000..746e6389 --- /dev/null +++ b/src/Tests/Sa.ConfigurationTests/ChainedSecretStoreTests.cs @@ -0,0 +1,124 @@ +using Sa.Configuration.SecretStore.Engine; +using Sa.Configuration.SecretStore.Stories; + +namespace Sa.ConfigurationTests; + +public class ChainedSecretStoreTests +{ + [Fact] + public void GetSecret_ReturnsFirstNonNullValue_FromChainedStores() + { + // Arrange + var store1 = new InMemorySecretStore(new Dictionary + { + ["key1"] = "value_from_store1", + ["key2"] = "also_store1" + }); + var store2 = new InMemorySecretStore(new Dictionary + { + ["key2"] = "value_from_store2", + ["key3"] = "value_from_store3" + }); + + var chained = new ChainedSecretStore([store1, store2]); + + // Act & Assert + Assert.Equal("value_from_store1", chained.GetSecret("key1")); + Assert.Equal("value_from_store2", chained.GetSecret("key2")); // last wins + Assert.Equal("value_from_store3", chained.GetSecret("key3")); + } + + [Fact] + public void GetSecret_ReturnsNull_WhenKeyNotFoundInAnyStore() + { + // Arrange + var store = new InMemorySecretStore(new Dictionary + { + ["key1"] = "value1" + }); + var chained = new ChainedSecretStore([store]); + + // Act + var result = chained.GetSecret("missing_key"); + + // Assert + Assert.Null(result); + } + + [Fact] + public void GetSecret_SkipsNullValues_AndReturnsSecondStoreValue() + { + // Arrange + var store1 = new InMemorySecretStore(new Dictionary + { + ["shared_key"] = null! + }); + var store2 = new InMemorySecretStore(new Dictionary + { + ["shared_key"] = "value_from_second" + }); + + var chained = new ChainedSecretStore([store1, store2]); + + // Act + var result = chained.GetSecret("shared_key"); + + // Assert + Assert.Equal("value_from_second", result); + } + + [Fact] + public void Add_NewStoreHasHigherPriority_LifoOrder() + { + // Arrange + var originalStore = new InMemorySecretStore(new Dictionary + { + ["priority_key"] = "original" + }); + var chained = new ChainedSecretStore([originalStore]); + + // Act — добавляем новое хранилище (оно должно иметь приоритет) + var newStore = new InMemorySecretStore(new Dictionary + { + ["priority_key"] = "new_priority" + }); + chained.Add(newStore); + + // Assert + Assert.Equal("new_priority", chained.GetSecret("priority_key")); + } + + [Fact] + public void GetSecret_EmptyStoresCollection_ReturnsNull() + { + // Arrange + var chained = new ChainedSecretStore([]); + + // Act + var result = chained.GetSecret("any_key"); + + // Assert + Assert.Null(result); + } + + [Fact] + public void Add_StoreWithEmptyString_DoesNotOverrideNonNullExistingValue() + { + // Arrange + var store1 = new InMemorySecretStore(new Dictionary + { + ["key"] = "existing_value" + }); + var chained = new ChainedSecretStore([store1]); + + // Act — добавляем хранилище с пустой строкой + var emptyStore = new InMemorySecretStore(new Dictionary + { + ["key"] = string.Empty + }); + chained.Add(emptyStore); + + // Assert — null пропускается дальше, но пустая строка != null, поэтому она вернётся + Assert.Equal(string.Empty, chained.GetSecret("key")); + } +} diff --git a/src/Tests/Sa.ConfigurationTests/CommandLineArgsSecretStoreTests.cs b/src/Tests/Sa.ConfigurationTests/CommandLineArgsSecretStoreTests.cs new file mode 100644 index 00000000..d81beb01 --- /dev/null +++ b/src/Tests/Sa.ConfigurationTests/CommandLineArgsSecretStoreTests.cs @@ -0,0 +1,75 @@ +using Sa.Configuration.SecretStore.Stories; + +namespace Sa.ConfigurationTests; + +public class CommandLineArgsSecretStoreTests +{ + [Fact] + public void GetSecret_ReturnsValue_WhenKeyExists() + { + // Arrange + var args = new[] { "--db_password", "my_secret_pass" }; + var store = new CommandLineArgsSecretStore(args); + + // Act + var result = store.GetSecret("db_password"); + + // Assert + Assert.Equal("my_secret_pass", result); + } + + [Fact] + public void GetSecret_ReturnsNull_WhenKeyDoesNotExist() + { + // Arrange + var args = new[] { "--existing_key", "value" }; + var store = new CommandLineArgsSecretStore(args); + + // Act + var result = store.GetSecret("non_existing_key"); + + // Assert + Assert.Null(result); + } + + [Fact] + public void GetSecret_ReturnsTrue_WhenFlagPresentWithoutValue() + { + // Arrange + var args = new[] { "--verbose_flag" }; + var store = new CommandLineArgsSecretStore(args); + + // Act + var result = store.GetSecret("verbose_flag"); + + // Assert + Assert.Equal("true", result); + } + + [Fact] + public void GetSecret_ReturnsNull_ForDefaultArgs_WhenNoArgsProvided() + { + // Arrange + var store = new CommandLineArgsSecretStore(); + + // Act + var result = store.GetSecret("any_key"); + + // Assert + Assert.Null(result); + } + + [Fact] + public void GetSecret_SupportsDoubleDashPrefix() + { + // Arrange + var args = new[] { "--api-key", "abc-123" }; + var store = new CommandLineArgsSecretStore(args); + + // Act + var result = store.GetSecret("api-key"); + + // Assert + Assert.Equal("abc-123", result); + } +} diff --git a/src/Tests/Sa.ConfigurationTests/EnvironmentVariableSecretStoreTests.cs b/src/Tests/Sa.ConfigurationTests/EnvironmentVariableSecretStoreTests.cs new file mode 100644 index 00000000..4a19e30d --- /dev/null +++ b/src/Tests/Sa.ConfigurationTests/EnvironmentVariableSecretStoreTests.cs @@ -0,0 +1,91 @@ +using Sa.Configuration.SecretStore.Stories; + +namespace Sa.ConfigurationTests; + +public class EnvironmentVariableSecretStoreTests +{ + [Fact] + public void GetSecret_ReturnsValue_WhenEnvironmentVariableExists() + { + // Arrange + const string key = "__TEST_ENV_VAR_EXISTS__"; + const string expectedValue = "test_secret_value"; + Environment.SetEnvironmentVariable(key, expectedValue); + + try + { + var store = new EnvironmentVariableSecretStore(); + + // Act + var result = store.GetSecret(key); + + // Assert + Assert.Equal(expectedValue, result); + } + finally + { + Environment.SetEnvironmentVariable(key, null); + } + } + + [Fact] + public void GetSecret_ReturnsEmptyString_WhenValueEqualsParenthesizedEmpty() + { + // Arrange + const string key = "__TEST_ENV_VAR_EMPTY__"; + Environment.SetEnvironmentVariable(key, "(empty)"); + + try + { + var store = new EnvironmentVariableSecretStore(); + + // Act + var result = store.GetSecret(key); + + // Assert + Assert.Equal(string.Empty, result); + } + finally + { + Environment.SetEnvironmentVariable(key, null); + } + } + + [Fact] + public void GetSecret_ReturnsNull_WhenEnvironmentVariableDoesNotExist() + { + // Arrange + const string key = "__NON_EXISTENT_TEST_ENV_VAR__"; + var store = new EnvironmentVariableSecretStore(); + + // Act + var result = store.GetSecret(key); + + // Assert + Assert.Null(result); + } + + [Fact] + public void GetSecret_ReturnsNormalValue_WhenNotParenthesizedEmpty() + { + // Arrange + const string key = "__TEST_ENV_VAR_NORMAL__"; + const string expectedValue = "(not_empty)"; + Environment.SetEnvironmentVariable(key, expectedValue); + + try + { + var store = new EnvironmentVariableSecretStore(); + + // Act + var result = store.GetSecret(key); + + // Assert + Assert.Equal(expectedValue, result); + } + finally + { + Environment.SetEnvironmentVariable(key, null); + } + } +} From c14a38167b86568e44c3247955e30432baad258e Mon Sep 17 00:00:00 2001 From: dundich Date: Thu, 25 Jun 2026 17:50:34 +0300 Subject: [PATCH 05/33] improve SaWorkQueue --- src/Sa.Utils.WorkQueue/Readme.md | 14 +++++++++----- src/Sa.Utils.WorkQueue/SaWorkQueue.cs | 15 +++++++++------ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/Sa.Utils.WorkQueue/Readme.md b/src/Sa.Utils.WorkQueue/Readme.md index e8cb40de..b74a5b74 100644 --- a/src/Sa.Utils.WorkQueue/Readme.md +++ b/src/Sa.Utils.WorkQueue/Readme.md @@ -11,9 +11,10 @@ | **Concurrency limiting** | Control the number of simultaneously executing tasks via `ConcurrencyLimit` | | **Dynamic scaling** | Change the limit at runtime: `queue.ConcurrencyLimit = newLimit` | | **Scaling strategies** | `Lifo` • `Fifo` • `RoundRobin` • `Random` — choose the one that fits your scenario | -| **DI integration** | Registration via `AddWorkQueue` with configuration support | +| **DI integration** | Registration via `AddSaWorkQueue` with configuration support | | **Zero-allocation logging** | `[LoggerMessage]` generation for `ILogger` | | **Safe shutdown** | `DisposeAsync`, `ShutdownAsync`, `WaitForIdleAsync` — idempotent and thread‑safe | +| **Fault tolerance** | Configurable error strategy: `ShutdownQueue` (default), `StopReader`, or `Continue` | --- @@ -55,7 +56,7 @@ public class OrderService(ISaWorkQueue queue) { await queue.Enqueue(order, ct); // Does not block the caller } - + public bool IsIdle() => queue.IsIdle(); public int Pending => queue.QueueTasks; } @@ -63,7 +64,7 @@ public class OrderService(ISaWorkQueue queue) --- -## ⚙️ `WorkQueueOptions` Configuration +## ⚙️ `SaWorkQueueOptions` Configuration ```csharp SaWorkQueueOptions.Create(processor) @@ -119,6 +120,9 @@ int limit = queue.ConcurrencyLimit; // current concurrency limit 1. **Lifecycle**: the queue is registered as `Singleton`. Do not use `Scoped`/`Transient`. 2. **`StatusChanged`**: invoked on a thread‑pool thread. Avoid long synchronous operations inside. 3. **Cancellation**: tasks receive a `CancellationToken`. Handle `OperationCanceledException` properly. -4. **`ForceCancelReaders`**: use only in emergencies — may leave the queue in an inconsistent state. -5. **Reusability**: all shutdown/cleanup methods (`Shutdown`, `Dispose`) are idempotent — safe to call multiple times. +4. **Default error strategy**: `Shutdown` — a faulted item shut down the queue. Override with `.WithHandleItemFaulted()` if you need different behavior. +5. **`ForceCancelReaders` / `ForceCancelReadersAsync`**: emergency stop — kills reader tasks immediately. After calling, restore concurrency by setting `ConcurrencyLimit = X` to spawn replacement readers. +6. **Thread safety**: all public methods are thread-safe. Changing `ConcurrencyLimit` at runtime adjusts reader count without losing queued items. +7. **Reusability**: all shutdown/cleanup methods (`Shutdown`, `ShutdownAsync`, `Dispose`, `DisposeAsync`) are idempotent — safe to call multiple times. +8. **`ConcurrencyLimit = 0`**: pauses all processing (kills all readers). Set back to a positive value to resume. diff --git a/src/Sa.Utils.WorkQueue/SaWorkQueue.cs b/src/Sa.Utils.WorkQueue/SaWorkQueue.cs index e4f460c9..f149a06a 100644 --- a/src/Sa.Utils.WorkQueue/SaWorkQueue.cs +++ b/src/Sa.Utils.WorkQueue/SaWorkQueue.cs @@ -68,7 +68,7 @@ public SaWorkQueue(SaWorkQueueOptions options, ILogger $"{item}"); - _handleItemFaulted = options.HandleItemFaulted ?? ((_, ex) => SaExecutionErrorStrategy.ShutdownQueue); + _handleItemFaulted = options.HandleItemFaulted ?? ((_, _) => SaExecutionErrorStrategy.ShutdownQueue); _queue = Channel.CreateBounded(new BoundedChannelOptions(_queueCapacity) { @@ -111,10 +111,10 @@ public async ValueTask Enqueue(TInput input, CancellationToken cancellationToken if (!IsEnabled) ThrowHelper.QueueStopped(); var wi = new WorkItem(input, cancellationToken); - MarkActive(); try { + MarkActive(); await _queue.Writer.WriteAsync(wi, cancellationToken).ConfigureAwait(false); } catch (Exception ex) @@ -125,6 +125,8 @@ public async ValueTask Enqueue(TInput input, CancellationToken cancellationToken { ThrowHelper.QueueStopped(); } + + throw; } } @@ -376,19 +378,19 @@ private async Task ExecuteItemAsync(WorkItem item, CancellationToken ct) } catch (Exception ex) { - var dislpayItem = _getItemDisplayName(item.Input); + var displayItem = _getItemDisplayName(item.Input); SaExecutionErrorStrategy errorStrategy = SaExecutionErrorStrategy.ShutdownQueue; try { OnStatusChanged(item.Input, SaWorkStatus.Faulted, ex); - LogItemExecutionFailed(_logger, dislpayItem, ex); + LogItemExecutionFailed(_logger, displayItem, ex); errorStrategy = _handleItemFaulted(item.Input, ex); } catch (Exception callbackEx) { - LogItemHandlerFailed(_logger, dislpayItem, callbackEx); + LogItemHandlerFailed(_logger, displayItem, callbackEx); } return errorStrategy switch @@ -506,7 +508,8 @@ public void Shutdown() tasks = [.. _taskReaders]; } - Task.WhenAll(tasks).GetAwaiter().GetResult(); + // Use ConfigureAwait(false) to avoid potential deadlocks from blocking on async operations + Task.WhenAll(tasks).ConfigureAwait(false).GetAwaiter().GetResult(); ClearRemainingItems(); } From b81d6612cabf53aeedb3ae97cc0ee1e23bcaa356 Mon Sep 17 00:00:00 2001 From: dundich Date: Thu, 25 Jun 2026 20:15:09 +0300 Subject: [PATCH 06/33] impove outbox --- .../Commands/SelectTenantCommand.cs | 5 +- .../SqlBuilder/SqlOutboxBuilder.cs | 2 +- src/Sa.Outbox/Delivery/ConsumeSettings.cs | 69 ++- .../ConsumeSettingsValidationResult.cs | 28 ++ src/Sa.Outbox/Delivery/DeliveryCourier.cs | 39 +- ...elivarySnapshot.cs => DeliverySnapshot.cs} | 4 +- .../ExponentialBackoffRetryStrategy.cs | 54 ++ ...livarySnapshot.cs => IDeliverySnapshot.cs} | 2 +- src/Sa.Outbox/Delivery/IRetryStrategy.cs | 14 + src/Sa.Outbox/Delivery/Setup.cs | 2 +- .../Partitional/OutboxPartitionalSupport.cs | 2 +- .../Publication/OutboxMessagePublisher.cs | 3 +- src/Sa.Outbox/Sa.Outbox.csproj | 6 + src/Sa.Outbox/Setup.cs | 5 + src/Sa.slnx | 1 + .../ConsumeSettingsValidationTests.cs | 172 +++++++ .../Sa.Outbox.Tests/DeliveryCourierTests.cs | 472 ++++++++++++++++++ .../ExponentialBackoffRetryStrategyTests.cs | 202 ++++++++ .../Sa.Outbox.Tests/FakeOutboxContext.cs | 153 ++++++ .../Sa.Outbox.Tests/Sa.Outbox.Tests.csproj | 13 + 20 files changed, 1207 insertions(+), 41 deletions(-) create mode 100644 src/Sa.Outbox/Delivery/ConsumeSettingsValidationResult.cs rename src/Sa.Outbox/Delivery/{DelivarySnapshot.cs => DeliverySnapshot.cs} (94%) create mode 100644 src/Sa.Outbox/Delivery/ExponentialBackoffRetryStrategy.cs rename src/Sa.Outbox/Delivery/{IDelivarySnapshot.cs => IDeliverySnapshot.cs} (89%) create mode 100644 src/Sa.Outbox/Delivery/IRetryStrategy.cs create mode 100644 src/Tests/Sa.Outbox.Tests/ConsumeSettingsValidationTests.cs create mode 100644 src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs create mode 100644 src/Tests/Sa.Outbox.Tests/ExponentialBackoffRetryStrategyTests.cs create mode 100644 src/Tests/Sa.Outbox.Tests/FakeOutboxContext.cs create mode 100644 src/Tests/Sa.Outbox.Tests/Sa.Outbox.Tests.csproj diff --git a/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs b/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs index 97703cc4..178ed449 100644 --- a/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs +++ b/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs @@ -7,14 +7,13 @@ namespace Sa.Outbox.PostgreSql.Commands; internal sealed class SelectTenantCommand( IPgDataSource dataSource, SqlOutboxBuilder sql, - NpqsqlOutboxReader outboxReader - ) : ISelectTenantCommand + NpqsqlOutboxReader outboxReader) : ISelectTenantCommand { public async Task> Execute(CancellationToken cancellationToken) { try { - return await dataSource.ExecuteReaderList(sql.SqlSelectTetant, + return await dataSource.ExecuteReaderList(sql.SqlSelectTenant, reader => outboxReader.Message.GetTenantId(reader), cancellationToken); } diff --git a/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs b/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs index 5278fdde..d8f0304b 100644 --- a/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs +++ b/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs @@ -135,7 +135,7 @@ ORDER BY ut.{settings.TaskQueue.Fields.TaskId} public string SqlSelectType = $"SELECT * FROM {settings.GetQualifiedTypeTableName()}"; - public string SqlSelectTetant = + public string SqlSelectTenant = $""" WITH ranked AS ( SELECT diff --git a/src/Sa.Outbox/Delivery/ConsumeSettings.cs b/src/Sa.Outbox/Delivery/ConsumeSettings.cs index d64f9fbf..3e77fbd3 100644 --- a/src/Sa.Outbox/Delivery/ConsumeSettings.cs +++ b/src/Sa.Outbox/Delivery/ConsumeSettings.cs @@ -1,14 +1,69 @@ namespace Sa.Outbox.Delivery; /// -/// Represents the consumption settings for retrieving & processing messages from the Outbox +/// Represents the consumption settings for retrieving & processing messages from the Outbox. /// -/// -/// Initializes a new instance of the class. -/// -/// Group identity for consuming. If null or empty, uses default. public sealed class ConsumeSettings { + /// + /// Validates all settings and returns a . + /// + public ConsumeSettingsValidationResult Validate() + { + var errors = new List(); + + if (MaxBatchSize <= 0) + errors.Add($"MaxBatchSize must be greater than 0, got {MaxBatchSize}."); + + if (MaxProcessingIterations < -1) + errors.Add($"MaxProcessingIterations must be >= -1, got {MaxProcessingIterations}."); + + if (IterationDelay.Ticks < 0) + errors.Add($"IterationDelay must be >= TimeSpan.Zero, got {IterationDelay}."); + + if (LockDuration <= TimeSpan.Zero) + errors.Add($"LockDuration must be greater than TimeSpan.Zero, got {LockDuration}."); + + if (LockRenewal >= LockDuration) + errors.Add($"LockRenewal ({LockRenewal}) must be less than LockDuration ({LockDuration})."); + + if (LockRenewal.Ticks < 0) + errors.Add($"LockRenewal must be >= TimeSpan.Zero, got {LockRenewal}."); + + if (LookbackInterval.Ticks <= 0) + errors.Add($"LookbackInterval must be greater than TimeSpan.Zero, got {LookbackInterval}."); + + if (MaxDeliveryAttempts <= 0) + errors.Add($"MaxDeliveryAttempts must be greater than 0, got {MaxDeliveryAttempts}."); + + if (ConsumeBatchSize.HasValue && ConsumeBatchSize.Value <= 0) + errors.Add($"ConsumeBatchSize must be greater than 0, got {ConsumeBatchSize}."); + + if (BatchingWindow.Ticks < 0) + errors.Add($"BatchingWindow must be >= TimeSpan.Zero, got {BatchingWindow}."); + + if (PerTenantTimeout.Ticks < 0) + errors.Add($"PerTenantTimeout must be >= TimeSpan.Zero, got {PerTenantTimeout}."); + + if (PerTenantMaxDegreeOfParallelism == 0) + errors.Add($"PerTenantMaxDegreeOfParallelism cannot be 0. Use 1 for sequential or > 1 for parallel."); + + return errors.Count == 0 + ? ConsumeSettingsValidationResult.Valid + : ConsumeSettingsValidationResult.Fail(errors); + } + + /// + /// Validates all settings, throwing if invalid. + /// + public void ThrowIfInvalid() + { + var result = Validate(); + if (!result.IsValid) + throw new InvalidOperationException( + $"Invalid ConsumeSettings: {string.Join("; ", result.Errors)}"); + } + /// /// Maximum number of processing iterations when greedy mode is enabled. /// -1 means unlimited iterations (greedy mode). @@ -33,13 +88,13 @@ public sealed class ConsumeSettings public int MaxBatchSize { get; set; } = 16; /// - /// Message lock expiration time. + /// Message lock expiration time. /// When a batch of messages for a bus instance is acquired, the messages will be locked (reserved) for that amount of time. /// public TimeSpan LockDuration { get; set; } = TimeSpan.FromSeconds(10); /// - /// How long before to request a lock renewal. + /// How long before to request a lock renewal. /// This should be much shorter than . /// public TimeSpan LockRenewal { get; set; } = TimeSpan.FromSeconds(3); diff --git a/src/Sa.Outbox/Delivery/ConsumeSettingsValidationResult.cs b/src/Sa.Outbox/Delivery/ConsumeSettingsValidationResult.cs new file mode 100644 index 00000000..a80f24a5 --- /dev/null +++ b/src/Sa.Outbox/Delivery/ConsumeSettingsValidationResult.cs @@ -0,0 +1,28 @@ +namespace Sa.Outbox.Delivery; + +/// +/// Result of . +/// +public sealed class ConsumeSettingsValidationResult +{ + internal static readonly ConsumeSettingsValidationResult Valid = new([]); + + private ConsumeSettingsValidationResult(List errors) + { + Errors = errors; + IsValid = errors.Count == 0; + } + + /// + /// True if all settings are valid. + /// + public bool IsValid { get; } + + /// + /// List of validation error messages. Empty when is true. + /// + public IReadOnlyList Errors { get; } + + internal static ConsumeSettingsValidationResult Fail(List errors) + => new(errors); +} diff --git a/src/Sa.Outbox/Delivery/DeliveryCourier.cs b/src/Sa.Outbox/Delivery/DeliveryCourier.cs index f1260717..4b229bc5 100644 --- a/src/Sa.Outbox/Delivery/DeliveryCourier.cs +++ b/src/Sa.Outbox/Delivery/DeliveryCourier.cs @@ -1,5 +1,4 @@ using Sa.Extensions; -using Sa.Outbox.Exceptions; using System.Runtime.CompilerServices; namespace Sa.Outbox.Delivery; @@ -7,8 +6,13 @@ namespace Sa.Outbox.Delivery; /// /// Delivers a batch of messages with error handling and retry mechanisms /// -internal sealed class DeliveryCourier(IDeliveryLifetimeInvoker processor) : IDeliveryCourier +internal sealed class DeliveryCourier( + IDeliveryLifetimeInvoker processor, + IRetryStrategy? retryStrategy = null) : IDeliveryCourier { + + private readonly IRetryStrategy _retryStrategy = retryStrategy ?? ExponentialBackoffRetryStrategy.Shared; + /// /// Asynchronous method to deliver messages /// @@ -34,15 +38,18 @@ public async ValueTask Deliver( // Method to handle errors during message delivery - private static void HandleError(Exception error, ReadOnlySpan> messages) + private void HandleError(Exception error, ReadOnlySpan> messages) { - foreach (IOutboxContextOperations item in messages) + foreach (IOutboxContextOperations message in messages) { - if (item.DeliveryResult.Code.IsPending()) + if (message.DeliveryResult.Code.IsPending()) { - item.Warn( - error ?? UnknownDeliveryException, - postpone: RetryStrategy.CalculateBackoff()); + var attempt = message.DeliveryInfo.Attempt + 1; + var backoff = _retryStrategy.GetBackoff(attempt); + + message.Warn( + error, + postpone: backoff); } } } @@ -79,20 +86,4 @@ private static int PostHandle(ReadOnlySpan message.DeliveryResult.Code.IsWarning() && message.DeliveryInfo.Attempt + 1 > maxDeliveryAttempts; - - - /// - /// todos: customization - /// Static class to generate random time spans for retry delays - /// - static class RetryStrategy - { - /// - /// Method to generate a random TimeSpan between 10 and 45 minutes - /// - public static TimeSpan CalculateBackoff() - => TimeSpan.FromSeconds(Random.Shared.Next(60 * 10, 60 * 45)); - } - - private readonly static DeliveryException UnknownDeliveryException = new("Unknown delivery error.", null, DeliveryStatusCode.Warn); } diff --git a/src/Sa.Outbox/Delivery/DelivarySnapshot.cs b/src/Sa.Outbox/Delivery/DeliverySnapshot.cs similarity index 94% rename from src/Sa.Outbox/Delivery/DelivarySnapshot.cs rename to src/Sa.Outbox/Delivery/DeliverySnapshot.cs index 1a85db7a..b5d7add0 100644 --- a/src/Sa.Outbox/Delivery/DelivarySnapshot.cs +++ b/src/Sa.Outbox/Delivery/DeliverySnapshot.cs @@ -4,9 +4,9 @@ namespace Sa.Outbox.Delivery; -internal sealed class DelivarySnapshot( +internal sealed class DeliverySnapshot( IScheduleSettings scheduleSettings, - IOutboxMessageMetadataProvider metadataProvider) : IDelivarySnapshot + IOutboxMessageMetadataProvider metadataProvider) : IDeliverySnapshot { private readonly Lazy _lazyJobs = new(() => [.. scheduleSettings.GetJobSettings()]); diff --git a/src/Sa.Outbox/Delivery/ExponentialBackoffRetryStrategy.cs b/src/Sa.Outbox/Delivery/ExponentialBackoffRetryStrategy.cs new file mode 100644 index 00000000..02e3f125 --- /dev/null +++ b/src/Sa.Outbox/Delivery/ExponentialBackoffRetryStrategy.cs @@ -0,0 +1,54 @@ +using System.Runtime.CompilerServices; + +namespace Sa.Outbox.Delivery; + +/// +/// Default exponential backoff with jitter retry strategy. +/// Formula: min(maxDelay, baseDelay * 2^(attempt-1)) * jitterFactor +/// +public sealed class ExponentialBackoffRetryStrategy : IRetryStrategy +{ + /// + /// The shared singleton instance using sensible defaults (5s base, 30min max). + /// + public static readonly ExponentialBackoffRetryStrategy Shared = new(); + + private readonly TimeSpan _baseDelay; + private readonly TimeSpan _maxDelay; + private readonly double _jitterFactorMin; + private readonly double _jitterFactorMax; + + /// + /// Creates a new exponential backoff strategy. + /// + /// Base delay for the first retry. Defaults to 5 seconds. + /// Maximum cap on backoff. Defaults to 30 minutes. + /// Minimum jitter multiplier (0..1). Defaults to 0.5. + /// Maximum jitter multiplier (jitterFactorMin..1). Defaults to 1.0. + public ExponentialBackoffRetryStrategy( + TimeSpan baseDelay = default, + TimeSpan maxDelay = default, + double jitterFactorMin = 0.5, + double jitterFactorMax = 1.0) + { + _baseDelay = baseDelay == TimeSpan.Zero ? TimeSpan.FromSeconds(5) : baseDelay; + _maxDelay = maxDelay == TimeSpan.Zero ? TimeSpan.FromMinutes(30) : maxDelay; + _jitterFactorMin = Math.Clamp(jitterFactorMin, 0.0, 1.0); + _jitterFactorMax = Math.Clamp(Math.Max(jitterFactorMax, jitterFactorMin), _jitterFactorMin, 1.0); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public TimeSpan GetBackoff(int attemptNumber) + { + if (attemptNumber < 1) attemptNumber = 1; + + var exponent = Math.Pow(2.0, attemptNumber - 1); + var cappedDelay = TimeSpan.FromTicks((long)(_baseDelay.Ticks * Math.Min(exponent, _maxDelay.Ticks / (double)_baseDelay.Ticks))); + + var jitter = _jitterFactorMin + Random.Shared.NextDouble() * (_jitterFactorMax - _jitterFactorMin); + var totalTicks = (long)(cappedDelay.Ticks * jitter); + + return TimeSpan.FromTicks(totalTicks); + } +} diff --git a/src/Sa.Outbox/Delivery/IDelivarySnapshot.cs b/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs similarity index 89% rename from src/Sa.Outbox/Delivery/IDelivarySnapshot.cs rename to src/Sa.Outbox/Delivery/IDeliverySnapshot.cs index 44000d65..4dea543a 100644 --- a/src/Sa.Outbox/Delivery/IDelivarySnapshot.cs +++ b/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs @@ -2,7 +2,7 @@ namespace Sa.Outbox.Delivery; -public interface IDelivarySnapshot +public interface IDeliverySnapshot { IJobSettings[] JobSettings { get; } ConsumerGroupSettings[] ConsumerSettings { get; } diff --git a/src/Sa.Outbox/Delivery/IRetryStrategy.cs b/src/Sa.Outbox/Delivery/IRetryStrategy.cs new file mode 100644 index 00000000..97e934ac --- /dev/null +++ b/src/Sa.Outbox/Delivery/IRetryStrategy.cs @@ -0,0 +1,14 @@ +namespace Sa.Outbox.Delivery; + +/// +/// Defines a strategy for calculating retry backoff delays. +/// +public interface IRetryStrategy +{ + /// + /// Calculates the backoff delay for the next retry attempt. + /// + /// The current attempt number (1-based). Higher attempts should typically yield longer delays. + /// The time span to wait before the next retry. + TimeSpan GetBackoff(int attemptNumber); +} diff --git a/src/Sa.Outbox/Delivery/Setup.cs b/src/Sa.Outbox/Delivery/Setup.cs index 0c3eb0a7..c6b575de 100644 --- a/src/Sa.Outbox/Delivery/Setup.cs +++ b/src/Sa.Outbox/Delivery/Setup.cs @@ -32,7 +32,7 @@ public static IServiceCollection AddOutboxDelivery( configure?.Invoke(new DeliveryBuilder(services)); - services.TryAddSingleton(); + services.TryAddSingleton(); return services; } diff --git a/src/Sa.Outbox/Partitional/OutboxPartitionalSupport.cs b/src/Sa.Outbox/Partitional/OutboxPartitionalSupport.cs index 3e76da12..aabaa48d 100644 --- a/src/Sa.Outbox/Partitional/OutboxPartitionalSupport.cs +++ b/src/Sa.Outbox/Partitional/OutboxPartitionalSupport.cs @@ -4,7 +4,7 @@ namespace Sa.Outbox.Partitional; internal sealed class OutboxPartitionalSupport( - IDelivarySnapshot? snapshot = null, + IDeliverySnapshot? snapshot = null, ITenantProvider? tenantProvider = null) : IOutboxPartitionalSupport { public async Task> GetMsgParts(CancellationToken cancellationToken) diff --git a/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs b/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs index 451d441e..66a7e985 100644 --- a/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs +++ b/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs @@ -12,10 +12,11 @@ internal sealed class OutboxMessagePublisher( { public async ValueTask Publish( IReadOnlyCollection messages, - int tenantId = 0, + int tenantId, CancellationToken cancellationToken = default) { if (messages.Count == 0) return 0; + return await Send(messages, tenantId, cancellationToken); } diff --git a/src/Sa.Outbox/Sa.Outbox.csproj b/src/Sa.Outbox/Sa.Outbox.csproj index bd4f815a..3d514150 100644 --- a/src/Sa.Outbox/Sa.Outbox.csproj +++ b/src/Sa.Outbox/Sa.Outbox.csproj @@ -5,6 +5,8 @@ 0.9.1 Simple Outbox infra for publishing and using messages + + @@ -26,4 +28,8 @@ + + + + diff --git a/src/Sa.Outbox/Setup.cs b/src/Sa.Outbox/Setup.cs index 8abf316e..d0eee382 100644 --- a/src/Sa.Outbox/Setup.cs +++ b/src/Sa.Outbox/Setup.cs @@ -1,4 +1,6 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Sa.Outbox.Delivery; namespace Sa.Outbox; @@ -10,6 +12,9 @@ public static IServiceCollection AddSaOutbox( { OutboxBuilder builder = OutboxBuilder.Create(services); build?.Invoke(builder); + + services.TryAddSingleton(); + return services; } } diff --git a/src/Sa.slnx b/src/Sa.slnx index da46cab0..989096d0 100644 --- a/src/Sa.slnx +++ b/src/Sa.slnx @@ -60,6 +60,7 @@ + diff --git a/src/Tests/Sa.Outbox.Tests/ConsumeSettingsValidationTests.cs b/src/Tests/Sa.Outbox.Tests/ConsumeSettingsValidationTests.cs new file mode 100644 index 00000000..e29838b7 --- /dev/null +++ b/src/Tests/Sa.Outbox.Tests/ConsumeSettingsValidationTests.cs @@ -0,0 +1,172 @@ +using Sa.Outbox.Delivery; +using Xunit; + +namespace Sa.Outbox.Tests; + +public class ConsumeSettingsValidationTests +{ + [Fact] + public void Default_Settings_Are_Valid() + { + var settings = new ConsumeSettings(); + var result = settings.Validate(); + + Assert.True(result.IsValid); + Assert.Empty(result.Errors); + } + + [Fact] + public void Zero_MaxBatchSize_Is_Invalid() + { + var settings = new ConsumeSettings { MaxBatchSize = 0 }; + var result = settings.Validate(); + + Assert.False(result.IsValid); + Assert.Single(result.Errors); + Assert.Contains("MaxBatchSize", result.Errors[0]); + } + + [Fact] + public void Negative_MaxBatchSize_Is_Invalid() + { + var settings = new ConsumeSettings { MaxBatchSize = -1 }; + var result = settings.Validate(); + + Assert.False(result.IsValid); + Assert.Contains("MaxBatchSize", result.Errors[0]); + } + + [Fact] + public void Invalid_MaxProcessingIterations_Is_Invalid() + { + var settings = new ConsumeSettings { MaxProcessingIterations = -5 }; + var result = settings.Validate(); + + Assert.False(result.IsValid); + Assert.Contains("MaxProcessingIterations", result.Errors[0]); + } + + [Fact] + public void Greedy_Mode_MaxProcessingIterations_MinusOne_Is_Valid() + { + var settings = new ConsumeSettings { MaxProcessingIterations = -1 }; + var result = settings.Validate(); + + Assert.True(result.IsValid); + } + + [Fact] + public void LockRenewal_Greater_Than_LockDuration_Is_Invalid() + { + var settings = new ConsumeSettings + { + LockDuration = TimeSpan.FromSeconds(5), + LockRenewal = TimeSpan.FromSeconds(10) + }; + var result = settings.Validate(); + + Assert.False(result.IsValid); + Assert.Contains("LockRenewal", result.Errors[0]); + } + + [Fact] + public void Equal_LockRenewal_And_LockDuration_Is_Invalid() + { + var settings = new ConsumeSettings + { + LockDuration = TimeSpan.FromSeconds(10), + LockRenewal = TimeSpan.FromSeconds(10) + }; + var result = settings.Validate(); + + Assert.False(result.IsValid); + Assert.Contains("LockRenewal", result.Errors[0]); + } + + [Fact] + public void Zero_PerTenantMaxDegreeOfParallelism_Is_Invalid() + { + var settings = new ConsumeSettings { PerTenantMaxDegreeOfParallelism = 0 }; + var result = settings.Validate(); + + Assert.False(result.IsValid); + Assert.Contains("PerTenantMaxDegreeOfParallelism", result.Errors[0]); + } + + [Fact] + public void Negative_MaxDeliveryAttempts_Is_Invalid() + { + var settings = new ConsumeSettings { MaxDeliveryAttempts = 0 }; + var result = settings.Validate(); + + Assert.False(result.IsValid); + Assert.Contains("MaxDeliveryAttempts", result.Errors[0]); + } + + [Fact] + public void Negative_ConsumeBatchSize_Is_Invalid() + { + var settings = new ConsumeSettings { ConsumeBatchSize = -1 }; + var result = settings.Validate(); + + Assert.False(result.IsValid); + Assert.Contains("ConsumeBatchSize", result.Errors[0]); + } + + [Fact] + public void ThrowIfInvalid_Does_Not_Throw_For_Valid_Settings() + { + var settings = new ConsumeSettings(); + var ex = Record.Exception(() => settings.ThrowIfInvalid()); + Assert.Null(ex); + } + + [Fact] + public void ThrowIfInvalid_Throws_For_Invalid_Settings() + { + var settings = new ConsumeSettings { MaxBatchSize = 0 }; + Assert.Throws(() => settings.ThrowIfInvalid()); + } + + [Fact] + public void Multiple_Violations_Return_All_Errors() + { + var settings = new ConsumeSettings + { + MaxBatchSize = 0, + MaxDeliveryAttempts = -1, + PerTenantMaxDegreeOfParallelism = 0 + }; + var result = settings.Validate(); + + Assert.False(result.IsValid); + Assert.True(result.Errors.Count >= 3); + } + + [Fact] + public void Zero_IterationDelay_Is_Valid() + { + var settings = new ConsumeSettings { IterationDelay = TimeSpan.Zero }; + var result = settings.Validate(); + + Assert.True(result.IsValid); + } + + [Fact] + public void Zero_BatchingWindow_Is_Valid() + { + var settings = new ConsumeSettings { BatchingWindow = TimeSpan.Zero }; + var result = settings.Validate(); + + Assert.True(result.IsValid); + } + + [Fact] + public void Zero_PerTenantTimeout_Is_Valid() + { + var settings = new ConsumeSettings { PerTenantTimeout = TimeSpan.Zero }; + var result = settings.Validate(); + + Assert.True(result.IsValid); + } +} diff --git a/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs b/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs new file mode 100644 index 00000000..759f22ae --- /dev/null +++ b/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs @@ -0,0 +1,472 @@ +using Sa.Outbox.Delivery; + +namespace Sa.Outbox.Tests; + +public class DeliveryCourierTests +{ + private sealed class TestMessage { } + + private static ConsumerGroupSettings CreateSettings(int maxDeliveryAttempts = 3) + => new("test-group", isSingleton: false) + { + ConsumeSettings = { MaxDeliveryAttempts = maxDeliveryAttempts } + }; + + private static OutboxMessageFilter CreateFilter() + => new( + "txn-1", + "test-group", + "Sa.Outbox.Tests.DeliveryCourierTests+TestMessage", + 1, + "part-1", + DateTimeOffset.MinValue, + DateTimeOffset.MaxValue, + DateTimeOffset.UtcNow); + + private static ReadOnlyMemory> ToMessages(params FakeOutboxContext[] contexts) + => new(contexts); + + #region Empty batch tests + + [Fact] + public async Task Deliver_EmptyBatch_ReturnsZero() + { + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.CompletedTask); + var courier = new DeliveryCourier(processor); + + var result = await courier.Deliver( + CreateSettings(), + CreateFilter(), + ReadOnlyMemory>.Empty, + CancellationToken.None); + + Assert.Equal(0, result); + Assert.Empty(processor.Invocations); + } + + #endregion + + #region Processor succeeds — all messages OK + + [Fact] + public async Task Deliver_ProcessorSucceeds_AllMessagesOk_ReturnsSuccessCount() + { + var ctx1 = new FakeOutboxContext(payloadId: "msg-1"); + var ctx2 = new FakeOutboxContext(payloadId: "msg-2"); + var ctx3 = new FakeOutboxContext(payloadId: "msg-3"); + var messages = ToMessages(ctx1, ctx2, ctx3); + + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.CompletedTask); + var courier = new DeliveryCourier(processor); + + var result = await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.Equal(3, result); + Assert.Equal(DeliveryStatusCode.Ok, ctx1.DeliveryResult.Code); + Assert.Equal(DeliveryStatusCode.Ok, ctx2.DeliveryResult.Code); + Assert.Equal(DeliveryStatusCode.Ok, ctx3.DeliveryResult.Code); + } + + [Fact] + public async Task Deliver_ProcessorSucceeds_SingleMessage_ReturnsOne() + { + var ctx = new FakeOutboxContext(payloadId: "single-msg"); + var messages = ToMessages(ctx); + + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.CompletedTask); + var courier = new DeliveryCourier(processor); + + var result = await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.Equal(1, result); + Assert.Equal(DeliveryStatusCode.Ok, ctx.DeliveryResult.Code); + } + + #endregion + + #region Processor throws — messages get postponed with retry backoff + + [Fact] + public async Task Deliver_ProcessorThrows_MessageWarnedWithBackoff() + { + var ctx = new FakeOutboxContext(payloadId: "msg-fail", attempt: 0); + var messages = ToMessages(ctx); + + var testException = new InvalidOperationException("processor broke"); + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.FromException(testException)); + + var retryStrategy = new FakeRetryStrategy(_ => TimeSpan.FromSeconds(5)); + var courier = new DeliveryCourier(processor, retryStrategy); + + var result = await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.Equal(0, result); + Assert.Equal(DeliveryStatusCode.Warn, ctx.DeliveryResult.Code); + Assert.Same(testException, ctx.Exception); + Assert.Equal(TimeSpan.FromSeconds(5), ctx.PostponeAt); + } + + [Fact] + public async Task Deliver_ProcessorThrows_MultipleMessagesAllWarned() + { + var ctx1 = new FakeOutboxContext(payloadId: "msg-1", attempt: 0); + var ctx2 = new FakeOutboxContext(payloadId: "msg-2", attempt: 1); + var ctx3 = new FakeOutboxContext(payloadId: "msg-3", attempt: 2); + var messages = ToMessages(ctx1, ctx2, ctx3); + + var testException = new TimeoutException("timeout"); + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.FromException(testException)); + + var callCount = 0; + var expectedBackoffs = new[] { TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(20) }; + var retryStrategy = new FakeRetryStrategy(attempt => + { + var backoff = expectedBackoffs[callCount++]; + return backoff; + }); + + var courier = new DeliveryCourier(processor, retryStrategy); + + var result = await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.Equal(0, result); + Assert.Equal(DeliveryStatusCode.Warn, ctx1.DeliveryResult.Code); + Assert.Equal(DeliveryStatusCode.Warn, ctx2.DeliveryResult.Code); + Assert.Equal(DeliveryStatusCode.Warn, ctx3.DeliveryResult.Code); + Assert.Equal(expectedBackoffs[0], ctx1.PostponeAt); + Assert.Equal(expectedBackoffs[1], ctx2.PostponeAt); + Assert.Equal(expectedBackoffs[2], ctx3.PostponeAt); + Assert.Same(testException, ctx1.Exception); + Assert.Same(testException, ctx2.Exception); + Assert.Same(testException, ctx3.Exception); + } + + [Fact] + public async Task Deliver_ProcessorThrows_UseDefaultRetryStrategy() + { + var ctx = new FakeOutboxContext(payloadId: "msg-default-retry", attempt: 1); + var messages = ToMessages(ctx); + + var testException = new InvalidOperationException("fail"); + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.FromException(testException)); + + // No custom retry strategy — should use ExponentialBackoffRetryStrategy.Shared + var courier = new DeliveryCourier(processor, null); + + var result = await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.Equal(0, result); + Assert.Equal(DeliveryStatusCode.Warn, ctx.DeliveryResult.Code); + Assert.NotNull(ctx.Exception); + Assert.True(ctx.PostponeAt > TimeSpan.Zero); + } + + #endregion + + #region Critical exceptions propagate + + //[Fact] + //public async Task Deliver_ProcessorThrowsCritical_ExceptionPropagates() + //{ + // var ctx = new FakeOutboxContext(payloadId: "msg-critical"); + // var messages = ToMessages(ctx); + + // var criticalException = new AccessViolationException("critical failure"); + // var processor = new FakeDeliveryLifetimeInvoker(_ => Task.FromException(criticalException)); + + // var courier = new DeliveryCourier(processor); + + // var ex = await Assert.ThrowsAsync(async () => + // await courier.Deliver(CreateSettings(), CreateFilter(), messages, CancellationToken.None)); + + // Assert.Same(criticalException, ex); + // // Critical exceptions skip the catch block, so messages are NOT touched + // Assert.Equal(DeliveryStatusCode.Pending, ctx.DeliveryResult.Code); + //} + + #endregion + + #region Pre-existing warning states + + [Fact] + public async Task Deliver_AlreadyWarning_BelowMaxAttempts_Skipped() + { + var ctx = new FakeOutboxContext( + payloadId: "msg-below-max", + attempt: 0, + initialStatus: DeliveryStatusCode.Warn); + var messages = ToMessages(ctx); + + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.CompletedTask); + var courier = new DeliveryCourier(processor); + + var result = await courier.Deliver( + CreateSettings(maxDeliveryAttempts: 3), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.Equal(0, result); + Assert.NotEqual(DeliveryStatusCode.MaximumAttemptsError, ctx.DeliveryResult.Code); + Assert.NotEqual(DeliveryStatusCode.Ok, ctx.DeliveryResult.Code); + } + + [Fact] + public async Task Deliver_AlreadyWarning_TrulyExceedsMaxAttempts_ErrorMaxAttemptsCalled() + { + var ctx = new FakeOutboxContext( + payloadId: "msg-max-attempts", + attempt: 3, // attempt(3) + 1 = 4 > max(3) ✓ + initialStatus: DeliveryStatusCode.Warn); + var messages = ToMessages(ctx); + + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.CompletedTask); + var courier = new DeliveryCourier(processor); + + var result = await courier.Deliver( + CreateSettings(maxDeliveryAttempts: 3), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.Equal(0, result); + Assert.Equal(DeliveryStatusCode.MaximumAttemptsError, ctx.DeliveryResult.Code); + } + + [Fact] + public async Task Deliver_AlreadySuccess_Skipped() + { + var ctx = new FakeOutboxContext( + payloadId: "msg-done", + attempt: 0, + initialStatus: DeliveryStatusCode.Ok); + var messages = ToMessages(ctx); + + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.CompletedTask); + var courier = new DeliveryCourier(processor); + + var result = await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.Equal(0, result); + Assert.Equal(DeliveryStatusCode.Ok, ctx.DeliveryResult.Code); + } + + [Fact] + public async Task Deliver_AlreadyProcessing_Skipped() + { + var ctx = new FakeOutboxContext( + payloadId: "msg-processing", + attempt: 0, + initialStatus: DeliveryStatusCode.Processing); + var messages = ToMessages(ctx); + + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.CompletedTask); + var courier = new DeliveryCourier(processor); + + var result = await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.Equal(0, result); + } + + #endregion + + #region Mixed batch — some succeed, some fail + + [Fact] + public async Task Deliver_MixedBatch_ProcessorSucceeds_PartialSuccess() + { + var ctxOk = new FakeOutboxContext(payloadId: "msg-ok", attempt: 0); + var ctxDone = new FakeOutboxContext( + payloadId: "msg-done", + attempt: 0, + initialStatus: DeliveryStatusCode.Ok); + var ctxPending = new FakeOutboxContext( + payloadId: "msg-pending", + attempt: 0, + initialStatus: DeliveryStatusCode.Pending); + + var messages = ToMessages(ctxOk, ctxDone, ctxPending); + + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.CompletedTask); + var courier = new DeliveryCourier(processor); + + var result = await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + CancellationToken.None); + + // msg-ok and msg-pending become Ok (2 successes), msg-done stays Ok but not counted + Assert.Equal(2, result); + Assert.Equal(DeliveryStatusCode.Ok, ctxOk.DeliveryResult.Code); + Assert.Equal(DeliveryStatusCode.Ok, ctxDone.DeliveryResult.Code); + Assert.Equal(DeliveryStatusCode.Ok, ctxPending.DeliveryResult.Code); + } + + #endregion + + #region Retry strategy integration + + [Fact] + public async Task Deliver_ProcessorThrows_CustomStrategyUsedNotShared() + { + var ctx = new FakeOutboxContext(payloadId: "custom-strategy", attempt: 5); + var messages = ToMessages(ctx); + + var testException = new InvalidOperationException("fail"); + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.FromException(testException)); + + var customStrategy = new FakeRetryStrategy(_ => TimeSpan.FromMilliseconds(42)); + var courier = new DeliveryCourier(processor, customStrategy); + + var result = await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.Equal(0, result); + } + + [Fact] + public async Task Deliver_ProcessorThrows_BackoffBasedOnCurrentAttempt() + { + var ctx = new FakeOutboxContext(payloadId: "attempt-aware", attempt: 0); + var messages = ToMessages(ctx); + + var testException = new InvalidOperationException("fail"); + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.FromException(testException)); + + var recordedAttempts = new List(); + var retryStrategy = new FakeRetryStrategy(attempt => + { + recordedAttempts.Add(attempt); + return TimeSpan.FromSeconds(attempt * 5); + }); + + var courier = new DeliveryCourier(processor, retryStrategy); + + await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.Single(recordedAttempts); + Assert.Equal(1, recordedAttempts[0]); // attempt 0 + 1 = 1 + Assert.Equal(TimeSpan.FromSeconds(5), ctx.PostponeAt); + } + + #endregion + + #region Cancellation + + [Fact] + public async Task Deliver_ProcessorCancelled_TreatedAsRegularError() + { + var ctx = new FakeOutboxContext(payloadId: "cancelled"); + var messages = ToMessages(ctx); + + var cts = new CancellationTokenSource(); + cts.Cancel(); + + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.FromCanceled(cts.Token)); + + var retryStrategy = new FakeRetryStrategy(_ => TimeSpan.FromSeconds(1)); + var courier = new DeliveryCourier(processor, retryStrategy); + + var result = await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + cts.Token); + + Assert.Equal(0, result); + Assert.Equal(DeliveryStatusCode.Warn, ctx.DeliveryResult.Code); + } + + #endregion + + #region Deterministic retry strategy (no jitter) + + [Fact] + public async Task Deliver_ProcessorThrows_DeterministicStrategy_PredictableDelays() + { + var ctx = new FakeOutboxContext(payloadId: "det-strategy", attempt: 0); + var messages = ToMessages(ctx); + + var testException = new InvalidOperationException("fail"); + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.FromException(testException)); + + // Strategy that always returns exactly 1 second regardless of attempt + var deterministicStrategy = new FakeRetryStrategy(_ => TimeSpan.FromSeconds(1)); + var courier = new DeliveryCourier(processor, deterministicStrategy); + + var result = await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.Equal(0, result); + Assert.Equal(TimeSpan.FromSeconds(1), ctx.PostponeAt); + } + + #endregion + + #region Exception types preserved + + [Fact] + public async Task Deliver_ProcessorThrows_PreservesOriginalExceptionType() + { + var ctx = new FakeOutboxContext(payloadId: "exception-type"); + var messages = ToMessages(ctx); + + var testException = new CustomTestException("specific error"); + var processor = new FakeDeliveryLifetimeInvoker(_ => Task.FromException(testException)); + + var retryStrategy = new FakeRetryStrategy(_ => TimeSpan.Zero); + var courier = new DeliveryCourier(processor, retryStrategy); + + await courier.Deliver( + CreateSettings(), + CreateFilter(), + messages, + CancellationToken.None); + + Assert.IsType(ctx.Exception); + Assert.Equal("specific error", ctx.Exception!.Message); + } + + private sealed class CustomTestException(string message) : Exception(message); + + #endregion +} diff --git a/src/Tests/Sa.Outbox.Tests/ExponentialBackoffRetryStrategyTests.cs b/src/Tests/Sa.Outbox.Tests/ExponentialBackoffRetryStrategyTests.cs new file mode 100644 index 00000000..0117ad68 --- /dev/null +++ b/src/Tests/Sa.Outbox.Tests/ExponentialBackoffRetryStrategyTests.cs @@ -0,0 +1,202 @@ +using Sa.Outbox.Delivery; + +namespace Sa.Outbox.Tests; + +public class ExponentialBackoffRetryStrategyTests +{ + [Fact] + public void Shared_Instance_Is_Not_Null() + { + Assert.NotNull(ExponentialBackoffRetryStrategy.Shared); + } + + [Fact] + public void GetBackoff_FirstAttempt_Returns_Base_Delay_Range() + { + var strategy = new ExponentialBackoffRetryStrategy( + baseDelay: TimeSpan.FromSeconds(5), + maxDelay: TimeSpan.FromMinutes(30)); + + var backoff = strategy.GetBackoff(1); + + // With jitter 0.5..1.0, first attempt (2^0 = 1) should be between 2.5s and 5s + Assert.True(backoff >= TimeSpan.FromSeconds(2.4)); + Assert.True(backoff <= TimeSpan.FromSeconds(5.1)); + } + + [Fact] + public void GetBackoff_SecondAttempt_Returns_Double_Base_Delay_Range() + { + var strategy = new ExponentialBackoffRetryStrategy( + baseDelay: TimeSpan.FromSeconds(5), + maxDelay: TimeSpan.FromMinutes(30)); + + var backoff = strategy.GetBackoff(2); + + // Second attempt: 5 * 2^1 = 10s, with jitter 0.5..1.0 → 5s..10s + Assert.True(backoff >= TimeSpan.FromSeconds(4.9)); + Assert.True(backoff <= TimeSpan.FromSeconds(10.1)); + } + + [Fact] + public void GetBackoff_HighAttempt_Capped_At_MaxDelay() + { + var strategy = new ExponentialBackoffRetryStrategy( + baseDelay: TimeSpan.FromSeconds(1), + maxDelay: TimeSpan.FromSeconds(10)); + + var backoff = strategy.GetBackoff(10); + + // 2^9 = 512, but capped at 10s, with jitter 0.5..1.0 → 5s..10s + Assert.True(backoff >= TimeSpan.FromSeconds(4.9)); + Assert.True(backoff <= TimeSpan.FromSeconds(10.1)); + } + + [Fact] + public void GetBackoff_IncreasingAttempts_Yield_IncreasingDelays_UntilCap() + { + var strategy = new ExponentialBackoffRetryStrategy( + baseDelay: TimeSpan.FromSeconds(1), + maxDelay: TimeSpan.FromMinutes(5)); + + var b1 = strategy.GetBackoff(1); + var b2 = strategy.GetBackoff(2); + var b3 = strategy.GetBackoff(3); + + // Generally increasing (jitter may cause occasional inversion, but trend should hold) + // We test the deterministic part: 1, 2, 4 seconds + Assert.True(b1 < b3); + Assert.True(b2 < b3); + } + + [Fact] + public void GetBackoff_AttemptZero_Treated_As_One() + { + var strategy = new ExponentialBackoffRetryStrategy( + baseDelay: TimeSpan.FromSeconds(5), + maxDelay: TimeSpan.FromMinutes(30)); + + var backoff0 = strategy.GetBackoff(0); + var backoff1 = strategy.GetBackoff(1); + + // Both should fall in the same range (2.5s..5s) + Assert.True(backoff0 >= TimeSpan.FromSeconds(2.4)); + Assert.True(backoff0 <= TimeSpan.FromSeconds(5.1)); + } + + [Fact] + public void Constructor_Clamps_Jitter_Factors() + { + var strategy = new ExponentialBackoffRetryStrategy( + jitterFactorMin: -0.5, // clamped to 0 + jitterFactorMax: 1.5); // clamped to 1.0 + + var backoff = strategy.GetBackoff(1); + // Should still produce a valid result + Assert.True(backoff > TimeSpan.Zero); + } + + [Fact] + public void GetBackoff_Custom_Jitter_Produces_Narrower_Range() + { + var strategy = new ExponentialBackoffRetryStrategy( + baseDelay: TimeSpan.FromSeconds(10), + maxDelay: TimeSpan.FromMinutes(1), + jitterFactorMin: 0.9, + jitterFactorMax: 0.9); + + var backoff = strategy.GetBackoff(1); + + // With jitter 0.9..0.9, first attempt should be very close to 9s + Assert.True(backoff >= TimeSpan.FromSeconds(8.8)); + Assert.True(backoff <= TimeSpan.FromSeconds(9.2)); + } + + [Fact] + public void GetBackoff_DeterministicWithFixedJitter_ProducesConsistentPattern() + { + var strategy = new ExponentialBackoffRetryStrategy( + baseDelay: TimeSpan.FromSeconds(1), + maxDelay: TimeSpan.FromMinutes(1), + jitterFactorMin: 0.95, + jitterFactorMax: 0.95); + + var backoffs = new List(); + for (int i = 1; i <= 5; i++) + { + backoffs.Add(strategy.GetBackoff(i)); + } + + // With deterministic jitter 0.95, delays should be strictly increasing until cap + Assert.True(backoffs[0] < backoffs[1]); + Assert.True(backoffs[1] < backoffs[2]); + Assert.True(backoffs[2] < backoffs[3]); + Assert.True(backoffs[3] < backoffs[4]); + } + + [Fact] + public void GetBackoff_ZeroAttempts_ReturnsSameRange_AsFirstAttempt() + { + var strategy = new ExponentialBackoffRetryStrategy( + baseDelay: TimeSpan.FromSeconds(2), + maxDelay: TimeSpan.FromMinutes(5)); + + var backoff0 = strategy.GetBackoff(0); + var backoffNeg = strategy.GetBackoff(-5); + + // Both should be in range of first attempt (base * jitter) + Assert.True(backoff0 >= TimeSpan.FromSeconds(0.9)); + Assert.True(backoff0 <= TimeSpan.FromSeconds(2.1)); + Assert.True(backoffNeg >= TimeSpan.FromSeconds(0.9)); + Assert.True(backoffNeg <= TimeSpan.FromSeconds(2.1)); + } + + [Fact] + public void GetBackoff_VeryLargeAttemptNumber_CappedAtMaxDelay() + { + var strategy = new ExponentialBackoffRetryStrategy( + baseDelay: TimeSpan.FromSeconds(1), + maxDelay: TimeSpan.FromSeconds(5)); + + var backoff = strategy.GetBackoff(100); + + // Even at attempt 100, should never exceed maxDelay + Assert.True(backoff <= TimeSpan.FromMilliseconds(5100)); + Assert.True(backoff > TimeSpan.Zero); + } + + [Fact] + public void Constructor_JitterMaxLessThanMin_SwapsCorrectly() + { + var strategy = new ExponentialBackoffRetryStrategy( + jitterFactorMin: 0.8, + jitterFactorMax: 0.3); + + var backoff = strategy.GetBackoff(1); + Assert.True(backoff > TimeSpan.Zero); + } + + [Fact] + public void Constructor_ZeroBaseDelay_UsesDefaultFiveSeconds() + { + var strategy = new ExponentialBackoffRetryStrategy( + baseDelay: TimeSpan.Zero, + maxDelay: TimeSpan.Zero); + + var backoff = strategy.GetBackoff(1); + + // Default base is 5s, so with jitter 0.5..1.0 → 2.5s..5s + Assert.True(backoff >= TimeSpan.FromSeconds(2.4)); + Assert.True(backoff <= TimeSpan.FromSeconds(5.1)); + } + + [Fact] + public void Shared_Instance_ProducesValidBackoffs() + { + for (int i = 1; i <= 10; i++) + { + var backoff = ExponentialBackoffRetryStrategy.Shared.GetBackoff(i); + Assert.True(backoff > TimeSpan.Zero); + } + } +} diff --git a/src/Tests/Sa.Outbox.Tests/FakeOutboxContext.cs b/src/Tests/Sa.Outbox.Tests/FakeOutboxContext.cs new file mode 100644 index 00000000..1e12125a --- /dev/null +++ b/src/Tests/Sa.Outbox.Tests/FakeOutboxContext.cs @@ -0,0 +1,153 @@ +namespace Sa.Outbox.Tests; + +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; +using Sa.Outbox.Delivery; +using Sa.Outbox.Exceptions; + +/// +/// Fake implementation of that accepts a payloadId parameter. +/// +public sealed class FakeOutboxContext( + string payloadId, + int attempt = 0, + DeliveryStatusCode initialStatus = DeliveryStatusCode.Pending) : IOutboxContextOperations +{ + private DeliveryStatus _deliveryResult = new(initialStatus, payloadId, DateTimeOffset.UtcNow); + private TimeSpan _postponeAt; + private Exception? _exception; + + public Guid OutboxId { get; set; } = Guid.NewGuid(); + public string PayloadId { get; set; } = payloadId; + public TMessage Payload { get; set; } = default!; + public OutboxPartInfo PartInfo { get; set; } = new(1, "part-1", DateTimeOffset.UtcNow); + + private int _attempt = attempt; + + public OutboxTaskDeliveryInfo DeliveryInfo + { + get + { + return new OutboxTaskDeliveryInfo(1L, 0L, _attempt, 0L, _deliveryResult, PartInfo); + } + } + + public DeliveryStatus DeliveryResult => _deliveryResult; + public Exception? Exception => _exception; + public TimeSpan PostponeAt => _postponeAt; + + public FakeOutboxContext() : this("fake-msg", 0, DeliveryStatusCode.Pending) { } + + public void Ok(string? message = null) + => SetStatus(DeliveryStatusCode.Ok, message); + + public void Created(string? message = null) + => SetStatus(DeliveryStatusCode.Created, message); + + public void Accepted(string? message = null) + => SetStatus(DeliveryStatusCode.Accepted, message); + + public void Ok203(string? message = null) + => SetStatus(DeliveryStatusCode.Ok203, message); + + public void NoContent(string? message = null) + => SetStatus(DeliveryStatusCode.NoContent, message); + + public void Aborted(string? message = null) + => SetStatus(DeliveryStatusCode.Aborted, message); + + public void MovedPermanently(string? message = null) + => SetStatus(DeliveryStatusCode.MovedPermanently, message); + + public void Postpone(TimeSpan postpone, string? message = null) + => SetStatus(DeliveryStatusCode.Postpone, message, postpone: postpone); + + public void Retry(TimeSpan postpone, string? message = null) + => SetStatus(DeliveryStatusCode.Retry, message, postpone: postpone); + + public void Warn(Exception exception, string? message = null, TimeSpan? postpone = null) + { + ArgumentNullException.ThrowIfNull(exception); + _exception = exception; + _postponeAt = postpone ?? TimeSpan.Zero; + _deliveryResult = new DeliveryStatus(DeliveryStatusCode.Warn, message ?? exception.Message, GetUtcNow()); + } + + public void Error(Exception exception, string? message = null) + => SetError(DeliveryStatusCode.Error, exception, message); + + public void Error501(Exception exception, string? message = null) + => SetError(DeliveryStatusCode.Error501, exception, message); + + public void Error502(Exception exception, string? message = null) + => SetError(DeliveryStatusCode.Error502, exception, message); + + public void Error503(Exception exception, string? message = null) + => SetError(DeliveryStatusCode.Error503, exception, message); + + public void Error504(Exception exception, string? message = null) + => SetError(DeliveryStatusCode.Error504, exception, message); + + public void Error505(Exception exception, string? message = null) + => SetError(DeliveryStatusCode.Error505, exception, message); + + public void Error506(Exception exception, string? message = null) + => SetError(DeliveryStatusCode.Error506, exception, message); + + public void Error507(Exception exception, string? message = null) + => SetError(DeliveryStatusCode.Error507, exception, message); + + public void ErrorMaxAttempts() + { + var permEx = new DeliveryPermanentException( + _exception?.Message ?? "Maximum delivery attempts exceeded", + statusCode: DeliveryStatusCode.MaximumAttemptsError); + SetStatus(DeliveryStatusCode.MaximumAttemptsError, permEx.Message, exception: permEx); + } + + public DateTimeOffset GetUtcNow() => DateTimeOffset.UtcNow; + + private void SetStatus(DeliveryStatusCode code, string? message, TimeSpan? postpone = null, Exception? exception = null) + { + _deliveryResult = new DeliveryStatus(code, message ?? "", GetUtcNow()); + _postponeAt = postpone ?? TimeSpan.Zero; + _exception = exception; + } + + private void SetError(DeliveryStatusCode code, Exception exception, string? message = null) + { + ArgumentNullException.ThrowIfNull(exception); + SetStatus(code, message ?? exception.Message, exception: exception); + } +} + +/// +/// Fake implementation of for testing. +/// +public sealed class FakeDeliveryLifetimeInvoker(Func consumeFn) : IDeliveryLifetimeInvoker +{ + public readonly List Invocations = []; + + public Task ConsumeInScope( + ConsumerGroupSettings settings, + OutboxMessageFilter filter, + ReadOnlyMemory> messages, + CancellationToken cancellationToken) + { + Invocations.Add(cancellationToken); + return consumeFn(cancellationToken); + } +} + +/// +/// Fake implementation of for testing. +/// +public sealed class FakeRetryStrategy(Func backoffFn) : IRetryStrategy +{ + public readonly List RecordedAttempts = []; + + public TimeSpan GetBackoff(int attemptNumber) + { + RecordedAttempts.Add(attemptNumber); + return backoffFn(attemptNumber); + } +} diff --git a/src/Tests/Sa.Outbox.Tests/Sa.Outbox.Tests.csproj b/src/Tests/Sa.Outbox.Tests/Sa.Outbox.Tests.csproj new file mode 100644 index 00000000..7999cab5 --- /dev/null +++ b/src/Tests/Sa.Outbox.Tests/Sa.Outbox.Tests.csproj @@ -0,0 +1,13 @@ + + + + + + true + + + + + + + From 6f394790354d3b14d36747022fe357521816e702 Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 26 Jun 2026 11:08:04 +0300 Subject: [PATCH 07/33] imrove Cron --- src/Sa.Outbox/Readme.md | 198 ++++++++- src/Sa.Schedule/Engine/CronTiming.cs | 334 +++++++++++++++ src/Sa.Schedule/Engine/IJobFactory.cs | 2 +- src/Sa.Schedule/Engine/JobErrorHandler.cs | 37 +- src/Sa.Schedule/Engine/JobFactory.cs | 9 +- src/Sa.Schedule/Engine/JobScheduler.cs | 24 +- src/Sa.Schedule/Engine/Scheduler.cs | 8 +- src/Sa.Schedule/IJobBuilder.cs | 51 +++ src/Sa.Schedule/IJobErrorHandlingBuilder.cs | 9 +- src/Sa.Schedule/IJobSnapshot.cs | 35 ++ src/Sa.Schedule/IJobTiming.cs | 16 + src/Sa.Schedule/JobException.cs | 49 ++- src/Sa.Schedule/Readme.md | 401 ++++++++++++++---- src/Sa.Schedule/Sa.Schedule.csproj | 4 +- src/Sa.Schedule/Settings/JobBuilder.cs | 10 +- src/Sa.Schedule/Settings/JobErrorHandling.cs | 7 +- .../{JobProperies.cs => JobProperties.cs} | 28 +- src/Sa.Schedule/Settings/JobSettings.cs | 2 +- src/Tests/Sa.ScheduleTests/CronTimingTests.cs | 313 ++++++++++++++ .../Sa.ScheduleTests/JobControllerTests.cs | 148 +++++++ .../JobErrorHandlerIntegrationTests.cs | 79 ++++ .../Sa.ScheduleTests/JobExceptionTests.cs | 167 ++++++++ .../Sa.ScheduleTests/JobSchedulerTests.cs | 54 +++ .../Sa.ScheduleTests/Sa.ScheduleTests.csproj | 4 + .../Sa.ScheduleTests/ScheduleBuilderTests.cs | 301 +++++++++++++ .../SchedulePostSetupTests.cs | 5 +- .../Sa.ScheduleTests/ScheduleSettingsTests.cs | 95 +++++ .../Sa.ScheduleTests/ScheduleSetupTests.cs | 7 +- 28 files changed, 2239 insertions(+), 158 deletions(-) create mode 100644 src/Sa.Schedule/Engine/CronTiming.cs create mode 100644 src/Sa.Schedule/IJobSnapshot.cs rename src/Sa.Schedule/Settings/{JobProperies.cs => JobProperties.cs} (72%) create mode 100644 src/Tests/Sa.ScheduleTests/CronTimingTests.cs create mode 100644 src/Tests/Sa.ScheduleTests/JobControllerTests.cs create mode 100644 src/Tests/Sa.ScheduleTests/JobErrorHandlerIntegrationTests.cs create mode 100644 src/Tests/Sa.ScheduleTests/JobExceptionTests.cs create mode 100644 src/Tests/Sa.ScheduleTests/ScheduleBuilderTests.cs create mode 100644 src/Tests/Sa.ScheduleTests/ScheduleSettingsTests.cs diff --git a/src/Sa.Outbox/Readme.md b/src/Sa.Outbox/Readme.md index 2e8a53ff..ebff10d5 100644 --- a/src/Sa.Outbox/Readme.md +++ b/src/Sa.Outbox/Readme.md @@ -1,39 +1,193 @@ -# Outbox +# Sa.Outbox -The base logic and abstractions designed for implementing the Outbox pattern, with support for partitioning. +Базовая инфраструктурная библиотека для реализации паттерна **Transactional Outbox** в распределённых .NET-системах. Гарантирует атомарную запись сообщения вместе с бизнес-операцией внутри одной транзакции БД и надёжную доставку с поддержкой повторных попыток, блокировок, многопоточности и мультитенантности. +Библиотека определяет абстракции и логику — конкретную работу с БД (PostgreSQL, SQL Server и т.д.) реализуют провайдеры-наследники (`Sa.Outbox.PostgreSql`, `Sa.Outbox.SqlServer` и др.). - -## Подключение +## Quick Start ### 1. Установите пакет провайдера + ```bash -# Для SQL Server -Install-Package Sa.Outbox.SqlServer +dotnet add package Sa.Outbox.PostgreSql ``` -### 2. Настройте в Program.cs +### 2. Настройте DI + ```csharp -// Minimal API -builder.Services.AddSaOutbox(); +builder.Services + .AddSaOutbox(builder => builder + .WithTenants((_, ts) => ts.WithTenantIds(1, 2, 3)) + .WithDeliveries(b => b.AddDelivery()) + ) + // провайдер (пример — PostgreSQL) + .AddSaOutboxUsingPostgreSql(cfg => cfg + .WithDataSource(ds => ds.WithConnectionString("Host=localhost;Database=outbox")) + ); ``` +### 3. Публикация сообщений + +```csharp +public sealed record OrderCreated(string OrderId); + +await publisher.Publish( + [new OrderCreated("ORD-001"), new OrderCreated("ORD-002")], + tenantId: 1); +``` + +### 4. Потребление сообщений + +```csharp +sealed class OrderCreatedConsumer : IConsumer +{ + public async ValueTask Consume( + ConsumerGroupSettings settings, + OutboxMessageFilter filter, + ReadOnlyMemory> messages, + CancellationToken cancellationToken) + { + foreach (var msg in messages.Span) + { + // обработка... + msg.Ok($"Processed order {msg.Payload.OrderId}"); + } + } +} +``` + +## Архитектура + +``` +┌──────────────┐ Publish ┌─────────────┐ +│ Application │ ───────────────► │ outbox__msg$│ +│ │ │ (source) │ +│ IConsumer │ └──────┬──────┘ +│ │ │ RentDelivery (SKIP LOCKED) +│ │ ▼ +└──────────────┘ ┌─────────────┐ + ▲ │ outbox │ + └── Ack/Warn/Error ◄─────┤ (queue) │ + └─────────────┘ +``` + +### Два этапа жизненного цикла + +| Этап | Описание | +|------|----------| +| **Publication** | Сообщения записываются в таблицу outbox внутри транзакции бизнес-операции через `IOutboxBulkWriter.InsertBulk()` | +| **Delivery** | Фоновые задачи (`Sa.Schedule`) захватывают заблокированные сообщения, вызывают потребителей, обновляют статус | + +## Основные типы + +| Тип | Назначение | +|-----|------------| +| `IOutboxBuilder` | Fluent-билдер для конфигурации outbox-системы | +| `IOutboxMessagePublisher` | Публикация сообщений в outbox | +| `IConsumer\` | Интерфейс потребителя сообщений | +| `IOutboxContextOperations\` | Операции изменения статуса доставки | +| `ConsumeSettings` | Настройки потребления (батчи, блокировки, повторы) | +| `ConsumerGroupSettings` | Группа потребителей + расписание | +| `DeliveryStatus` / `DeliveryStatusCode` | HTTP-подобные статусы доставки | +| `ExponentialBackoffRetryStrategy` | Экспоненциальный бэкофф с джиттером | +| `OutboxPartInfo` | Информация о части: TenantId, PartName | + +## Статусы доставки + +Полный набор HTTP-подобных кодов состояния: + +| Код | Статус | Значение | +|-----|--------|----------| +| 200 | `Ok()` | Успешно обработано | +| 201 | `Created()` | Создан побочный ресурс | +| 202 | `Accepted()` | Принято в обработку | +| 204 | `NoContent()` | Обработано, нет данных | +| 299 | `Aborted()` | Пропущено | +| 400 | `Warn()` | Временная ошибка → повтор | +| 500–508 | `Error()` | Постоянная ошибка | +| 508 | `ErrorMaxAttempts()` | Исчерпан максимум попыток | +| 103 | `Postpone()` | Отложенная обработка | +| 104 | `Retry()` | Повторить сейчас | + +## Конфигурация + +### Настройка потребителей + +```csharp +builder.Services.AddSaOutbox(builder => builder + .WithDeliveries(d => d + // Singleton delivery (один экземпляр на всё приложение) + .AddDelivery("orders", (sp, cs) => { + cs.ConsumeSettings + .WithMaxBatchSize(32) + .WithLockDuration(TimeSpan.FromSeconds(10)) + .WithMaxDeliveryAttempts(5); + cs.ScheduleSettings + .WithInterval(TimeSpan.FromSeconds(30)) + .WithInitialDelay(TimeSpan.FromSeconds(5)); + }) + // Scoped delivery (DI-скон на каждую доставку) + .AddDeliveryScoped("events") + ) +); +``` + +### Настройки потребления + +| Параметр | По умолчанию | Описание | +|----------|-------------|----------| +| `MaxBatchSize` | 16 | Макс. размер батча | +| `LockDuration` | 10 сек | Время блокировки сообщения | +| `LockRenewal` | 3 сек | Период продления блокировки | +| `MaxDeliveryAttempts` | 3 | Максимум попыток доставки | +| `LookbackInterval` | 7 дней | История обработки | +| `ConcurrencyLimit` | 1 | Одновременных задач | +| `PerTenantMaxDegreeOfParallelism` | 1 | Параллельность по тенантам | + +### Мультитенантность + +```csharp +.WithTenants((_, ts) => ts + .WithTenantIds(1, 2, 3) // Явный список + .WithAutoDetect() // Автоопределение из БД + .WithTenantDetector() // Кастомный детектор + .WithTenantParallelProcessing(3) // Параллельная обработка +) +``` + +### Метаданные сообщений + +```csharp +// Вариант 1: явное указание partName и PayloadId +options.AddMetadata(partName: "orders", getPayloadId: m => m.Id); + +// Вариант 2: из IOutboxPublishable +options.AddMetadata(); +``` + +## Доступные провайдеры + +| Провайдер | Пакет | Статус | +|-----------|-------|--------| +| PostgreSQL | `Sa.Outbox.PostgreSql` | ✅ production-ready | +| SQL Server | `Sa.Outbox.SqlServer` | 🔧 в разработке | +| Redis | `Sa.Outbox.Redis` | 🔧 в разработке | + +## Требования к провайдеру -## Реализация своего провайдера +Провайдер должен реализовать три ключевых интерфейса: +| Интерфейс | Назначение | +|-----------|------------| +| `IOutboxBulkWriter` | Массовая вставка сообщений в БД | +| `IOutboxDeliveryManager` | Управление блокировкой и выдачей сообщений | +| `ITenantSource` | Источник идентификаторов тенантов | -### Реализуйте обязательные интерфейсы: -- **`IOutboxBulkWriter`** - массовая запись сообщений -- **`IOutboxDeliveryManager`** - управление доставкой -- **`ITenantSource`** - поддержка мультитенантности +## Зависимости +- **Sa.Schedule** — планировщик фоновых задач +- Ссылочные классы из **Sa** (LockRenewer, MurmurHash3, Retry, расширения) -## 🛠 Доступные провайдеры -- ✅ **PostgreSQL** - `Sa.Outbox.Postgres` -- ✅ **Redis** - `Sa.Outbox.Redis` (в разработке) +## License -## 📝 Требования к реализации -1. **Idempotency** - гарантия однократной доставки -2. **Transactional** - согласованность с бизнес-операциями -3. **Tenant-aware** - поддержка изоляции клиентов -4. **Async** - полная асинхронность +MIT diff --git a/src/Sa.Schedule/Engine/CronTiming.cs b/src/Sa.Schedule/Engine/CronTiming.cs new file mode 100644 index 00000000..178bf217 --- /dev/null +++ b/src/Sa.Schedule/Engine/CronTiming.cs @@ -0,0 +1,334 @@ +using System.Linq; + +namespace Sa.Schedule.Engine; + +using System; +using System.Collections.Generic; + +/// +/// Implements cron-based scheduling using standard 5-field cron expressions. +/// Optimized with O(1) membership tests and precomputed jump tables (.NET 8–10). +/// +internal sealed class CronTiming : IJobTiming +{ + private const string DefaultName = "cron"; + + // Flag arrays for O(1) membership checks + private readonly bool[] _minuteFlags = new bool[60]; + private readonly bool[] _hourFlags = new bool[24]; + private readonly bool[] _domFlags = new bool[32]; // index 1..31 + private readonly bool[] _monthFlags = new bool[13]; // index 1..12 + private readonly bool[] _dowFlags = new bool[7]; // 0=Sunday..6=Saturday + + // Lookup tables: next valid value >= index (sentinel = -1) + private readonly int[] _nextMinute = new int[61]; + private readonly int[] _nextHour = new int[25]; + private readonly int[] _nextMonth = new int[14]; + + // First valid value in each field (used as default when jumping) + private readonly int _firstMinute; + private readonly int _firstHour; + private readonly int _firstMonth; + + private readonly bool _dowWildcard; + private readonly bool _domWildcard; + + public string TimingName { get; } + + public CronTiming(string expression, string? name = null) + { + TimingName = name ?? DefaultName; + var fields = ParseExpression(expression); + + // Minute + PopulateFlags(fields[0], _minuteFlags, 0, 59); + _firstMinute = BuildNextTable(_minuteFlags, _nextMinute, 0, 59); + + // Hour + PopulateFlags(fields[1], _hourFlags, 0, 23); + _firstHour = BuildNextTable(_hourFlags, _nextHour, 0, 23); + + // Day-of-month (no jump table needed, only flags) + PopulateFlags(fields[2], _domFlags, 1, 31); + + // Month + PopulateFlags(fields[3], _monthFlags, 1, 12); + _firstMonth = BuildNextTable(_monthFlags, _nextMonth, 1, 12); + + // Day-of-week + PopulateFlags(fields[4], _dowFlags, 0, 6); + + // Wildcard detection + _dowWildcard = AreAllFlagsSet(_dowFlags, 0, 6); + _domWildcard = AreAllFlagsSet(_domFlags, 1, 31); + } + + public static CronTiming Every(string expression, string? name = null) + => new(expression, name); + + public DateTimeOffset? GetNextOccurrence(DateTimeOffset dateTime, IJobContext context) + { + var candidate = TruncateToMinute(dateTime.AddMinutes(1)); + var maxSearch = dateTime.AddYears(2); + + while (candidate <= maxSearch) + { + if (Matches(candidate)) + return candidate; + candidate = Advance(candidate); + } + return null; + } + + private bool Matches(DateTimeOffset dt) => + _monthFlags[dt.Month] && + MatchDay(dt) && + _hourFlags[dt.Hour] && + _minuteFlags[dt.Minute]; + + private bool MatchDay(DateTimeOffset dt) + { + bool domMatch = _domFlags[dt.Day]; // 1..31 + bool dowMatch = _dowFlags[(int)dt.DayOfWeek]; // 0..6 + + if (!_domWildcard && !_dowWildcard) + return domMatch && dowMatch; // Both restricted → both must match + if (_domWildcard && _dowWildcard) + return true; // No restrictions → any day + return _domWildcard ? dowMatch : domMatch; + } + + private DateTimeOffset Advance(DateTimeOffset dt) + { + // 1. Try later minute this hour + int nextMin = dt.Minute + 1; + if (nextMin <= 59) + { + int m = _nextMinute[nextMin]; + if (m != -1) + return new DateTimeOffset(dt.Year, dt.Month, dt.Day, dt.Hour, m, 0, dt.Offset); + } + + // 2. Try later hour today (with first valid minute) + int nextH = dt.Hour + 1; + if (nextH <= 23) + { + int h = _nextHour[nextH]; + if (h != -1) + return new DateTimeOffset(dt.Year, dt.Month, dt.Day, h, _firstMinute, 0, dt.Offset); + } + + // 3. Jump to tomorrow + var nextDay = new DateTimeOffset(dt.Year, dt.Month, dt.Day, 0, 0, 0, dt.Offset).AddDays(1); + return FindEarliestOnOrAfter(nextDay); + } + + private DateTimeOffset FindEarliestOnOrAfter(DateTimeOffset from) + { + var maxYear = from.AddYears(2); + var current = from; + + while (current <= maxYear) + { + // Month skip + if (!_monthFlags[current.Month]) + { + int nm = _nextMonth[current.Month + 1]; + current = nm != -1 + ? new DateTimeOffset(current.Year, nm, 1, 0, 0, 0, current.Offset) + : new DateTimeOffset(current.Year + 1, _firstMonth, 1, 0, 0, 0, current.Offset); + continue; + } + + // Day skip + if (!MatchDay(current)) + { + current = current.AddDays(1); + continue; + } + + // First valid hour today + int hour = _nextHour[current.Hour]; + if (hour == -1) + { + current = current.AddDays(1); + continue; + } + + // First valid minute in that hour + int minute = _nextMinute[current.Minute]; + if (minute != -1) + return new DateTimeOffset(current.Year, current.Month, current.Day, hour, minute, 0, current.Offset); + + // No minute at this hour → try next valid hour + int nextHour = _nextHour[hour + 1]; + if (nextHour != -1) + return new DateTimeOffset(current.Year, current.Month, current.Day, nextHour, _firstMinute, 0, current.Offset); + + current = current.AddDays(1); + } + return from; // fallback (never reached) + } + + private static DateTimeOffset TruncateToMinute(DateTimeOffset dt) => + new(dt.Year, dt.Month, dt.Day, dt.Hour, dt.Minute, 0, dt.Offset); + + // ------------------------------------------------------- + // Initialization helpers + // ------------------------------------------------------- + private static void PopulateFlags(int[] values, bool[] flags, int min, int max) + { + foreach (var v in from int v in values + where v >= min && v <= max + select v) + { + flags[v] = true; + } + } + + /// Fills the jump table and returns the smallest valid value. + private static int BuildNextTable(bool[] flags, int[] next, int min, int max) + { + int lastValid = -1; + for (int i = max; i >= min; i--) + { + if (flags[i]) lastValid = i; + next[i] = lastValid; + } + next[max + 1] = -1; + for (int i = 0; i < min; i++) next[i] = -1; + + // Return the smallest valid value + for (int i = min; i <= max; i++) + if (flags[i]) + return i; + return -1; // no valid value (should not happen for well-formed expressions) + } + + private static bool AreAllFlagsSet(bool[] flags, int min, int max) + { + for (int i = min; i <= max; i++) + if (!flags[i]) return false; + return true; + } + + // ------------------------------------------------------- + // Parser (unchanged) + // ------------------------------------------------------- + private static int[][] ParseExpression(string expression) + { + if (string.IsNullOrWhiteSpace(expression)) + throw new FormatException("WithCron expression cannot be null or empty."); + + var fields = expression.Trim().Split([' ', '\t'], StringSplitOptions.RemoveEmptyEntries); + if (fields.Length != 5) + throw new FormatException($"WithCron expression must have exactly 5 fields, got {fields.Length}."); + + return + [ + ParseField(fields[0], "minute", 0, 59), + ParseField(fields[1], "hour", 0, 23), + ParseField(fields[2], "day-of-month", 1, 31), + ParseField(fields[3], "month", 1, 12), + ParseField(fields[4], "day-of-week", 0, 6), + ]; + } + + private static int[] ParseField(string field, string name, int min, int max) + { + var values = new List(); + + if (field == "*") + { + for (int i = min; i <= max; i++) values.Add(i); + } + else if (field.Contains('/')) + { + ParseStep(field, name, min, max, values); + } + else if (field.Contains('-')) + { + ParseRange(field, name, min, max, values); + } + else if (field.Contains(',')) + { + ParseList(field, name, min, max, values); + } + else if (int.TryParse(field, out int v)) + { + Validate(v, min, max, name); + values.Add(v); + } + else + { + throw new FormatException($"Invalid cron field '{field}' for {name}."); + } + + values.Sort(); + return [.. values]; + } + + private static void ParseStep(string field, string name, int min, int max, List values) + { + var parts = field.Split('/', 2); + if (parts.Length != 2) throw new FormatException($"Invalid step '{field}' for {name}."); + + int start, end; + if (parts[0] == "*") + { + start = min; end = max; + } + else if (parts[0].Contains('-')) + { + var rp = parts[0].Split('-', 2); + if (rp.Length != 2 || !int.TryParse(rp[0], out start) || !int.TryParse(rp[1], out end)) + throw new FormatException($"Invalid range/step '{parts[0]}' for {name}."); + start = Validate(start, min, max, name); + end = Validate(end, min, max, name); + } + else if (int.TryParse(parts[0], out int s)) + { + start = Validate(s, min, max, name); + end = max; + } + else + { + throw new FormatException($"Invalid step start '{parts[0]}' for {name}."); + } + + if (!int.TryParse(parts[1], out int step) || step <= 0) + throw new FormatException($"Invalid step value '{parts[1]}' for {name}."); + + for (int i = start; i <= end; i += step) + values.Add(i); + } + + private static void ParseRange(string field, string name, int min, int max, List values) + { + var parts = field.Split('-', 2); + if (parts.Length != 2 || !int.TryParse(parts[0], out int s) || !int.TryParse(parts[1], out int e)) + throw new FormatException($"Invalid range '{field}' for {name}."); + + s = Validate(s, min, max, name); + e = Validate(e, min, max, name); + if (s > e) throw new FormatException($"Range {s}-{e} invalid for {name}."); + for (int i = s; i <= e; i++) values.Add(i); + } + + private static void ParseList(string field, string name, int min, int max, List values) + { + foreach (var part in field.Split(',', StringSplitOptions.RemoveEmptyEntries)) + { + if (int.TryParse(part.Trim(), out int v)) + values.Add(Validate(v, min, max, name)); + else + throw new FormatException($"Invalid value '{part}' in list for {name}."); + } + } + + private static int Validate(int v, int min, int max, string name) + { + if (v < min || v > max) throw new FormatException($"Value {v} out of range [{min}-{max}] for {name}."); + return v; + } +} diff --git a/src/Sa.Schedule/Engine/IJobFactory.cs b/src/Sa.Schedule/Engine/IJobFactory.cs index 2ec3c723..90738035 100644 --- a/src/Sa.Schedule/Engine/IJobFactory.cs +++ b/src/Sa.Schedule/Engine/IJobFactory.cs @@ -2,5 +2,5 @@ internal interface IJobFactory { - IJobScheduler CreateJobSchedule(IJobSettings settings); + IJobScheduler? CreateJobSchedule(IJobSettings settings); } diff --git a/src/Sa.Schedule/Engine/JobErrorHandler.cs b/src/Sa.Schedule/Engine/JobErrorHandler.cs index bbac4acb..8a389a90 100644 --- a/src/Sa.Schedule/Engine/JobErrorHandler.cs +++ b/src/Sa.Schedule/Engine/JobErrorHandler.cs @@ -11,13 +11,13 @@ internal sealed partial class JobErrorHandler( { public void HandleError(IJobContext context, Exception exception) { - if (settings.HandleError?.Invoke(context, exception) != true) + if (settings.HandleError?.Invoke(context, exception) == true) { - // default handle - DoHandleError(context, exception); + // Global handler consumed the error — do not rethrow + return; } - throw exception; + DoHandleError(context, exception); } private void DoHandleError(IJobContext context, Exception exception) @@ -29,15 +29,11 @@ private void DoHandleError(IJobContext context, Exception exception) break; case ErrorHandlingAction.CloseApplication: - CloseApplication( - context.JobName, - context.ServiceProvider.GetRequiredService(), exception); + CloseApplication(context.JobName, exception); break; case ErrorHandlingAction.StopAllJobs: - StopAllJobs( - context.JobName, - context.ServiceProvider.GetRequiredService(), exception); + StopAllJobs(context.JobName, context); break; default: @@ -48,27 +44,24 @@ private void DoHandleError(IJobContext context, Exception exception) throw context.LastError ?? exception; } - private void StopAllJobs(string jobName, IScheduler scheduler, Exception exception) + private void StopAllJobs(string jobName, IJobContext context) { - LogStopAllJobs(jobName, exception.ToString()); + LogStopAllJobs(jobName, context.LastError?.ToString() ?? string.Empty); - if (scheduler == null) throw exception; - scheduler.Stop(); + var scheduler = context.ServiceProvider.GetService(); + scheduler?.Stop(); } - private void CloseApplication(string jobName, IScheduler scheduler, Exception exception) + private void CloseApplication(string jobName, Exception exception) { LogCloseApplication(jobName, exception.ToString()); - if (lifetime == null) throw exception; + if (lifetime == null) return; - if (scheduler.Settings.IsHostedService) + if (lifetime is IHostApplicationLifetime hostAppLifetime) { - lifetime.StopApplication(); - } - else - { - scheduler.Stop().ContinueWith(_ => lifetime.StopApplication()); + // Safe fire-and-forget — StopApplication is designed for this + hostAppLifetime.StopApplication(); } } diff --git a/src/Sa.Schedule/Engine/JobFactory.cs b/src/Sa.Schedule/Engine/JobFactory.cs index 0dd03eb2..27a57748 100644 --- a/src/Sa.Schedule/Engine/JobFactory.cs +++ b/src/Sa.Schedule/Engine/JobFactory.cs @@ -9,11 +9,16 @@ internal sealed class JobFactory( IJobRunner jobRunner, TimeProvider? timeProvider = null) : IJobFactory { - public IJobScheduler CreateJobSchedule(IJobSettings settings) - => new JobScheduler( + public IJobScheduler? CreateJobSchedule(IJobSettings settings) + { + if (settings.Properties.Disabled == true) + return null; + + return new JobScheduler( settings, jobRunner, i => CreateController(i, settings)); + } private JobController CreateController(int index, IJobSettings settings) { diff --git a/src/Sa.Schedule/Engine/JobScheduler.cs b/src/Sa.Schedule/Engine/JobScheduler.cs index 47e96629..ee16e780 100644 --- a/src/Sa.Schedule/Engine/JobScheduler.cs +++ b/src/Sa.Schedule/Engine/JobScheduler.cs @@ -25,6 +25,8 @@ private readonly static IChangeToken NoneChangeToken private IReadOnlyList _jobControllers = []; + private static readonly TimeSpan DefaultShutdownTimeout = TimeSpan.FromSeconds(30); + public JobScheduler( IJobSettings settings, @@ -129,6 +131,15 @@ public async Task Start(CancellationToken cancellationToken) await _jobs.Enqueue(controller, stoppingToken); } } + catch (OperationCanceledException) + { + // Shutdown requested during startup — dispose already-enqueued controllers + foreach (var controller in controllers) + { + controller.Shutdown(); + } + return false; + } finally { lock (_lock) @@ -178,7 +189,18 @@ public async Task Stop() _started = false; } - await _jobs.WaitForIdleAsync(_originalToken); + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource( + _originalToken, CancellationToken.None); + timeoutCts.CancelAfter(DefaultShutdownTimeout); + + try + { + await _jobs.WaitForIdleAsync(timeoutCts.Token); + } + catch (OperationCanceledException) + { + // Timeout or original cancellation — log but don't block forever + } } public void Dispose() diff --git a/src/Sa.Schedule/Engine/Scheduler.cs b/src/Sa.Schedule/Engine/Scheduler.cs index 42fb5eb2..689bcc03 100644 --- a/src/Sa.Schedule/Engine/Scheduler.cs +++ b/src/Sa.Schedule/Engine/Scheduler.cs @@ -9,14 +9,18 @@ internal sealed class Scheduler(IScheduleSettings settings, IJobFactory factory) public IReadOnlyCollection Schedules { get; } = [.. settings .GetJobSettings() - .Select(factory.CreateJobSchedule)]; + .Select(factory.CreateJobSchedule) + .OfType()]; /// /// Start all jobs /// public async Task Start(CancellationToken cancellationToken) { - var results = await Task.WhenAll(Schedules.Select(c => c.Start(cancellationToken))); + var results = await Task.WhenAll( + Schedules + .Where(s => s.ConcurrencyLimit >= 0) + .Select(c => c.Start(cancellationToken))); return results.Count(r => r); } diff --git a/src/Sa.Schedule/IJobBuilder.cs b/src/Sa.Schedule/IJobBuilder.cs index 4543323e..78fe873e 100644 --- a/src/Sa.Schedule/IJobBuilder.cs +++ b/src/Sa.Schedule/IJobBuilder.cs @@ -76,11 +76,62 @@ IJobBuilder EverySeconds(int seconds = 1) IJobBuilder EveryMinutes(int minutes = 1) => EveryTime(TimeSpan.FromMinutes(minutes), $"every {minutes} minutes"); + /// + /// Configures the job to run every specified number of hours. + /// + /// The number of hours (default is 1). + /// The current builder instance. + IJobBuilder EveryHours(int hours = 1) + => EveryTime(TimeSpan.FromHours(hours), $"every {hours} hours"); + + /// + /// Configures the job to run every specified number of days. + /// + /// The number of days (default is 1). + /// The current builder instance. + IJobBuilder EveryDays(int days = 1) + => EveryTime(TimeSpan.FromDays(days), $"every {days} days"); + + /// + /// Configures the job to run once after the specified delay. + /// + /// The delay before the single execution. + /// The current builder instance. + IJobBuilder OnceIn(TimeSpan delay) + => WithInitialDelay(delay).RunOnce(); + + /// + /// Configures the job using a cron expression for scheduling. + /// Format: "minute hour day-of-month month day-of-week" + /// + /// Examples: + /// "0 9 * * *" — Every day at 9:00 AM + /// "0 */2 * * *" — Every 2 hours at minute 0 + /// "30 14 * * 1-5" — Weekdays (Mon-Fri) at 2:30 PM + /// "0 0 1 * *" — First day of every month at midnight + /// + /// The cron expression. + /// Optional display name for the timing. + /// The current builder instance. + IJobBuilder WithCron(string cronExpression, string? name = null); + IJobBuilder WithConcurrencyLimit(int limit); + /// + /// Sets the maximum concurrency limit for the job. + /// + /// The maximum concurrency limit. + /// The current builder instance. IJobBuilder WithMaxConcurrency(int limit); + /// + /// Sets the maximum concurrency limit for the job (alias for ). + /// + /// The maximum concurrency limit. + /// The current builder instance. + IJobBuilder WithMaxConcurrencyLimit(int limit) => WithMaxConcurrency(limit); + /// /// Merges the specified job properties into the current job configuration. diff --git a/src/Sa.Schedule/IJobErrorHandlingBuilder.cs b/src/Sa.Schedule/IJobErrorHandlingBuilder.cs index 1cb330f6..175a0bca 100644 --- a/src/Sa.Schedule/IJobErrorHandlingBuilder.cs +++ b/src/Sa.Schedule/IJobErrorHandlingBuilder.cs @@ -15,6 +15,12 @@ public interface IJobErrorHandlingBuilder /// The current IJobErrorHandlingBuilder instance. IJobErrorHandlingBuilder ThenCloseApplication(); + /// + /// Specifies that the current job should be aborted (stopped) if an error occurs. + /// + /// The current IJobErrorHandlingBuilder instance. + IJobErrorHandlingBuilder ThenAbortJob(); + /// /// Specifies that all jobs should be stopped if an error occurs. /// @@ -25,7 +31,8 @@ public interface IJobErrorHandlingBuilder /// Specifies that the current job should be stopped if an error occurs. /// /// The current IJobErrorHandlingBuilder instance. - IJobErrorHandlingBuilder ThenStopJob(); + [Obsolete("Use ThenAbortJob instead. This method will be removed in a future version.")] + IJobErrorHandlingBuilder ThenStopJob() => ThenAbortJob(); /// /// Specifies a custom error suppression policy. diff --git a/src/Sa.Schedule/IJobSnapshot.cs b/src/Sa.Schedule/IJobSnapshot.cs new file mode 100644 index 00000000..63cf9a3a --- /dev/null +++ b/src/Sa.Schedule/IJobSnapshot.cs @@ -0,0 +1,35 @@ +namespace Sa.Schedule; + +/// +/// Read-only snapshot of job context properties captured at the time of an error. +/// Designed to be lightweight — avoids cloning the full context stack. +/// +public interface IJobSnapshot +{ + /// Gets the job name. + string JobName { get; } + + /// Gets the total number of iterations attempted. + ulong NumIterations { get; } + + /// Gets the number of failed iterations. + ulong FailedIterations { get; } + + /// Gets the number of completed (successful) iterations. + ulong CompetedIterations { get; } + + /// Gets the time when the job was first created. + DateTimeOffset CreatedAt { get; } + + /// Gets the time of the last execution, if any. + DateTimeOffset? ExecuteAt { get; } + + /// Gets the number of retry attempts for the current failure. + int FailedRetries { get; } + + /// Gets the message of the last error, if any. + string? LastErrorMessage { get; } + + /// Gets the number of previous context entries on the stack (capped at 10). + int StackDepth { get; } +} diff --git a/src/Sa.Schedule/IJobTiming.cs b/src/Sa.Schedule/IJobTiming.cs index 144289c7..ef5d62b3 100644 --- a/src/Sa.Schedule/IJobTiming.cs +++ b/src/Sa.Schedule/IJobTiming.cs @@ -17,4 +17,20 @@ public interface IJobTiming /// The job context. /// The next occurrence of the job timing, or null if no next occurrence is found. DateTimeOffset? GetNextOccurrence(DateTimeOffset dateTime, IJobContext context); + + /// + /// Creates a cron-based timing from a standard 5-field cron expression. + /// Format: "minute hour day-of-month month day-of-week" + /// + /// Examples: + /// "0 9 * * *" — Every day at 9:00 AM + /// "0 */2 * * *" — Every 2 hours at minute 0 + /// "30 14 * * 1-5" — Weekdays (Mon-Fri) at 2:30 PM + /// "0 0 1 * *" — First day of every month at midnight + /// + /// The cron expression. + /// Optional display name. + /// An IJobTiming configured with cron scheduling. + static IJobTiming FromCron(string expression, string? name = null) + => new Engine.CronTiming(expression, name); } diff --git a/src/Sa.Schedule/JobException.cs b/src/Sa.Schedule/JobException.cs index eac6df31..bb88dc25 100644 --- a/src/Sa.Schedule/JobException.cs +++ b/src/Sa.Schedule/JobException.cs @@ -1,7 +1,54 @@ namespace Sa.Schedule; +/// +/// Represents an error that occurred during job execution. +/// +/// The job context at the time of the error. +/// The underlying exception that caused this error. public class JobException(IJobContext context, Exception? innerException) : Exception($"[{context.JobName}] job error", innerException) { - public IJobContext JobContext { get; } = context.Clone(); + /// + /// Gets a lightweight snapshot of the job context at the time of the error. + /// Contains only scalar properties and stack depth (capped at 10). + /// Avoids cloning the full context stack. + /// + public IJobSnapshot ContextSnapshot { get; } = new JobSnapshot(context); + + private sealed class JobSnapshot : IJobSnapshot + { + public string JobName { get; } + public ulong NumIterations { get; } + public ulong FailedIterations { get; } + public ulong CompetedIterations { get; } + public DateTimeOffset CreatedAt { get; } + public DateTimeOffset? ExecuteAt { get; } + public int FailedRetries { get; } + public string? LastErrorMessage { get; } + public int StackDepth { get; } + + public JobSnapshot(IJobContext context) + { + JobName = context.JobName; + NumIterations = context.NumIterations; + FailedIterations = context.FailedIterations; + CompetedIterations = context.CompetedIterations; + CreatedAt = context.CreatedAt; + ExecuteAt = context.ExecuteAt; + FailedRetries = context.FailedRetries; + LastErrorMessage = context.LastError?.Message; + + // Count stack depth without cloning (cap at 10) + var count = 0; + if (context.Stack != null) + { + foreach (var _entry in context.Stack) + { + count++; + if (count >= 10) break; + } + } + StackDepth = count; + } + } } diff --git a/src/Sa.Schedule/Readme.md b/src/Sa.Schedule/Readme.md index 5780be19..7fb08b22 100644 --- a/src/Sa.Schedule/Readme.md +++ b/src/Sa.Schedule/Readme.md @@ -1,113 +1,350 @@ # Sa.Schedule -The Sa.Schedule library provides a way to configure and execute scheduled tasks. It allows you to manage a set of tasks that will be executed at a specific time or at a defined frequency. +The **Sa.Schedule** library provides a robust, production-ready framework for configuring and executing scheduled tasks in .NET applications. It supports periodic jobs, one-shot executions, dynamic concurrency control, error recovery strategies, interceptors, and graceful shutdown. +--- -## Example Usage - -### Configuring Schedule DI +## Quick Start ```csharp -Services.AddSaSchedule(b => +var builder = Host.CreateEmptyApplicationBuilder(args); + +builder.Services.AddSaSchedule(b => { - b.AddJob((sp, builder) => - { - builder - .EveryTime(TimeSpan.FromMilliseconds(100)) - .RunOnce() - .StartImmediate(); - }); + b.UseHostedService() + .AddJob((sp, job) => + { + job.EveryMinutes(5) + .WithName("Database cleanup") + .WithConcurrencyLimit(2) + .ConfigureErrorHandling(err => err + .IfErrorRetry(3) + .ThenAbortJob()); + }) + .AddJob(id: Guid.Parse("xxxx-xxxx")) + .EveryHours(1) + .StartImmediate(); }); + +var app = builder.Build(); +await app.RunAsync(); ``` -### Job +--- + +## Defining Jobs -A job implements the IJob interface and the Execute method, which contains the main logic. +Jobs implement the `IJob` interface: ```csharp -class SomeJob : IJob +public class CleanupJob : IJob { - // IJobContext provides access to the execution context + private readonly ILogger _logger; + private readonly IDbConnection _db; + + public CleanupJob(ILogger logger, IDbConnection db) + { + _logger = logger; + _db = db; + } + public async Task Execute(IJobContext context, CancellationToken cancellationToken) { - await Task.Delay(10, cancellationToken); + _logger.LogInformation("Running cleanup — iteration #{Num}", context.NumIterations); + await _db.ExecuteAsync("DELETE FROM temp_table WHERE created_at < @now", + new { now = DateTimeOffset.UtcNow }, cancellationToken); } } ``` +Scoped services (DbContext, IDbConnection, etc.) are resolved automatically within an DI scope per execution. + +### Lambda Jobs + +For quick one-off tasks without a dedicated class: + +```csharp +b.AddJob((context, ct) => +{ + Console.WriteLine($"Hello at {context.ExecuteAt}"); + return Task.CompletedTask; +}, jobId: Guid.NewGuid()) + .EverySeconds(10); +``` + +--- + +## Job Configuration (Builder API) + +| Method | Description | +|---|---| +| `.WithName(string)` | Human-readable job name | +| `.StartImmediate()` | Execute on first start without waiting for the interval | +| `.RunOnce()` | Execute exactly once, then stop permanently | +| `.WithInitialDelay(TimeSpan)` | Delay before the first execution | +| `.EveryTime(TimeSpan, string?)` | Periodic interval with optional timing name | +| `.EverySeconds(int)` | Convenience alias for seconds | +| `.EveryMinutes(int)` | Convenience alias for minutes | +| `.EveryHours(int)` | Convenience alias for hours | +| `.EveryDays(int)` | Convenience alias for days | +| `.OnceIn(TimeSpan)` | Run once after a delay | +| `.Cron(string, string?)` | Schedule using cron expression (minute hour dayOfMonth month dayOfWeek) | +| `.WithContextStackSize(int)` | Keep N previous contexts on a stack for debugging | +| `.WithTag(object)` | Attach arbitrary metadata | +| `.WithConcurrencyLimit(int)` | Number of concurrent executions | +| `.WithMaxConcurrency(int)` | Maximum slots allocated | +| `.Disabled()` | Register but don't start | +| `.Merge(IJobProperties)` | Merge another configuration | +| `.ConfigureErrorHandling(Action)` | Error recovery policy | + +### Cron Scheduling + +Use cron expressions for precise scheduling control. The format follows standard 5-field cron: + +``` +minute hour day-of-month month day-of-week +``` + +**Supported features:** +- `*` — wildcard (any value) +- `,` — comma-separated list (e.g., `1,15,30`) +- `-` — range (e.g., `1-5`) +- `/` — step values (e.g., `*/5`, `1-20/3`) + +**Examples:** + +```csharp +// Every day at 9:00 AM +b.AddJob() + .Cron("0 9 * * *") + .WithName("Daily report"); + +// Every 2 hours at minute 0 +b.AddJob() + .Cron("0 */2 * * *") + .WithName("Hourly sync"); + +// Weekdays (Mon-Fri) at 2:30 PM +b.AddJob() + .Cron("30 14 * * 1-5") + .WithName("Weekday cleanup"); + +// First day of every month at midnight +b.AddJob[MonthlyBackup]() + .Cron("0 0 1 * *") + .WithName("Monthly backup"); + +// Every Monday, Wednesday, Friday at 6:00 AM +b.AddJob[TriWeeklyTask]() + .Cron("0 6 * * 1,3,5") + .WithName("Tri-weekly task"); + +// Every 15 minutes +b.AddJob[HealthCheck]() + .Cron("*/15 * * * *") + .WithName("Health check"); + +// Combined range and step: every 3rd hour from 9 AM to 5 PM +b.AddJob[BusinessMetrics]() + .Cron("0 9-17/3 * * 1-5") + .WithName("Business metrics"); +``` + +**Advanced examples:** + +```csharp +// Last day of month (approximate — use 28-31 and let cron filter) +b.AddJob[EndOfMonthReport]() + .Cron("0 0 28-31 * *") + .WithName("End of month report"); + +// Leap year only (Feb 29) +b.AddJob[LeapYearTask]() + .Cron("0 0 29 2 *") + .WithName("Leap year task"); + +// Multiple days of week (Mon, Wed, Fri at 9:00 and 17:00) +b.AddJob[PeakMonitor]() + .Cron("0 9,17 * * 1,3,5") + .WithName("Peak monitoring"); +``` + +### Concurrency Model + +- **`ConcurrencyLimit`** — how many slots are actively running at any time (initially). Can be changed dynamically via `IJobScheduler.ConcurrencyLimit`. +- **`MaxConcurrency`** — total number of slot pre-allocated. `ConcurrencyLimit ≤ MaxConcurrency`. +- Dynamic adjustment pauses/resumes individual slots without recreating them. + +--- + +## Error Handling + +Each job defines its own error policy: + +```csharp +.ConfigureErrorHandling(err => err + .IfErrorRetry(count: 3) // Retry up to 3 times + .DoSuppressError(ex => ex is TimeoutException) // Suppress timeouts silently + .ThenAbortJob()) // After retries exhausted, stop this job only +``` + +### Error Handling Actions -## Managing Schedules +| Action | Behavior | +|---|---| +| `CloseApplication` | Stop the entire application via `IHostApplicationLifetime.StopApplication()` (**default**) | +| `AbortJob` | Stop only the current job; other jobs continue | +| `StopAllJobs` | Stop all registered jobs | -Management is done through the IScheduler and IJobScheduler interfaces. +### Global Error Handler + +Register a global handler that runs *before* per-job handling: ```csharp -/// -/// This scheduler that manages multiple job schedulers. -/// -public interface IScheduler +b.AddErrorHandler((context, exception) => { - /// - /// Gets the schedule settings. - /// - IScheduleSettings Settings { get; } - - /// - /// Gets the collection of job schedulers. - /// - IReadOnlyCollection Schedules { get; } - - /// - /// Starts the scheduler. - /// - /// The cancellation token. - /// The number of jobs started. - int Start(CancellationToken cancellationToken); - - /// - /// Restarts the scheduler. - /// - int Restart(); - - /// - /// Stops the scheduler. - /// - Task Stop(); + // Return true to consume (suppress) the error + // Return false to let per-job handling decide + if (exception is InvalidOperationException) + { + context.Logger.LogWarning("Known issue: {Msg}", exception.Message); + return true; + } + return false; +}); +``` + +### JobException + +When a job throws, it's wrapped in `JobException` containing: +- `JobContext` — full context at failure time +- `ContextSnapshot` — lightweight snapshot (scalar properties + stack depth), avoids expensive deep clone +- `InnerException` — the original exception + +--- + +## Interceptors + +Interceptors wrap every job execution, implementing chain-of-responsibility: + +```csharp +public class LoggingInterceptor : IJobInterceptor +{ + private readonly ILogger _logger; + + public LoggingInterceptor(ILogger logger) + => _logger = logger; + + public async Task OnHandle(IJobContext context, Func next, object? key, CancellationToken ct) + { + _logger.LogInformation("[{Job}] Starting", context.JobName); + var sw = Stopwatch.StartNew(); + try + { + await next(); + sw.Stop(); + _logger.LogInformation("[{Job}] Completed in {Ms}ms", context.JobName, sw.ElapsedMilliseconds); + } + catch (Exception ex) + { + sw.Stop(); + _logger.LogError(ex, "[{Job}] Failed after {Ms}ms", context.JobName, sw.ElapsedMilliseconds); + throw; + } + } } -/// -/// This individual task scheduler is responsible for managing specific tasks. -/// -public interface IJobScheduler +// Register globally +b.AddInterceptor(); +``` + +Multiple interceptors can be registered — they apply in LIFO order (last added = outermost wrapper). + +--- + +## Runtime Management + +Access the scheduler via DI: + +```csharp +public class Controller { - /// - /// Gets a value indicating whether the job scheduler is currently active. - /// - bool IsActive { get; } - - /// - /// Gets the context associated with the job scheduler. - /// - IJobContext Context { get; } - - /// - /// Gets a change token that can be used to track changes to the active state of the scheduler. - /// - IChangeToken GetActiveChangeToken(); - - /// - /// Starts the job scheduler asynchronously. - /// - bool Start(CancellationToken cancellationToken); - - /// - /// Restarts the job scheduler. - /// - bool Restart(); - - /// - /// Stops the job scheduler. - /// - Task Stop(); + private readonly IScheduler _scheduler; + + public Controller(IScheduler scheduler) + => _scheduler = scheduler; + + public async Task RestartAll() + { + var count = await _scheduler.Restart(TestContext.Current.CancellationToken); + Console.WriteLine($"Restarted {count} jobs"); + } + + public async Task StopAll() + => await _scheduler.Stop(); + + public void ChangeConcurrency(Guid jobId, int newLimit) + { + var schedule = _scheduler.GetSchedule(jobId); + schedule?.ConcurrencyLimit = newLimit; + } } +``` + +### IScheduler + +| Member | Description | +|---|---| +| `Settings` | Schedule-wide settings | +| `Schedules` | Collection of `IJobScheduler` | +| `Start(ct)` | Start all non-disabled jobs | +| `Restart(ct)` | Stop + restart all started jobs | +| `Stop()` | Graceful stop with 30s timeout | +| `GetSchedule(id)` | Find a specific job scheduler | + +### IJobScheduler + +| Member | Description | +|---|---| +| `JobId` | Unique identifier | +| `IsStarted` | Whether the job is currently running | +| `ActiveTasks` | Pending tasks in queue | +| `ConcurrencyLimit` | Get/set active concurrency | +| `StartChangeToken()` | Track start/stop state changes | +| `Start(ct)` | Start this job | +| `Stop()` | Stop with timeout | + +--- +## Architecture + +``` +DI Setup (Setup.cs + ScheduleBuilder.cs) + ↓ +Configuration (JobSettings, JobProperties, JobErrorHandling) + ↓ +Factory (JobFactory → creates IJobScheduler) + ↓ +Scheduler (IScheduler → manages IReadOnlyCollection) + ↓ +JobScheduler (one per IJob, backed by SaWorkQueue) + ↓ +JobController (pre-allocated slots, pause/resume via SemaphoreSlim) + ↓ +JobExecutor (DI scope + interceptor chain) + ↓ +IJob.Execute(...) ``` + +--- + +## Best Practices + +1. **Always use `UseHostedService()`** — integrates with Generic Host lifecycle +2. **Prefer typed jobs over lambdas** — better testability and DI resolution +3. **Set `ConcurrencyLimit` appropriately** — avoid overwhelming downstream systems +4. **Use `DoSuppressError` for transient failures** — don't crash on recoverable errors +5. **Add interceptors for cross-cutting concerns** — logging, metrics, distributed tracing +6. **Monitor via `IJobScheduler.IsStarted` and `ActiveTasks`** — integrate with health checks +7. **Use `OnceIn(TimeSpan)` for migration jobs** — run once after deployment delay +8. **Disable jobs instead of removing** — useful for feature flags and gradual rollout diff --git a/src/Sa.Schedule/Sa.Schedule.csproj b/src/Sa.Schedule/Sa.Schedule.csproj index efa3060c..b2728de6 100644 --- a/src/Sa.Schedule/Sa.Schedule.csproj +++ b/src/Sa.Schedule/Sa.Schedule.csproj @@ -8,11 +8,11 @@ - 1701;1702;CS8602; + 1701;1702; - 1701;1702;CS8602; + 1701;1702; diff --git a/src/Sa.Schedule/Settings/JobBuilder.cs b/src/Sa.Schedule/Settings/JobBuilder.cs index fcce2385..a24b51fd 100644 --- a/src/Sa.Schedule/Settings/JobBuilder.cs +++ b/src/Sa.Schedule/Settings/JobBuilder.cs @@ -1,4 +1,6 @@ -namespace Sa.Schedule.Settings; +using Sa.Schedule.Engine; + +namespace Sa.Schedule.Settings; internal sealed class JobBuilder(JobSettings settings) : IJobBuilder { @@ -79,4 +81,10 @@ public IJobBuilder Disabled() settings.Properties.SetDisabled(); return this; } + + public IJobBuilder WithCron(string cronExpression, string? name = null) + { + settings.Properties.WithTiming(CronTiming.Every(cronExpression, name)); + return this; + } } diff --git a/src/Sa.Schedule/Settings/JobErrorHandling.cs b/src/Sa.Schedule/Settings/JobErrorHandling.cs index 2d911ea9..c7ddf11b 100644 --- a/src/Sa.Schedule/Settings/JobErrorHandling.cs +++ b/src/Sa.Schedule/Settings/JobErrorHandling.cs @@ -38,12 +38,17 @@ public IJobErrorHandlingBuilder ThenCloseApplication() return this; } - public IJobErrorHandlingBuilder ThenStopJob() + public IJobErrorHandlingBuilder ThenAbortJob() { ThenAction = ErrorHandlingAction.AbortJob; return this; } + public IJobErrorHandlingBuilder ThenStopJob() + { + return ThenAbortJob(); + } + public IJobErrorHandlingBuilder ThenStopAllJobs() { ThenAction = ErrorHandlingAction.StopAllJobs; diff --git a/src/Sa.Schedule/Settings/JobProperies.cs b/src/Sa.Schedule/Settings/JobProperties.cs similarity index 72% rename from src/Sa.Schedule/Settings/JobProperies.cs rename to src/Sa.Schedule/Settings/JobProperties.cs index 6c483650..9a9acc99 100644 --- a/src/Sa.Schedule/Settings/JobProperies.cs +++ b/src/Sa.Schedule/Settings/JobProperties.cs @@ -1,8 +1,8 @@ -using Sa.Schedule.Engine; +using Sa.Schedule.Engine; namespace Sa.Schedule.Settings; -internal sealed class JobProperies : IJobProperties +internal sealed class JobProperties : IJobProperties { public string? JobName { get; private set; } public bool? Immediate { get; private set; } @@ -15,68 +15,68 @@ internal sealed class JobProperies : IJobProperties public int? ConcurrencyLimit { get; private set; } public int? MaxConcurrency { get; private set; } - public JobProperies WithName(string name) + public JobProperties WithName(string name) { JobName = name; return this; } - public JobProperies RunOnce() + public JobProperties RunOnce() { IsRunOnce = true; return this; } - public JobProperies StartImmediate() + public JobProperties StartImmediate() { Immediate = true; return this; } - public JobProperies WithInitialDelay(TimeSpan time) + public JobProperties WithInitialDelay(TimeSpan time) { InitialDelay = time; return this; } - public JobProperies WithTiming(IJobTiming timing) + public JobProperties WithTiming(IJobTiming timing) { Timing = timing; return this; } - public JobProperies SetDisabled() + public JobProperties SetDisabled() { Disabled = true; return this; } - public JobProperies WithContextStackSize(int size) + public JobProperties WithContextStackSize(int size) { ContextStackSize = size; return this; } - public JobProperies WithTag(object tag) + public JobProperties WithTag(object tag) { Tag = tag; return this; } - public JobProperies EveryTime(TimeSpan timeSpan, string? name = null) + public JobProperties EveryTime(TimeSpan timeSpan, string? name = null) { Timing = JobTiming.EveryTime(timeSpan, name); return this; } - public JobProperies WithConcurrencyLimit(int limit) + public JobProperties WithConcurrencyLimit(int limit) { ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); ConcurrencyLimit = limit; return this; } - public JobProperies WithMaxConcurrencyLimit(int limit) + public JobProperties WithMaxConcurrencyLimit(int limit) { ArgumentOutOfRangeException.ThrowIfLessThan(limit, 1); MaxConcurrency = limit; @@ -84,7 +84,7 @@ public JobProperies WithMaxConcurrencyLimit(int limit) } - internal JobProperies Merge(IJobProperties props) + internal JobProperties Merge(IJobProperties props) { JobName ??= props.JobName; Immediate ??= props.Immediate; diff --git a/src/Sa.Schedule/Settings/JobSettings.cs b/src/Sa.Schedule/Settings/JobSettings.cs index 15f52876..f9a21228 100644 --- a/src/Sa.Schedule/Settings/JobSettings.cs +++ b/src/Sa.Schedule/Settings/JobSettings.cs @@ -9,7 +9,7 @@ internal sealed class JobSettings(Type jobType, Guid jobId) : IJobSettings public Type JobType => jobType; - public JobProperies Properties { get; } = new(); + public JobProperties Properties { get; } = new(); public JobErrorHandling ErrorHandling { get; } = new(); diff --git a/src/Tests/Sa.ScheduleTests/CronTimingTests.cs b/src/Tests/Sa.ScheduleTests/CronTimingTests.cs new file mode 100644 index 00000000..6aa3210a --- /dev/null +++ b/src/Tests/Sa.ScheduleTests/CronTimingTests.cs @@ -0,0 +1,313 @@ +using Sa.Schedule.Engine; +using System.Globalization; + +namespace Sa.ScheduleTests; + +public class CronTimingTests +{ + [Theory] + [InlineData("* * * * *", "2026-06-25T10:30:00Z", "2026-06-25T10:31:00Z")] + [InlineData("0 9 * * *", "2026-06-25T08:00:00Z", "2026-06-25T09:00:00Z")] + [InlineData("0 9 * * *", "2026-06-25T09:00:00Z", "2026-06-26T09:00:00Z")] + [InlineData("0 9 * * *", "2026-06-25T09:01:00Z", "2026-06-26T09:00:00Z")] + [InlineData("30 14 * * 1-5", "2026-06-25T14:30:00Z", "2026-06-26T14:30:00Z")] // Friday -> Monday + [InlineData("0 0 1 * *", "2026-06-25T00:00:00Z", "2026-07-01T00:00:00Z")] + [InlineData("0 */2 * * *", "2026-06-25T10:00:00Z", "2026-06-25T12:00:00Z")] + [InlineData("15 10 * * *", "2026-06-25T10:14:00Z", "2026-06-25T10:15:00Z")] + [InlineData("0 0 1 1 *", "2026-06-25T00:00:00Z", "2027-01-01T00:00:00Z")] + public void GetNextOccurrence_ReturnsCorrectNextTime(string cronExpression, string startTime, string expectedNext) + { + var timing = new CronTiming(cronExpression); + var start = DateTimeOffset.Parse(startTime, CultureInfo.InvariantCulture); + var expected = DateTimeOffset.Parse(expectedNext, CultureInfo.InvariantCulture); + + var result = timing.GetNextOccurrence(start, null!); + + Assert.NotNull(result); + Assert.Equal(expected, result); + } + + [Fact] + public void GetNextOccurrence_EveryMinute_NextMinute() + { + var timing = new CronTiming("* * * * *"); + var start = new DateTimeOffset(2026, 6, 25, 10, 30, 45, TimeSpan.Zero); + + var result = timing.GetNextOccurrence(start, null!); + + Assert.Equal(new DateTimeOffset(2026, 6, 25, 10, 31, 0, TimeSpan.Zero), result); + } + + [Fact] + public void GetNextOccurrence_HourlyAtMinute0_NextHour() + { + var timing = new CronTiming("0 * * * *"); + var start = new DateTimeOffset(2026, 6, 25, 10, 15, 0, TimeSpan.Zero); + + var result = timing.GetNextOccurrence(start, null!); + + Assert.Equal(new DateTimeOffset(2026, 6, 25, 11, 0, 0, TimeSpan.Zero), result); + } + + [Fact] + public void GetNextOccurrence_Every2Hours_CorrectIntervals() + { + var timing = new CronTiming("0 */2 * * *"); + + // Test multiple intervals + DateTimeOffset t1 = new(2026, 6, 25, 0, 0, 0, TimeSpan.Zero); + DateTimeOffset t2 = new(2026, 6, 25, 2, 0, 0, TimeSpan.Zero); + DateTimeOffset t3 = new(2026, 6, 25, 4, 0, 0, TimeSpan.Zero); + DateTimeOffset t4 = new(2026, 6, 25, 22, 0, 0, TimeSpan.Zero); + + Assert.Equal(t2, timing.GetNextOccurrence(t1, null!)); + Assert.Equal(t3, timing.GetNextOccurrence(t2, null!)); + // From 04:00 → next slot is 06:00 same day + Assert.Equal(new DateTimeOffset(2026, 6, 25, 6, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(t3, null!)); + // From 22:00 → next slot is 00:00 next day + Assert.Equal(new DateTimeOffset(2026, 6, 26, 0, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(t4, null!)); + } + + [Fact] + public void GetNextOccurrence_WeekdayOnly_SkipsWeekend() + { + var timing = new CronTiming("0 9 * * 1-5"); // Mon-Fri at 9 AM + + // Friday June 26, 2026 + var friday = new DateTimeOffset(2026, 6, 26, 10, 0, 0, TimeSpan.Zero); + var nextMonday = new DateTimeOffset(2026, 6, 29, 9, 0, 0, TimeSpan.Zero); + + Assert.Equal(nextMonday, timing.GetNextOccurrence(friday, null!)); + } + + [Fact] + public void GetNextOccurrence_FirstDayOfMonth_CorrectMonthTransition() + { + var timing = new CronTiming("0 0 1 * *"); + + var june25 = new DateTimeOffset(2026, 6, 25, 12, 0, 0, TimeSpan.Zero); + var july1 = new DateTimeOffset(2026, 7, 1, 0, 0, 0, TimeSpan.Zero); + + Assert.Equal(july1, timing.GetNextOccurrence(june25, null!)); + } + + [Fact] + public void GetNextOccurrence_CommaSeparatedMinutes_MultipleValues() + { + var timing = new CronTiming("10,30,50 * * * *"); + var start = new DateTimeOffset(2026, 6, 25, 10, 15, 0, TimeSpan.Zero); + + var result = timing.GetNextOccurrence(start, null!); + + Assert.Equal(new DateTimeOffset(2026, 6, 25, 10, 30, 0, TimeSpan.Zero), result); + } + + [Fact] + public void GetNextOccurrence_RangeOfDays_MatchesCorrectly() + { + var timing = new CronTiming("0 12 1-15 * *"); // Noon on 1st-15th + + var midMonth = new DateTimeOffset(2026, 6, 20, 12, 0, 0, TimeSpan.Zero); + // After June 20, next valid day is July 1 (1-15 range includes the 1st) + var nextMonth = new DateTimeOffset(2026, 7, 1, 12, 0, 0, TimeSpan.Zero); + + Assert.Equal(nextMonth, timing.GetNextOccurrence(midMonth, null!)); + } + + [Fact] + public void GetNextOccurrence_StepInRange_CorrectIntervals() + { + var timing = new CronTiming("0 9-17/2 * * *"); // Every 2 hours from 9-17 → slots: 9,11,13,15,17 + + var morning = new DateTimeOffset(2026, 6, 25, 9, 0, 0, TimeSpan.Zero); + var noon = new DateTimeOffset(2026, 6, 25, 11, 0, 0, TimeSpan.Zero); + var afternoon = new DateTimeOffset(2026, 6, 25, 17, 0, 0, TimeSpan.Zero); + + // From 9:00 → next slot is 11:00 + Assert.Equal(new DateTimeOffset(2026, 6, 25, 11, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(morning, null!)); + // From 11:00 → next slot is 13:00 (not 17!) + Assert.Equal(new DateTimeOffset(2026, 6, 25, 13, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(noon, null!)); + // From 17:00 → next slot is 9:00 next day + Assert.Equal(new DateTimeOffset(2026, 6, 26, 9, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(afternoon, null!)); + } + + [Fact] + public void GetNextOccurrence_LargeInterval_CrossesYearBoundary() + { + var timing = new CronTiming("0 0 29 2 *"); // Leap day + + var normalYear = new DateTimeOffset(2026, 6, 25, 0, 0, 0, TimeSpan.Zero); + var leapDay2028 = new DateTimeOffset(2028, 2, 29, 0, 0, 0, TimeSpan.Zero); + + Assert.Equal(leapDay2028, timing.GetNextOccurrence(normalYear, null!)); + } + + [Fact] + public void TimingName_UsesCustomName_WhenProvided() + { + var timing = new CronTiming("0 9 * * *", "Daily at 9 AM"); + Assert.Equal("Daily at 9 AM", timing.TimingName); + } + + [Fact] + public void TimingName_UsesDefaultName_WhenNotProvided() + { + var timing = new CronTiming("0 9 * * *"); + Assert.Equal("cron", timing.TimingName); + } + + [Theory] + [InlineData("invalid")] + [InlineData("* *")] + [InlineData("* * * * * *")] + [InlineData("60 * * * *")] + [InlineData("* 24 * * *")] + [InlineData("* * 32 * *")] + [InlineData("* * * 13 *")] + [InlineData("* * * * 7")] + [InlineData("abc * * * *")] + public void Constructor_ThrowsFormatException_ForInvalidExpression(string invalidExpression) + { + CronTiming act() => new(invalidExpression); + Assert.Throws((Func)act); + } + + [Fact] + public void Constructor_NullExpression_ThrowsArgumentException() + { + Assert.ThrowsAny(() => new CronTiming(null!)); + } + + [Fact] + public void StaticEvery_Method_CreatesCronTimingWithName() + { + var timing = CronTiming.Every("0 9 * * *", "Morning job"); + + Assert.Equal("Morning job", timing.TimingName); + } + + [Fact] + public void GetNextOccurrence_Timezone_Aware() + { + // Test with timezone offset + var tzOffset = TimeSpan.FromHours(2); + var timing = new CronTiming("0 10 * * *"); + + var dt = new DateTimeOffset(2026, 6, 25, 9, 30, 0, tzOffset); + var expected = new DateTimeOffset(2026, 6, 25, 10, 0, 0, tzOffset); + + Assert.Equal(expected, timing.GetNextOccurrence(dt, null!)); + } + + [Fact] + public void GetNextOccurrence_BothDayConstraints_BothMustMatch() + { + // Day of month AND day of week both specified — both must match + var timing = new CronTiming("0 12 15 * 3"); // Wednesday (3) on 15th + + // Find a date that's both the 15th and a Wednesday + // June 15, 2026 is a Tuesday, so it should skip + var start = new DateTimeOffset(2026, 6, 15, 13, 0, 0, TimeSpan.Zero); + + // July 15, 2026 is a Wednesday + var expected = new DateTimeOffset(2026, 7, 15, 12, 0, 0, TimeSpan.Zero); + + Assert.Equal(expected, timing.GetNextOccurrence(start, null!)); + } + + [Fact] + public void GetNextOccurrence_MinuteStep_CorrectIntervals() + { + var timing = new CronTiming("*/15 * * * *"); // Every 15 minutes: 0, 15, 30, 45 + + var start = new DateTimeOffset(2026, 6, 25, 10, 0, 0, TimeSpan.Zero); + Assert.Equal(new DateTimeOffset(2026, 6, 25, 10, 15, 0, TimeSpan.Zero), timing.GetNextOccurrence(start, null!)); + Assert.Equal(new DateTimeOffset(2026, 6, 25, 10, 30, 0, TimeSpan.Zero), timing.GetNextOccurrence(new DateTimeOffset(2026, 6, 25, 10, 15, 0, TimeSpan.Zero), null!)); + Assert.Equal(new DateTimeOffset(2026, 6, 25, 10, 45, 0, TimeSpan.Zero), timing.GetNextOccurrence(new DateTimeOffset(2026, 6, 25, 10, 30, 0, TimeSpan.Zero), null!)); + Assert.Equal(new DateTimeOffset(2026, 6, 25, 11, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(new DateTimeOffset(2026, 6, 25, 10, 45, 0, TimeSpan.Zero), null!)); + } + + [Fact] + public void GetNextOccurrence_HourRangeWithStep_CorrectSlots() + { + // Every 3 hours from 8-18 → slots: 8, 11, 14, 17 + var timing = new CronTiming("0 8-17/3 * * *"); + + var start = new DateTimeOffset(2026, 6, 25, 8, 0, 0, TimeSpan.Zero); + Assert.Equal(new DateTimeOffset(2026, 6, 25, 11, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(start, null!)); + Assert.Equal(new DateTimeOffset(2026, 6, 25, 14, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(new DateTimeOffset(2026, 6, 25, 11, 0, 0, TimeSpan.Zero), null!)); + Assert.Equal(new DateTimeOffset(2026, 6, 25, 17, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(new DateTimeOffset(2026, 6, 25, 14, 0, 0, TimeSpan.Zero), null!)); + Assert.Equal(new DateTimeOffset(2026, 6, 26, 8, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(new DateTimeOffset(2026, 6, 25, 17, 0, 0, TimeSpan.Zero), null!)); + } + + [Fact] + public void GetNextOccurrence_WildcardMonth_AnyMonth() + { + var timing = new CronTiming("0 0 29 2 *"); // Feb 29 only + + var jan2028 = new DateTimeOffset(2028, 1, 1, 0, 0, 0, TimeSpan.Zero); + Assert.Equal(new DateTimeOffset(2028, 2, 29, 0, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(jan2028, null!)); + + // From within 2028 (after Feb 29), next leap day is 2032 — but that's 4 years away + // The 2-year search limit means this returns null + var mar2028 = new DateTimeOffset(2028, 3, 1, 0, 0, 0, TimeSpan.Zero); + var result = timing.GetNextOccurrence(mar2028, null!); + // Beyond 2-year horizon → null + Assert.Null(result); + } + + [Fact] + public void GetNextOccurrence_DayOfWeekWildcard_AnyDay() + { + var timing = new CronTiming("0 9 * * *"); // Any day at 9 AM + + var friday = new DateTimeOffset(2026, 6, 26, 10, 0, 0, TimeSpan.Zero); + Assert.Equal(new DateTimeOffset(2026, 6, 27, 9, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(friday, null!)); + } + + [Fact] + public void GetNextOccurrence_PreciseMinute_NonZeroSeconds() + { + // When seconds are non-zero, we should still land on the next minute boundary + var timing = new CronTiming("30 * * * *"); + + var start = new DateTimeOffset(2026, 6, 25, 10, 30, 30, TimeSpan.Zero); + // Already past 10:30, so next is 11:30 + Assert.Equal(new DateTimeOffset(2026, 6, 25, 11, 30, 0, TimeSpan.Zero), timing.GetNextOccurrence(start, null!)); + } + + [Fact] + public void GetNextOccurrence_LastDayOfMonth_January() + { + var timing = new CronTiming("0 0 31 1 *"); // Jan 31 only + + var jan2026 = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + Assert.Equal(new DateTimeOffset(2026, 1, 31, 0, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(jan2026, null!)); + + var feb2026 = new DateTimeOffset(2026, 2, 1, 0, 0, 0, TimeSpan.Zero); + // Next Jan 31 is 2027 + Assert.Equal(new DateTimeOffset(2027, 1, 31, 0, 0, 0, TimeSpan.Zero), timing.GetNextOccurrence(feb2026, null!)); + } + + [Fact] + public void GetNextOccurrence_NoValidSlot_ReturnsNull() + { + // February 30 doesn't exist — this expression can never match + var timing = new CronTiming("0 0 30 2 *"); + + var start = new DateTimeOffset(2026, 6, 25, 0, 0, 0, TimeSpan.Zero); + var result = timing.GetNextOccurrence(start, null!); + + Assert.Null(result); + } + + [Theory] + [InlineData("* * * * *", "cron")] + [InlineData("0 0 * * *", "midnight")] + [InlineData("*/5 * * * *", "five-min")] + public void Constructor_StoresCustomNames(string expression, string name) + { + var timing = new CronTiming(expression, name); + Assert.Equal(name, timing.TimingName); + } +} diff --git a/src/Tests/Sa.ScheduleTests/JobControllerTests.cs b/src/Tests/Sa.ScheduleTests/JobControllerTests.cs new file mode 100644 index 00000000..7ae68e68 --- /dev/null +++ b/src/Tests/Sa.ScheduleTests/JobControllerTests.cs @@ -0,0 +1,148 @@ +using Microsoft.Extensions.DependencyInjection; +using Sa.Schedule; +using Sa.Schedule.Engine; +using Sa.Schedule.Settings; + +namespace Sa.ScheduleTests; + +public class JobControllerTests +{ + [Fact] + public void PauseAndResume_WorkCorrectly() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var controller = CreateController(settings); + + Assert.False(controller.IsPaused); + + controller.Pause(); + Assert.True(controller.IsPaused); + + controller.Resume(); + Assert.False(controller.IsPaused); + } + + [Fact] + public void Shutdown_IsIdempotent() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var controller = CreateController(settings); + + // Shutdown without Start should be safe + controller.Shutdown(); + + // Subsequent calls should be no-op + controller.Shutdown(); + controller.Pause(); + controller.Resume(); + Assert.True(true); + } + + [Fact] + public void Index_ReturnsConstructorValue() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var controller = CreateController(settings, index: 42); + + Assert.Equal(42, controller.Index); + } + + [Fact] + public void Pause_WhileAlreadyPaused_IsNoop() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var controller = CreateController(settings); + + controller.Pause(); + controller.Pause(); // Double pause — should not throw or deadlock + + Assert.True(controller.IsPaused); + } + + [Fact] + public void Resume_WhileNotPaused_IsNoop() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var controller = CreateController(settings); + + controller.Resume(); // Double resume — should not throw + + Assert.False(controller.IsPaused); + } + + [Fact] + public void Dispose_CallsShutdown() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var controller = CreateController(settings); + + controller.Dispose(); // Should not throw + + // After dispose, operations should be safe no-ops + controller.Pause(); + controller.Resume(); + controller.Shutdown(); + Assert.True(true); + } + + [Fact] + public void Index_IsImmutable() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var controllerA = CreateController(settings, index: 0); + var controllerB = CreateController(settings, index: 99); + + Assert.Equal(0, controllerA.Index); + Assert.Equal(99, controllerB.Index); + } + + [Fact] + public void WithMaxConcurrency_RejectsZero() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var ex = Record.Exception(() => settings.Properties.WithMaxConcurrencyLimit(0)); + Assert.IsType(ex); + } + + [Fact] + public void WithConcurrencyLimit_RejectsNegative() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var ex = Record.Exception(() => settings.Properties.WithConcurrencyLimit(-1)); + Assert.IsType(ex); + } + + private static JobController CreateController(IJobSettings settings, int index = 0, TimeProvider? timeProvider = null) + { + var scopeFactory = new MockScopeFactory(); + return new JobController( + index, + settings, + new InterceptorSettings([]), + scopeFactory, + timeProvider ?? TimeProvider.System); + } + + sealed class TestJob : IJob + { + public Task Execute(IJobContext context, CancellationToken cancellationToken) + => Task.CompletedTask; + } + + sealed class MockScopeFactory : IServiceScopeFactory + { + private readonly IServiceScope _scope = new MockScope(); + public IServiceScope CreateScope() => _scope; + } + + sealed class MockScope : IServiceScope + { + public IServiceProvider ServiceProvider => new MockServiceProvider(); + public void Dispose() { } + } + + sealed class MockServiceProvider : IServiceProvider + { + public object? GetService(Type serviceType) => null; + } +} diff --git a/src/Tests/Sa.ScheduleTests/JobErrorHandlerIntegrationTests.cs b/src/Tests/Sa.ScheduleTests/JobErrorHandlerIntegrationTests.cs new file mode 100644 index 00000000..e5156d4b --- /dev/null +++ b/src/Tests/Sa.ScheduleTests/JobErrorHandlerIntegrationTests.cs @@ -0,0 +1,79 @@ +using Microsoft.Extensions.DependencyInjection; +using Sa.Schedule; + +namespace Sa.ScheduleTests; + +public sealed class JobErrorHandlerIntegrationTests : IAsyncDisposable +{ + private readonly ScheduleSetupTests.Fixture _fixture; + + public JobErrorHandlerIntegrationTests() + { + _fixture = new ScheduleSetupTests.Fixture(); + } + + [Fact] + public async Task GlobalErrorHandler_ConsumesError_DoesNotCrash() + { + // This tests that the global error handler registered via AddErrorHandler + // can consume errors before per-job handling kicks in + var scheduler = _fixture.Sub; + + int started = await scheduler.Start(TestContext.Current.CancellationToken); + Assert.True(started > 0); + + // Let it run and process normally + await Task.Delay(350, TestContext.Current.CancellationToken); + + await scheduler.Stop(); + } + + [Fact] + public void ErrorHandlingBuilder_ChainMethods_WorkCorrectly() + { + // Verify the fluent builder chain compiles and methods are accessible + var services = new ServiceCollection(); + services.AddSaSchedule(b => + { + b.AddJob((sp, job) => + { + job + .EverySeconds(1) + .ConfigureErrorHandling(err => err + .IfErrorRetry(3) + .DoSuppressError(ex => ex is TimeoutException) + .ThenAbortJob()); + }); + }); + + var provider = services.BuildServiceProvider(); + var scheduleSettings = provider.GetRequiredService(); + var jobSettings = scheduleSettings.GetJobSettings().First(); + + Assert.Equal(ErrorHandlingAction.AbortJob, jobSettings.ErrorHandling.ThenAction); + Assert.Equal(3, jobSettings.ErrorHandling.RetryCount); + Assert.NotNull(jobSettings.ErrorHandling.SuppressError); + } + + [Fact] + public async Task Scheduler_StartStopsCleanly() + { + var scheduler = _fixture.Sub; + + int started = await scheduler.Start(TestContext.Current.CancellationToken); + Assert.True(started > 0); + + await Task.Delay(250, TestContext.Current.CancellationToken); + + await scheduler.Stop(); + } + + private sealed class TestJob : IJob + { + public Task Execute(IJobContext context, CancellationToken cancellationToken) + => Task.Delay(50, cancellationToken); + } + + public ValueTask DisposeAsync() + => _fixture.DisposeAsync(); +} diff --git a/src/Tests/Sa.ScheduleTests/JobExceptionTests.cs b/src/Tests/Sa.ScheduleTests/JobExceptionTests.cs new file mode 100644 index 00000000..007933a4 --- /dev/null +++ b/src/Tests/Sa.ScheduleTests/JobExceptionTests.cs @@ -0,0 +1,167 @@ +using Sa.Schedule; +using Sa.Schedule.Engine; +using Sa.Schedule.Settings; + +namespace Sa.ScheduleTests; + +public class JobExceptionTests +{ + [Fact] + public void ContextSnapshot_JobName_MatchesContext() + { + var settings = JobSettings.Create(Guid.NewGuid()); + settings.Properties.WithName("TestJobName"); + var context = CreateContext(settings); + + var innerEx = new InvalidOperationException("inner error"); + var jobException = new JobException(context, innerEx); + + Assert.Equal("TestJobName", jobException.ContextSnapshot.JobName); + } + + [Fact] + public void ContextSnapshot_NumIterations_CapturedCorrectly() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var context = CreateContext(settings); + context.NumIterations = 5; + + var innerEx = new Exception("error"); + var jobException = new JobException(context, innerEx); + + Assert.Equal(5UL, jobException.ContextSnapshot.NumIterations); + } + + [Fact] + public void ContextSnapshot_FailedIterations_CapturedCorrectly() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var context = CreateContext(settings); + context.FailedIterations = 3; + + var innerEx = new Exception("error"); + var jobException = new JobException(context, innerEx); + + Assert.Equal(3UL, jobException.ContextSnapshot.FailedIterations); + } + + [Fact] + public void ContextSnapshot_CompletedIterations_CapturedCorrectly() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var context = CreateContext(settings); + context.CompetedIterations = 10; + + var innerEx = new Exception("error"); + var jobException = new JobException(context, innerEx); + + Assert.Equal(10UL, jobException.ContextSnapshot.CompetedIterations); + } + + [Fact] + public void ContextSnapshot_ExecuteAt_CapturedCorrectly() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var context = CreateContext(settings); + var now = DateTimeOffset.UtcNow; + context.ExecuteAt = now; + + var innerEx = new Exception("error"); + var jobException = new JobException(context, innerEx); + + Assert.Equal(now, jobException.ContextSnapshot.ExecuteAt); + } + + [Fact] + public void ContextSnapshot_FailedRetries_CapturedCorrectly() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var context = CreateContext(settings); + context.FailedRetries = 2; + + var innerEx = new Exception("error"); + var jobException = new JobException(context, innerEx); + + Assert.Equal(2, jobException.ContextSnapshot.FailedRetries); + } + + [Fact] + public void ContextSnapshot_LastErrorMessage_CapturedFromLastError() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var context = CreateContext(settings); + // LastError is a JobException, its .Message format is "[JobName] job error" + context.LastError = new JobException(context, new Exception("inner")); + + var outerEx = new Exception("new error"); + var jobException = new JobException(context, outerEx); + + // LastErrorMessage reads LastError.Message which is the JobException's formatted message + Assert.NotNull(jobException.ContextSnapshot.LastErrorMessage); + Assert.Contains("job error", jobException.ContextSnapshot.LastErrorMessage!); + } + + [Fact] + public void ContextSnapshot_StackDepth_CountsEntries() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var context = CreateContext(settings); + var stack = context; + stack.Stack.Enqueue(stack.Clone()); + stack.Stack.Enqueue(stack.Clone()); + stack.Stack.Enqueue(stack.Clone()); + + var innerEx = new Exception("error"); + var jobException = new JobException(context, innerEx); + + Assert.Equal(3, jobException.ContextSnapshot.StackDepth); + } + + [Fact] + public void InnerException_ProvidedInnerException() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var context = CreateContext(settings); + var innerEx = new ArgumentException("arg error"); + + var jobException = new JobException(context, innerEx); + + Assert.Same(innerEx, jobException.InnerException); + } + + [Fact] + public void Message_IncludesJobNameAndError() + { + var settings = JobSettings.Create(Guid.NewGuid()); + settings.Properties.WithName("MyJob"); + var context = CreateContext(settings); + + var innerEx = new Exception("boom"); + var jobException = new JobException(context, innerEx); + + Assert.Contains("MyJob", jobException.Message); + Assert.Contains("job error", jobException.Message); + } + + [Fact] + public void ContextSnapshot_CreatedAt_CapturedCorrectly() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var context = CreateContext(settings); + var expectedCreatedAt = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + context.CreatedAt = expectedCreatedAt; + + var innerEx = new Exception("error"); + var jobException = new JobException(context, innerEx); + + Assert.Equal(expectedCreatedAt, jobException.ContextSnapshot.CreatedAt); + } + + private static JobContext CreateContext(IJobSettings settings) => new(settings); + + sealed class TestJob : IJob + { + public Task Execute(IJobContext context, CancellationToken cancellationToken) + => Task.CompletedTask; + } +} diff --git a/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs b/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs index c512cc35..e13a416a 100644 --- a/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs +++ b/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs @@ -95,6 +95,60 @@ public async Task ConcurrencyLimit_ConcurrentCalls_Succeeds() await scheduler.DisposeAsync(); } + [Fact] + public async Task IsStarted_True_AfterSuccessfulStart() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + + Assert.False(scheduler.IsStarted); + + var started = await scheduler.Start(TestContext.Current.CancellationToken); + Assert.True(started); + Assert.True(scheduler.IsStarted); + } + + [Fact] + public void ActiveTasks_ReturnsZero_BeforeStart() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + + Assert.Equal(0, scheduler.ActiveTasks); + } + + [Fact] + public void Dispose_AfterStop_IsSafe() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + + scheduler.Dispose(); + scheduler.Dispose(); // Double dispose should not throw + Assert.True(true); + } + + [Fact] + public async Task DisposeAsync_AfterStop_IsSafe() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + + await scheduler.DisposeAsync(); + await scheduler.DisposeAsync(); // Double dispose should not throw + Assert.True(true); + } + + [Fact] + public void ChangeToken_ReflectsStoppedState() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + + var token = scheduler.StartChangeToken(); + Assert.NotNull(token); + } + class TestJob : IJob diff --git a/src/Tests/Sa.ScheduleTests/Sa.ScheduleTests.csproj b/src/Tests/Sa.ScheduleTests/Sa.ScheduleTests.csproj index 7a9e09e3..2e6be8ca 100644 --- a/src/Tests/Sa.ScheduleTests/Sa.ScheduleTests.csproj +++ b/src/Tests/Sa.ScheduleTests/Sa.ScheduleTests.csproj @@ -11,4 +11,8 @@ + + + + diff --git a/src/Tests/Sa.ScheduleTests/ScheduleBuilderTests.cs b/src/Tests/Sa.ScheduleTests/ScheduleBuilderTests.cs new file mode 100644 index 00000000..09278bda --- /dev/null +++ b/src/Tests/Sa.ScheduleTests/ScheduleBuilderTests.cs @@ -0,0 +1,301 @@ +using Microsoft.Extensions.DependencyInjection; +using Sa.Schedule; +using Sa.Schedule.Settings; + +namespace Sa.ScheduleTests; + +public class ScheduleBuilderTests +{ + [Fact] + public void AddJob_Type_RegistersJobSettings() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddJob(); + + var settings = services.FirstOrDefault(d => d.ServiceType == typeof(JobSettings)); + Assert.NotNull(settings); + } + + [Fact] + public void AddJob_Func_RegistersFuncJob() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddJob((ctx, ct) => Task.CompletedTask); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetServices().ToList(); + Assert.NotEmpty(jobSettings); + } + + [Fact] + public void AddInterceptor_RegistersInterceptorSettings() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddInterceptor(); + + var provider = services.BuildServiceProvider(); + var interceptorSettings = provider.GetService(); + Assert.NotNull(interceptorSettings); + Assert.Single(interceptorSettings!.Interceptors); + } + + [Fact] + public void UseHostedService_MarksAsHostedService() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.UseHostedService().AddJob(); + + var provider = services.BuildServiceProvider(); + var scheduleSettings = provider.GetService(); + Assert.True(scheduleSettings!.IsHostedService); + } + + [Fact] + public void JobBuilder_WithName_SetsJobName() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddJob().WithName("MyCustomJob"); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + Assert.Equal("MyCustomJob", jobSettings.Properties.JobName); + } + + [Fact] + public void JobBuilder_Disabled_SetsDisabledFlag() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddJob().Disabled(); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + Assert.True(jobSettings.Properties.Disabled); + } + + [Fact] + public void JobBuilder_Cron_SetsCronTiming() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddJob().WithCron("0 9 * * *", "MorningJob"); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + Assert.NotNull(jobSettings.Properties.Timing); + Assert.Equal("MorningJob", jobSettings.Properties.Timing!.TimingName); + } + + [Fact] + public void JobBuilder_EverySeconds_SetsTiming() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddJob().EverySeconds(30); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + Assert.NotNull(jobSettings.Properties.Timing); + } + + [Fact] + public void JobBuilder_WithInitialDelay_SetsDelay() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + var delay = TimeSpan.FromSeconds(5); + builder.AddJob().WithInitialDelay(delay); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + Assert.Equal(delay, jobSettings.Properties.InitialDelay); + } + + [Fact] + public void JobBuilder_WithTag_SetsTag() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + var tag = new { Priority = 42 }; + + builder.AddJob().WithTag(tag); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + Assert.Same(tag, jobSettings.Properties.Tag); + } + + [Fact] + public void JobBuilder_WithContextStackSize_SetsStackSize() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddJob().WithContextStackSize(10); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + Assert.Equal(10, jobSettings.Properties.ContextStackSize); + } + + [Fact] + public void JobBuilder_ConfigureErrorHandling_SetsRetryAndAction() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddJob((sp, job) => + { + job.ConfigureErrorHandling(err => err + .IfErrorRetry(5) + .ThenAbortJob()); + }); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + Assert.Equal(5, jobSettings.ErrorHandling.RetryCount); + Assert.Equal(ErrorHandlingAction.AbortJob, jobSettings.ErrorHandling.ThenAction); + } + + [Fact] + public void JobBuilder_ConfigureErrorHandling_SetsSuppressError() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddJob((sp, job) => + { + job.ConfigureErrorHandling(err => err + .DoSuppressError(ex => ex is OperationCanceledException) + .ThenAbortJob()); + }); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + Assert.NotNull(jobSettings.ErrorHandling.SuppressError); + Assert.True(jobSettings.ErrorHandling.SuppressError!(new OperationCanceledException())); + Assert.False(jobSettings.ErrorHandling.SuppressError!(new InvalidOperationException())); + } + + [Fact] + public void JobBuilder_ConfigureErrorHandling_DefaultCloseApplication() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddJob(); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + // Default error handling action is CloseApplication + Assert.Equal(ErrorHandlingAction.CloseApplication, jobSettings.ErrorHandling.ThenAction); + // Default retry count is 0 (not set unless IfErrorRetry is called) + Assert.Equal(0, jobSettings.ErrorHandling.RetryCount); + } + + [Fact] + public void JobBuilder_OnceIn_SetsInitialDelayAndRunOnce() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + var delay = TimeSpan.FromSeconds(10); + + builder.AddJob().OnceIn(delay); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + Assert.Equal(delay, jobSettings.Properties.InitialDelay); + Assert.True(jobSettings.Properties.IsRunOnce); + } + + [Fact] + public void JobBuilder_WithConcurrencyLimit_ValidatesRange() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + IJobBuilder act() => builder.AddJob().WithConcurrencyLimit(-1); + + Assert.Throws((Func)act); + } + + [Fact] + public void AddJob_WithCustomId_UsesProvidedId() + { + var services = new ServiceCollection(); + var customId = Guid.NewGuid(); + var builder = new ScheduleBuilder(services); + + builder.AddJob(customId); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + Assert.Equal(customId, jobSettings.JobId); + } + + [Fact] + public void AddJob_WithoutId_GeneratesNewId() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddJob(); + + var provider = services.BuildServiceProvider(); + var jobSettings = provider.GetRequiredService(); + + Assert.NotEqual(Guid.Empty, jobSettings.JobId); + } + + [Fact] + public void ScheduleBuilder_AddErrorHandler_RegistersGlobalHandler() + { + var services = new ServiceCollection(); + var builder = new ScheduleBuilder(services); + + builder.AddErrorHandler((ctx, ex) => true).AddJob(); + + var provider = services.BuildServiceProvider(); + var scheduleSettings = provider.GetRequiredService(); + + // The handler should be registered — verify via Settings + Assert.NotNull(scheduleSettings); + } + + sealed class TestJob : IJob + { + public Task Execute(IJobContext context, CancellationToken cancellationToken) => Task.CompletedTask; + } + + sealed class TestInterceptor : IJobInterceptor + { + public Task OnHandle(IJobContext context, Func next, object? key, CancellationToken cancellationToken) + => next(); + } +} diff --git a/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs b/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs index ed6e5253..d51fcb47 100644 --- a/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs +++ b/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs @@ -65,12 +65,13 @@ public Fixture() [Fact] public async Task Check_Executing_RunOnce_ForMultiJobs() { - int i = await Sub.Start(CancellationToken.None); + int started = await Sub.Start(CancellationToken.None); - Assert.Equal(2, i); + Assert.Equal(2, started); await Task.Delay(300, TestContext.Current.CancellationToken); + // Each RunOnce job should execute exactly once Assert.Equal(2, Fixture.Count); } } diff --git a/src/Tests/Sa.ScheduleTests/ScheduleSettingsTests.cs b/src/Tests/Sa.ScheduleTests/ScheduleSettingsTests.cs new file mode 100644 index 00000000..709e290d --- /dev/null +++ b/src/Tests/Sa.ScheduleTests/ScheduleSettingsTests.cs @@ -0,0 +1,95 @@ +using Microsoft.Extensions.DependencyInjection; +using Sa.Schedule; +using Sa.Schedule.Engine; +using Sa.Schedule.Settings; + +namespace Sa.ScheduleTests; + +public class ScheduleSettingsTests +{ + [Fact] + public void IsHostedService_ReturnsCorrectValue() + { + var settings = ScheduleSettings.Create([], isHostedService: true, null); + Assert.True(settings.IsHostedService); + + var settings2 = ScheduleSettings.Create([], isHostedService: false, null); + Assert.False(settings2.IsHostedService); + } + + [Fact] + public void GetJobSettings_ReturnsRegisteredJobs() + { + var id1 = Guid.NewGuid(); + var id2 = Guid.NewGuid(); + var job1 = JobSettings.Create(id1); + var job2 = JobSettings.Create(id2); + var settings = ScheduleSettings.Create([job1, job2], false, null); + + var jobs = settings.GetJobSettings().ToList(); + Assert.Equal(2, jobs.Count); + Assert.Contains(jobs, j => j.JobId == id1); + Assert.Contains(jobs, j => j.JobId == id2); + } + + [Fact] + public void GetJobSettings_EmptyList_ReturnsEmpty() + { + var settings = ScheduleSettings.Create([], false, null); + Assert.Empty(settings.GetJobSettings()); + } + + [Fact] + public void HandleError_GlobalHandler_CanConsumeErrors() + { + var settings = ScheduleSettings.Create( + [], + isHostedService: false, + (ctx, ex) => true); + + // The handler should be set — verify by checking the property exists + Assert.NotNull(settings.HandleError); + } + + [Fact] + public void Merge_JobProperties_PrioritizesNonDefault() + { + var job1 = JobSettings.Create(Guid.NewGuid()); + job1.ErrorHandling.IfErrorRetry(5).ThenAbortJob(); + + var job2 = JobSettings.Create(Guid.NewGuid()); + // job2 keeps defaults + + var merged = JobSettings.Create(job1); + Assert.Equal(5, merged.ErrorHandling.RetryCount); + Assert.Equal(ErrorHandlingAction.AbortJob, merged.ErrorHandling.ThenAction); + } + + [Fact] + public void JobSettings_CreateGeneratesNewId() + { + var settings = JobSettings.Create(Guid.NewGuid()); + Assert.NotEqual(Guid.Empty, settings.JobId); + } + + [Fact] + public void JobSettings_Clone_ReturnsIndependentCopy() + { + var original = JobSettings.Create(Guid.NewGuid()); + original.Properties.WithName("Original"); + original.ErrorHandling.IfErrorRetry(3).ThenStopAllJobs(); + + var clone = original.Clone(); + + Assert.Equal(original.JobId, clone.JobId); + Assert.Equal(original.Properties.JobName, clone.Properties.JobName); + Assert.Equal(original.ErrorHandling.RetryCount, clone.ErrorHandling.RetryCount); + Assert.Same(original.JobType, clone.JobType); + } + + private sealed class TestJob : IJob + { + public Task Execute(IJobContext context, CancellationToken cancellationToken) + => Task.CompletedTask; + } +} diff --git a/src/Tests/Sa.ScheduleTests/ScheduleSetupTests.cs b/src/Tests/Sa.ScheduleTests/ScheduleSetupTests.cs index 544d9be1..008c3afa 100644 --- a/src/Tests/Sa.ScheduleTests/ScheduleSetupTests.cs +++ b/src/Tests/Sa.ScheduleTests/ScheduleSetupTests.cs @@ -46,12 +46,13 @@ public Fixture() [Fact] public async Task Check_ExecuteCounterJob() { - int i = await Sub.Start(CancellationToken.None); + int started = await Sub.Start(CancellationToken.None); - Assert.NotEqual(0, i); + Assert.Equal(1, started); await Task.Delay(300, TestContext.Current.CancellationToken); - Assert.True(Fixture.Count > 0); + // Job should have executed multiple times (100ms interval, 300ms runtime) + Assert.InRange(Fixture.Count, 2, 10); } } From cbb1baecaedf440f49941722d4ea9e4dc87073ad Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 26 Jun 2026 11:39:59 +0300 Subject: [PATCH 08/33] improve media --- src/Sa.Media/PipeReaderExtensions.cs | 16 +++++++++++----- src/Sa.Media/WavHeaderReader.cs | 11 +++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/Sa.Media/PipeReaderExtensions.cs b/src/Sa.Media/PipeReaderExtensions.cs index 631e8f1f..1d769969 100644 --- a/src/Sa.Media/PipeReaderExtensions.cs +++ b/src/Sa.Media/PipeReaderExtensions.cs @@ -4,14 +4,20 @@ namespace Sa.Media; internal static class PipeReaderExtensions { - public static async ValueTask SkipAsync(this PipeReader reader, long count, CancellationToken ct = default) + public static async ValueTask SkipAsync(this PipeReader reader, long count, CancellationToken ct = default) { - while (count > 0) + long remaining = count; + while (remaining > 0) { ReadResult result = await reader.ReadAsync(ct); - SequencePosition consumed = result.Buffer.GetPosition(Math.Min(count, result.Buffer.Length)); - reader.AdvanceTo(consumed); - count -= result.Buffer.Length; + if (result.Buffer.IsEmpty && result.IsCompleted) + break; // или throw new InvalidOperationException("Недостаточно данных") + + var toConsume = Math.Min(remaining, result.Buffer.Length); + var consumed = result.Buffer.GetPosition(toConsume); + reader.AdvanceTo(consumed, consumed); + remaining -= toConsume; } + return count - remaining; // сколько фактически пропущено } } diff --git a/src/Sa.Media/WavHeaderReader.cs b/src/Sa.Media/WavHeaderReader.cs index 4c8037e9..abb69cca 100644 --- a/src/Sa.Media/WavHeaderReader.cs +++ b/src/Sa.Media/WavHeaderReader.cs @@ -78,8 +78,10 @@ public static async Task ReadHeaderAsync( private static async Task<(long, uint dataSize)> FindDataChunkAsync( BinaryPipeReader reader, CancellationToken cancellationToken = default) { - while (reader.Position < 4096) + while (true) { + cancellationToken.ThrowIfCancellationRequested(); + var chunkId = await reader.ReadUInt32Async(cancellationToken); var chunkSize = await reader.ReadUInt32Async(cancellationToken); @@ -88,11 +90,12 @@ public static async Task ReadHeaderAsync( return (reader.Position, chunkSize); // возвращаем смещение и размер данных } - // Пропускаем чанк (с выравниванием) + // Пропускаем чанк (с выравниванием на чётную границу) long paddedSize = (chunkSize % 2 == 0) ? chunkSize : chunkSize + 1; + if (paddedSize == 0) + throw new InvalidDataException("Invalid WAV file: zero-size chunk"); + await reader.SkeepBytesAsync(paddedSize, cancellationToken); } - - throw new InvalidDataException("WAV file does not contain a 'data' chunk."); } } From 92780551ae4963f934b47c31ecea58c865ac7030 Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 26 Jun 2026 12:31:04 +0300 Subject: [PATCH 09/33] improve media --- src/Sa.Media/AsyncWavReader.cs | 123 ++++++++++++++++----------- src/Sa.Media/AsyncWavWriter.cs | 47 ++++++---- src/Sa.Media/BinaryPipeReader.cs | 51 +++++++++-- src/Sa.Media/PipeReaderExtensions.cs | 38 ++++++++- src/Sa.Media/SampleConverter.cs | 13 ++- src/Sa.Media/WavHeaderReader.cs | 4 +- 6 files changed, 189 insertions(+), 87 deletions(-) diff --git a/src/Sa.Media/AsyncWavReader.cs b/src/Sa.Media/AsyncWavReader.cs index b2c4e6fb..6ce11406 100644 --- a/src/Sa.Media/AsyncWavReader.cs +++ b/src/Sa.Media/AsyncWavReader.cs @@ -85,7 +85,9 @@ public Task GetHeaderAsync(CancellationToken cancellationToken) /// но данные должны быть скопированы до следующего yield return). /// Если false - возвращает копию данных (безопасно, но с аллокациями). /// +#pragma warning disable S3776 public async IAsyncEnumerable ReadSamplesPerChannelAsync( +#pragma warning restore S3776 TimeRange? cutRange = null, bool allowBufferReuse = true, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -107,58 +109,67 @@ public async IAsyncEnumerable ReadSamplesPerChannelAsync( int sampleSize = header.SampleSize; long currentOffset = cutFrom; - var sampleBuffer = new byte[sampleSize]; + var samplePool = MemoryPool.Shared.Rent(sampleSize); - while (true) + try { - cancellationToken.ThrowIfCancellationRequested(); - - ReadResult result = await _reader.ReadAsync(cancellationToken); - ReadOnlySequence sequence = result.Buffer; - - SequencePosition consumed = sequence.Start; - bool success = false; - try + while (true) { - // Обрабатываем полные блоки - while (sequence.Length >= blockAlign && currentOffset < cutTo) - { - ReadOnlySequence block = sequence.Slice(0, blockAlign); + cancellationToken.ThrowIfCancellationRequested(); - bool blockIsEof = (currentOffset + blockAlign >= cutTo) || result.IsCompleted; + ReadResult result = await _reader.ReadAsync(cancellationToken); + ReadOnlySequence sequence = result.Buffer; - // Извлекаем сэмплы по каналам - for (int channelId = 0; channelId < channels; channelId++) + SequencePosition consumed = sequence.Start; + bool success = false; + try + { + // Обрабатываем полные блоки + while (sequence.Length >= blockAlign && currentOffset < cutTo) { - int offsetInBlock = channelId * sampleSize; - - block.Slice(offsetInBlock, sampleSize).CopyTo(sampleBuffer); - var chunk = allowBufferReuse ? sampleBuffer : [.. sampleBuffer]; - yield return new(channelId, chunk, currentOffset, blockIsEof); + ReadOnlySequence block = sequence.Slice(0, blockAlign); + + bool blockIsEof = (currentOffset + blockAlign >= cutTo) || result.IsCompleted; + + // Извлекаем сэмплы по каналам + for (int channelId = 0; channelId < channels; channelId++) + { + int offsetInBlock = channelId * sampleSize; + + block.Slice(offsetInBlock, sampleSize).CopyTo(samplePool.Memory.Span); + var chunk = allowBufferReuse + ? samplePool.Memory[..sampleSize] + : samplePool.Memory[..sampleSize].ToArray(); + yield return new(channelId, chunk, currentOffset, blockIsEof); + } + + // Продвигаем позиции + currentOffset += blockAlign; + sequence = sequence.Slice(blockAlign); + consumed = sequence.Start; } - // Продвигаем позиции - currentOffset += blockAlign; - sequence = sequence.Slice(blockAlign); - consumed = sequence.Start; - } - - success = true; - } - finally - { - if (success) - { - _reader.AdvanceTo(consumed, result.IsCompleted ? sequence.End : consumed); + success = true; } - else + finally { - _reader.AdvanceTo(sequence.Start, sequence.End); // Сброс при ошибке + if (success) + { + _reader.AdvanceTo(consumed, result.IsCompleted ? sequence.End : consumed); + } + else + { + _reader.AdvanceTo(sequence.Start, sequence.End); // Сброс при ошибке + } } - } - if (result.IsCompleted || currentOffset >= cutTo) - yield break; + if (result.IsCompleted || currentOffset >= cutTo) + yield break; + } + } + finally + { + samplePool.Dispose(); } } @@ -193,18 +204,24 @@ public async IAsyncEnumerable ConvertToFormatAsync( { int bytesPerSample = targetFormat.GetBytesPerSample(); - var buffer = new byte[bytesPerSample]; - - var convert = SampleConverter.GetConverter(targetFormat); + var buffer = ArrayPool.Shared.Rent(bytesPerSample); + try + { + var convert = SampleConverter.GetConverter(targetFormat); - await foreach (var (channelId, sample, offset, isEof) in - ReadDoubleSamplesAsync(cutRange, allowBufferReuse, cancellationToken) - .WithCancellation(cancellationToken)) + await foreach (var (channelId, sample, offset, isEof) in + ReadDoubleSamplesAsync(cutRange, allowBufferReuse, cancellationToken) + .WithCancellation(cancellationToken)) + { + ReadOnlyMemory result = convert(sample, buffer.AsMemory()[..bytesPerSample]); + // При allowBufferReuse=true все пакеты используют ОДИН внутренний буфер! + var chunk = allowBufferReuse ? result : result.ToArray(); + yield return new AudioPacket(channelId, chunk, offset, isEof); + } + } + finally { - ReadOnlyMemory result = convert(sample, buffer); - // При true все пакеты используют ОДИН внутренний буфер! - var chunk = allowBufferReuse ? result : result.ToArray(); - yield return new(channelId, chunk, offset, isEof); + ArrayPool.Shared.Return(buffer); } } @@ -246,8 +263,10 @@ public async IAsyncEnumerable ReadStreamableChunksAsync( await foreach (var (channelId, sample, position, isEof) in ConvertToFormatAsync( targetFormat, cutRange, - allowBufferReuse, - cancellationToken: cancellationToken).WithCancellation(cancellationToken)) + allowBufferReuse: false, // Внутренний буфер уже переиспользуется, нужен новый пакет + cancellationToken: cancellationToken) + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) { lastOffset = position; diff --git a/src/Sa.Media/AsyncWavWriter.cs b/src/Sa.Media/AsyncWavWriter.cs index 949e4410..4046f7e9 100644 --- a/src/Sa.Media/AsyncWavWriter.cs +++ b/src/Sa.Media/AsyncWavWriter.cs @@ -85,23 +85,20 @@ private void WriteHeader() ///// Каждый элемент samples — семплы для одного канала ///// Все ReadOnlyMemory должны быть одинаковой длины ///// - public ValueTask WriteSamplesAsync(ReadOnlyMemory interleavedSamples, CancellationToken cancellationToken = default) + public async ValueTask WriteSamplesAsync(ReadOnlyMemory interleavedSamples, CancellationToken cancellationToken = default) { - if (MemoryMarshal.TryGetArray(interleavedSamples, out ArraySegment array)) + if (MemoryMarshal.TryGetArray(interleavedSamples, out var segment)) { - return new ValueTask(WriteSamplesAsync(array.Array!, array.Offset, array.Count, cancellationToken)); + await WriteSamplesAsync(segment.Array!, segment.Offset, segment.Count, cancellationToken).ConfigureAwait(false); + return; } - var sharedBuffer = ArrayPool.Shared.Rent(interleavedSamples.Length); - interleavedSamples.Span.CopyTo(sharedBuffer); - return new ValueTask(FinishWriteAsync(WriteSamplesAsync(sharedBuffer, 0, interleavedSamples.Length, cancellationToken), sharedBuffer)); - } - - private static async Task FinishWriteAsync(Task writeTask, double[] localBuffer) - { + // Данные не в backing-массиве — копируем в локальный буфер + var localBuffer = ArrayPool.Shared.Rent(interleavedSamples.Length); try { - await writeTask.ConfigureAwait(false); + interleavedSamples.Span.CopyTo(localBuffer); + await WriteSamplesAsync(localBuffer, 0, interleavedSamples.Length, cancellationToken).ConfigureAwait(false); } finally { @@ -115,7 +112,7 @@ private async Task WriteSamplesAsync(double[] samples, int offset, int count, Ca { WriteSampleCore(samples[i]); if (_currentBufferSize >= _currentBuffer.Length) - await FlushBufferAsync(cancellationToken); + await FlushBufferAsync(cancellationToken).ConfigureAwait(false); } } @@ -161,11 +158,28 @@ private void WriteSample16Bit(double value) private void WriteSample24Bit(double value) { - int i = (int)(value * int.MaxValue / short.MaxValue); + // Защита от невалидных входных значений + if (double.IsNaN(value) || double.IsInfinity(value)) + value = 0.0; + + // Корректный масштаб для 24 бит + const double max24Bit = 8388607.0; // 2^23 - 1 + double scaled = value * max24Bit; + + // Клиппинг + if (scaled > max24Bit) scaled = max24Bit; + else if (scaled < -max24Bit - 1) scaled = -max24Bit - 1; + + int i = (int)scaled; + + // Проверка доступного места (если не гарантировано снаружи) + if (_currentBufferSize + 3 > _currentBuffer.Span.Length) + throw new InvalidOperationException("Buffer overflow"); + var span = _currentBuffer.Span[_currentBufferSize..]; - span[0] = (byte)(i & 0xFF); - span[1] = (byte)((i >> 8) & 0xFF); - span[2] = (byte)((i >> 16) & 0xFF); + span[0] = (byte)(i & 0xFF); // младший байт + span[1] = (byte)((i >> 8) & 0xFF); // средний + span[2] = (byte)((i >> 16) & 0xFF);// старший (little-endian) _currentBufferSize += 3; _dataSize += 3; } @@ -185,7 +199,6 @@ private void WriteSample64Bit(double value) _dataSize += sizeof(double); } - private async Task FlushBufferAsync(CancellationToken cancellationToken) { if (_currentBufferSize == 0) return; diff --git a/src/Sa.Media/BinaryPipeReader.cs b/src/Sa.Media/BinaryPipeReader.cs index 73d71e0f..68676663 100644 --- a/src/Sa.Media/BinaryPipeReader.cs +++ b/src/Sa.Media/BinaryPipeReader.cs @@ -1,4 +1,6 @@ -using System.IO.Pipelines; +using System.Buffers; +using System.Buffers.Binary; +using System.IO.Pipelines; namespace Sa.Media; @@ -6,27 +8,60 @@ internal sealed class BinaryPipeReader(PipeReader reader) { public long Position { get; private set; } - public async ValueTask ReadUInt32Async(CancellationToken cancellationToken) + public async ValueTask ReadUInt32Async(CancellationToken cancellationToken = default) { var idBuffer = await reader.ReadAtLeastAsync(4, cancellationToken); - uint result = BitConverter.ToUInt32(idBuffer.Buffer.First.Span); + uint result = ReadUInt32Little(idBuffer.Buffer); reader.AdvanceTo(idBuffer.Buffer.GetPosition(4)); Position += 4; return result; } - public async ValueTask ReadUInt16Async(CancellationToken cancellationToken) + public async ValueTask ReadUInt16Async(CancellationToken cancellationToken = default) { var idBuffer = await reader.ReadAtLeastAsync(2, cancellationToken); - ushort result = BitConverter.ToUInt16(idBuffer.Buffer.First.Span); + ushort result = ReadUInt16Little(idBuffer.Buffer); reader.AdvanceTo(idBuffer.Buffer.GetPosition(2)); Position += 2; return result; } - public async Task SkeepBytesAsync(long offset, CancellationToken cancellationToken) + public async Task SkipBytesAsync(long count, CancellationToken cancellationToken = default) { - await reader.SkipAsync(offset, cancellationToken); - Position += offset; + Position += count; + await PipeReaderExtensions.SkipAsync(reader, count, cancellationToken); + } + + private static uint ReadUInt32Little(ReadOnlySequence seq) + { + if (seq.Length >= 4 && seq.IsSingleSegment) + return BinaryPrimitives.ReadUInt32LittleEndian(seq.First.Span); + + // Многосегментный или недостаточно байт в одном сегменте + Span buf = stackalloc byte[4]; + CopyTo(seq, buf); + return BinaryPrimitives.ReadUInt32LittleEndian(buf); + } + + private static ushort ReadUInt16Little(ReadOnlySequence seq) + { + if (seq.Length >= 2 && seq.IsSingleSegment) + return BinaryPrimitives.ReadUInt16LittleEndian(seq.First.Span); + + Span buf = stackalloc byte[2]; + CopyTo(seq, buf); + return BinaryPrimitives.ReadUInt16LittleEndian(buf); + } + + private static void CopyTo(ReadOnlySequence seq, Span destination) + { + int copied = 0; + foreach (var segment in seq) + { + int toCopy = Math.Min(segment.Length, destination.Length - copied); + segment.Span[..toCopy].CopyTo(destination[copied..]); + copied += toCopy; + if (copied >= destination.Length) break; + } } } diff --git a/src/Sa.Media/PipeReaderExtensions.cs b/src/Sa.Media/PipeReaderExtensions.cs index 1d769969..d18f650d 100644 --- a/src/Sa.Media/PipeReaderExtensions.cs +++ b/src/Sa.Media/PipeReaderExtensions.cs @@ -1,9 +1,13 @@ -using System.IO.Pipelines; +using System.Buffers; +using System.IO.Pipelines; namespace Sa.Media; internal static class PipeReaderExtensions { + /// + /// Пропускает указанное количество байт в PipeReader, эффективно обрабатывая многосегментные последовательности. + /// public static async ValueTask SkipAsync(this PipeReader reader, long count, CancellationToken ct = default) { long remaining = count; @@ -11,13 +15,39 @@ public static async ValueTask SkipAsync(this PipeReader reader, long count { ReadResult result = await reader.ReadAsync(ct); if (result.Buffer.IsEmpty && result.IsCompleted) - break; // или throw new InvalidOperationException("Недостаточно данных") + break; // Недостаточно данных - var toConsume = Math.Min(remaining, result.Buffer.Length); + var toConsume = Math.Min(remaining, (long)result.Buffer.Length); var consumed = result.Buffer.GetPosition(toConsume); reader.AdvanceTo(consumed, consumed); remaining -= toConsume; + + // Если буфер маленький, но нам нужно больше — продолжаем читать + if ((long)result.Buffer.Length <= toConsume && !result.IsCompleted) + continue; + } + return count - remaining; + } + + /// + /// Эффективно пропускает данные, продвигая Buffer полностью когда возможно. + /// Минимизирует количество вызовов AdvanceTo. + /// + public static async ValueTask SkipFullSegmentsAsync(this PipeReader reader, long count, CancellationToken ct = default) + { + long remaining = count; + while (remaining > 0) + { + ReadResult result = await reader.ReadAsync(ct); + if (result.Buffer.IsEmpty && result.IsCompleted) + return; + + var toConsume = Math.Min(remaining, (long)result.Buffer.Length); + var consumed = result.Buffer.GetPosition(toConsume); + + // Продвигаем Buffer до consumed, frontier тоже + reader.AdvanceTo(consumed, consumed); + remaining -= toConsume; } - return count - remaining; // сколько фактически пропущено } } diff --git a/src/Sa.Media/SampleConverter.cs b/src/Sa.Media/SampleConverter.cs index 6f0b964a..0d58b41e 100644 --- a/src/Sa.Media/SampleConverter.cs +++ b/src/Sa.Media/SampleConverter.cs @@ -83,15 +83,20 @@ public static double Convert16BitToDouble(ReadOnlySpan source) } /// - /// Конвертирует 24-битный signed PCM (упакован в 3 байта) в double [-1.0, 1.0] + /// Конвертирует 24-битный signed PCM (упакован в 3 байта, little-endian) в double [-1.0, 1.0] /// public static double Convert24BitToDouble(ReadOnlySpan source) { if (source.Length < 3) throw new ArgumentException("Not enough data for 24-bit sample", nameof(source)); - // Читаем 3 байта и расширяем до int с учётом знака - int value = (source[0] << 8) | (source[1] << 16) | (source[2] << 24); - return (value >> 8) / (double)(1 << 23); // 24 бита → [-8388608..8388607] + // 24-bit little-endian: байты [0]=LSB, [1], [2]=MSB + int value = source[0] | (source[1] << 8) | (source[2] << 16); + + // Знаковое расширение: если бит знака установлен — расширить до int + if ((source[2] & 0x80) != 0) + value |= ~0xFFFFFF; + + return value / (double)(1 << 23); // 24 бита → [-8388608..8388607] } /// diff --git a/src/Sa.Media/WavHeaderReader.cs b/src/Sa.Media/WavHeaderReader.cs index abb69cca..53aee33b 100644 --- a/src/Sa.Media/WavHeaderReader.cs +++ b/src/Sa.Media/WavHeaderReader.cs @@ -31,7 +31,7 @@ public static async Task ReadHeaderAsync( { uint junkSize = await reader.ReadUInt32Async(cancellationToken); if (junkSize % 2 == 1) junkSize++; // align to even size - await reader.SkeepBytesAsync(junkSize, cancellationToken); + await reader.SkipBytesAsync(junkSize, cancellationToken); subchunk1Id = await reader.ReadUInt32Async(cancellationToken); } @@ -95,7 +95,7 @@ public static async Task ReadHeaderAsync( if (paddedSize == 0) throw new InvalidDataException("Invalid WAV file: zero-size chunk"); - await reader.SkeepBytesAsync(paddedSize, cancellationToken); + await reader.SkipBytesAsync(paddedSize, cancellationToken); } } } From 1ca30dc0f72f4c23324d102a01494f343f8d0906 Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 26 Jun 2026 13:07:38 +0300 Subject: [PATCH 10/33] sa ffmpeg --- src/Sa.Media.FFmpeg/FFMpegOptions.cs | 23 ++- src/Sa.Media.FFmpeg/IFFmpegExecutor.cs | 23 ++- .../Services/FFMpegExecutor.cs | 17 ++ .../Services/FFProbeExecutor.cs | 6 +- .../Services/PcmS16LeChannelManipulator.cs | 72 +++++--- .../Services/ProcessExecutor.cs | 47 +++-- src/Sa.Media.FFmpeg/Services/StrExtensions.cs | 9 +- src/Sa.Media/Readme.md | 170 +++++++++++++++--- src/Sa.Media/TimeRangeExtensions.cs | 35 +--- .../FFMpegProcessorTests.cs | 31 ++++ .../Sa.MediaTests/TimeRangeExpanderTests.cs | 2 +- 11 files changed, 329 insertions(+), 106 deletions(-) diff --git a/src/Sa.Media.FFmpeg/FFMpegOptions.cs b/src/Sa.Media.FFmpeg/FFMpegOptions.cs index 5ce1f4c6..858cf182 100644 --- a/src/Sa.Media.FFmpeg/FFMpegOptions.cs +++ b/src/Sa.Media.FFmpeg/FFMpegOptions.cs @@ -4,19 +4,35 @@ namespace Sa.Media.FFmpeg; public sealed record FFMpegOptions { + /// + /// Полный путь к исполняемому файлу ffmpeg/ffprobe. Если null, используется поиск через PATH или sa/native/. + /// [StringLength(255)] public string? ExecutablePath { get; set; } = null; + /// + /// Директория, в которую FFmpeg может записывать выходные файлы. + /// [StringLength(255)] public string? WritableDirectory { get; set; } = null; + /// + /// Таймаут выполнения команд в секундах. По умолчанию используется 5 минут (Constants.DefaultTimeout). + /// public int? TimeoutSeconds { get; set; } + /// + /// Вычисленный таймаут на основе . + /// public TimeSpan? Timeout => TimeoutSeconds > 0 ? TimeSpan.FromSeconds(TimeoutSeconds.Value) : null; - // Валидация после десериализации + /// + /// Валидирует параметры после десериализации. + /// + /// Если WritableDirectory не существует. + /// Если TimeoutSeconds отрицательный. public void Validate() { if (WritableDirectory is not null && !Directory.Exists(WritableDirectory)) @@ -24,5 +40,10 @@ public void Validate() throw new DirectoryNotFoundException( $"FFmpeg writable directory does not exist: {WritableDirectory}"); } + + if (TimeoutSeconds.HasValue && TimeoutSeconds.Value < 0) + { + throw new ArgumentException("TimeoutSeconds must be non-negative", nameof(TimeoutSeconds)); + } } } diff --git a/src/Sa.Media.FFmpeg/IFFmpegExecutor.cs b/src/Sa.Media.FFmpeg/IFFmpegExecutor.cs index 29275219..e42cf1ed 100644 --- a/src/Sa.Media.FFmpeg/IFFmpegExecutor.cs +++ b/src/Sa.Media.FFmpeg/IFFmpegExecutor.cs @@ -38,11 +38,12 @@ public interface IFFMpegExecutor /// /// Path to the input audio file. /// Path to the output file. - /// Optional target sample rate. - /// Optional number of output channels. + /// Optional target sample rate. Use null to preserve the original rate. + /// Optional number of output channels. Use null to preserve the original channel count. /// If true, overwrites the output file if it already exists. + /// Optional override for the operation timeout. /// Cancellation token to cancel the operation. - /// Stdout for log + /// Stderr output from FFmpeg (useful for logging warnings/errors). Task ConvertToPcmS16Le( string inputFileName, string outputFileName, @@ -52,6 +53,22 @@ Task ConvertToPcmS16Le( TimeSpan? timeout = null, CancellationToken cancellationToken = default); + /// + /// Converts an audio file to PCM S16 LE format, preserving the original sample rate and channel count. + /// + /// Path to the input audio file. + /// Path to the output file. + /// If true, overwrites the output file if it already exists. + /// Optional override for the operation timeout. + /// Cancellation token to cancel the operation. + /// Stderr output from FFmpeg (useful for logging warnings/errors). + Task ConvertToPcmS16LePreservingFormat( + string inputFileName, + string outputFileName, + bool isOverwrite = true, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default); + /// /// Converts the input audio stream to PCM S16 LE (16-bit signed integer, little-endian) diff --git a/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs b/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs index a041695f..209f7f16 100644 --- a/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs +++ b/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs @@ -117,4 +117,21 @@ private static string LibopuArg(bool isLibopus) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string QuotePath(string path) => $"\"{path}\""; + + public async Task ConvertToPcmS16LePreservingFormat( + string inputFileName, + string outputFileName, + bool isOverwrite = true, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + return await ConvertToPcmS16Le( + inputFileName, + outputFileName, + outputSampleRate: null, + outputChannelCount: null, + isOverwrite, + timeout, + cancellationToken).ConfigureAwait(false); + } } diff --git a/src/Sa.Media.FFmpeg/Services/FFProbeExecutor.cs b/src/Sa.Media.FFmpeg/Services/FFProbeExecutor.cs index f3467d33..710d50fd 100644 --- a/src/Sa.Media.FFmpeg/Services/FFProbeExecutor.cs +++ b/src/Sa.Media.FFmpeg/Services/FFProbeExecutor.cs @@ -39,8 +39,12 @@ public async Task GetMetaInfo(string filePath, CancellationToken Size: metaDataInfo?.Format?.Size.StrToInt() ); } - catch + catch (JsonException jsonEx) { + logger?.LogWarning( + jsonEx, + "Failed to deserialize FFprobe JSON output for '{FilePath}'. Returning empty metadata.", + filePath); return MediaMetadata.Empty; } } diff --git a/src/Sa.Media.FFmpeg/Services/PcmS16LeChannelManipulator.cs b/src/Sa.Media.FFmpeg/Services/PcmS16LeChannelManipulator.cs index ac5ce5dc..aee200b2 100644 --- a/src/Sa.Media.FFmpeg/Services/PcmS16LeChannelManipulator.cs +++ b/src/Sa.Media.FFmpeg/Services/PcmS16LeChannelManipulator.cs @@ -1,9 +1,10 @@ namespace Sa.Media.FFmpeg.Services; -internal sealed class PcmS16LeChannelManipulator(IFFMpegExecutor? ffmpeg = null, IFFProbeExecutor? ffprobe = null) +internal sealed class PcmS16LeChannelManipulator( + IFFMpegExecutor? ffmpeg = null, + IFFProbeExecutor? ffprobe = null) : IPcmS16LeChannelManipulator { - private readonly IFFMpegExecutor _ffmpeg = ffmpeg ?? IFFMpegExecutor.Default; private readonly IFFProbeExecutor _ffprobe = ffprobe ?? IFFProbeExecutor.Default; @@ -34,16 +35,13 @@ public async Task> SplitAsync( } var outFileExtension = Path.GetExtension(outputFileName) ?? ".wav"; - string outFilePrefix = Path.Combine( Path.GetDirectoryName(outputFileName) ?? string.Empty, - Path.GetFileNameWithoutExtension(outputFileName) - ); - - var file0 = $"{outFilePrefix}{channelSuffix}0{outFileExtension}"; + Path.GetFileNameWithoutExtension(outputFileName)); if (channels == 1) { + var file0 = $"{outFilePrefix}{channelSuffix}0{outFileExtension}"; await _ffmpeg.ConvertToPcmS16Le( inputFileName, file0, @@ -52,23 +50,16 @@ await _ffmpeg.ConvertToPcmS16Le( isOverwrite, timeout, cancellationToken); - return [file0]; } - List files = [ + var files = new[] + { $"{outFilePrefix}{channelSuffix}0{outFileExtension}", $"{outFilePrefix}{channelSuffix}1{outFileExtension}" - ]; - - string over = isOverwrite ? "-y" : string.Empty; - var sampleRate = outputSampleRate.HasValue ? $"-ar {outputSampleRate}" : string.Empty; - - string cmd = $"{over} {Constants.CleanBannerFlags} -i \"{inputFileName}\" " + - $"-filter_complex \"[0:a]channelsplit=channel_layout=stereo[left][right]\" " + - $"-map \"[left]\" -acodec pcm_s16le -ac 1 -sample_fmt s16 {sampleRate} -f wav \"{files[0]}\" " + - $"-map \"[right]\" -acodec pcm_s16le -ac 1 -sample_fmt s16 {sampleRate} -f wav \"{files[1]}\""; + }; + var cmd = BuildSplitCommand(inputFileName, files, outputSampleRate, isOverwrite); _ = await _ffmpeg.Executor.ExecuteAsync( cmd, @@ -93,15 +84,7 @@ public async Task JoinAsync( ArgumentNullException.ThrowIfNullOrWhiteSpace(rightFileName); ArgumentNullException.ThrowIfNullOrWhiteSpace(outputFileName); - string over = isOverwrite ? "-y" : string.Empty; - var sampleRate = outputSampleRate.HasValue ? $"-ar {outputSampleRate}" : string.Empty; - - string cmd = - $"{over} {Constants.CleanBannerFlags} -i \"{leftFileName}\" -i \"{rightFileName}\" " + - $"-filter_complex \"[0:a][1:a]amerge=inputs=2[a]\" -map \"[a]\" -ac 2 " + - $"-acodec pcm_s16le -sample_fmt s16 {sampleRate} " + - $"-f wav {Constants.CleanWavOutputFlags} \"{outputFileName}\""; - + var cmd = BuildJoinCommand(leftFileName, rightFileName, outputFileName, outputSampleRate, isOverwrite); _ = await _ffmpeg.Executor.ExecuteAsync( cmd, @@ -111,4 +94,39 @@ public async Task JoinAsync( return outputFileName; } + + #region Private command builders + + private static string BuildSplitCommand( + string inputFileName, + string[] outputFiles, + int? outputSampleRate, + bool isOverwrite) + { + string over = isOverwrite ? "-y" : string.Empty; + var sampleRate = outputSampleRate.HasValue ? $"-ar {outputSampleRate}" : string.Empty; + + return $"{over} {Constants.CleanBannerFlags} -i \"{inputFileName}\" " + + $"-filter_complex \"[0:a]channelsplit=channel_layout=stereo[left][right]\" " + + $"-map \"[left]\" -acodec pcm_s16le -ac 1 -sample_fmt s16 {sampleRate} -f wav \"{outputFiles[0]}\" " + + $"-map \"[right]\" -acodec pcm_s16le -ac 1 -sample_fmt s16 {sampleRate} -f wav \"{outputFiles[1]}\""; + } + + private static string BuildJoinCommand( + string leftFileName, + string rightFileName, + string outputFileName, + int? outputSampleRate, + bool isOverwrite) + { + string over = isOverwrite ? "-y" : string.Empty; + var sampleRate = outputSampleRate.HasValue ? $"-ar {outputSampleRate}" : string.Empty; + + return $"{over} {Constants.CleanBannerFlags} -i \"{leftFileName}\" -i \"{rightFileName}\" " + + $"-filter_complex \"[0:a][1:a]amerge=inputs=2[a]\" -map \"[a]\" -ac 2 " + + $"-acodec pcm_s16le -sample_fmt s16 {sampleRate} " + + $"-f wav {Constants.CleanWavOutputFlags} \"{outputFileName}\""; + } + + #endregion } diff --git a/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs b/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs index 56804e32..aee7a054 100644 --- a/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs +++ b/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs @@ -1,4 +1,5 @@ -using System.Diagnostics; +using Microsoft.Extensions.Logging; +using System.Diagnostics; using System.Text; namespace Sa.Media.FFmpeg.Services; @@ -6,8 +7,14 @@ namespace Sa.Media.FFmpeg.Services; internal interface IProcessExecutor { /// - /// Executes a process with real-time output handling + /// Executes a process with real-time output handling via data received events. /// + /// Process start configuration. + /// Callback for stdout lines. If null, stdout is not redirected. + /// Callback for stderr lines. If null, stderr is not redirected. + /// Operation timeout. Use null for no timeout. + /// Cancellation token. + /// The process exit code. Task ExecuteAsync( ProcessStartInfo startInfo , Action? outputDataReceived = null @@ -17,7 +24,8 @@ ProcessStartInfo startInfo /// - /// Executes a process and returns complete output + /// Executes a process and collects all output into a . + /// Throws if exit code is non-zero. /// async Task ExecuteWithResultAsync( ProcessStartInfo startInfo @@ -47,9 +55,15 @@ ProcessStartInfo startInfo } /// - /// Executes stdout as a stream. - /// Stderr is captured and checked on completion + /// Executes a process and streams stdout through a callback. Stderr is collected and checked on completion. + /// The input stream is copied to stdin asynchronously, then stdin is closed automatically. /// + /// Process start configuration. + /// Readable stream to copy to stdin. + /// Callback that receives the stdout stream. Must read until EOF. + /// Operation timeout. + /// Cancellation token. + /// Thrown when FFmpeg returns a non-zero exit code. Task ExecuteStdOutAsync( ProcessStartInfo startInfo , Stream inputStream @@ -63,7 +77,7 @@ ProcessStartInfo startInfo -internal sealed class ProcessExecutor : IProcessExecutor +internal sealed class ProcessExecutor(ILogger? logger = null) : IProcessExecutor { public async Task ExecuteAsync( ProcessStartInfo startInfo @@ -94,7 +108,7 @@ ProcessStartInfo startInfo return exitCode; } - private static async Task ExecuteProcessWithHandlersAsync( + private async Task ExecuteProcessWithHandlersAsync( Process process, Action? outputDataReceived, Action? errorDataReceived, @@ -324,14 +338,19 @@ await inputStream.CopyToAsync(process.StandardInput.BaseStream, cancellationToke await process.StandardInput.FlushAsync(cancellationToken).ConfigureAwait(false); process.StandardInput.Close(); } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + catch (OperationCanceledException) { + // Cancellation — close stdin to unblock FFmpeg process.StandardInput.Close(); } catch (IOException) { process.StandardInput.Close(); } + catch (ObjectDisposedException) + { + // StandardInput may be disposed if process has exited + } catch (Exception ex) { process.StandardInput.Close(); @@ -339,7 +358,7 @@ await inputStream.CopyToAsync(process.StandardInput.BaseStream, cancellationToke } } - private static int SafeDisposeProcess(Process process) + private int SafeDisposeProcess(Process process) { int exitCode = -1; @@ -389,16 +408,16 @@ private static int SafeDisposeProcess(Process process) } else { - Console.WriteLine("Process did not terminate after Kill()"); + logger?.LogWarning("Process did not terminate after Kill()"); } } - catch (InvalidOperationException) + catch (InvalidOperationException ex) { - Console.WriteLine("Process is invalid or already disposed."); + logger?.LogWarning(ex, "Process is invalid or already disposed."); } catch (Exception ex) { - Console.WriteLine($"Unexpected error during process termination: {ex.Message}"); + logger?.LogError(ex, "Unexpected error during process termination"); } finally { @@ -408,7 +427,7 @@ private static int SafeDisposeProcess(Process process) } catch { - // skeep + // skip } } diff --git a/src/Sa.Media.FFmpeg/Services/StrExtensions.cs b/src/Sa.Media.FFmpeg/Services/StrExtensions.cs index d623c742..d00af0ea 100644 --- a/src/Sa.Media.FFmpeg/Services/StrExtensions.cs +++ b/src/Sa.Media.FFmpeg/Services/StrExtensions.cs @@ -6,9 +6,10 @@ namespace Sa.Media.FFmpeg.Services; internal static class StrExtensions { [DebuggerStepThrough] - public static int? StrToInt(this string? str) - => int.TryParse(str, CultureInfo.InvariantCulture, out int result) ? result : null; + public static int? StrToInt(this ReadOnlySpan str) + => int.TryParse(str, CultureInfo.InvariantCulture, out var r) ? r : null; + [DebuggerStepThrough] - public static double? StrToDouble(this string? str) - => double.TryParse(str, CultureInfo.InvariantCulture, out double result) ? result : null; + public static double? StrToDouble(this ReadOnlySpan str) + => double.TryParse(str, CultureInfo.InvariantCulture, out var r) ? r : null; } diff --git a/src/Sa.Media/Readme.md b/src/Sa.Media/Readme.md index 1699f30b..5803a21a 100644 --- a/src/Sa.Media/Readme.md +++ b/src/Sa.Media/Readme.md @@ -1,44 +1,166 @@ -# AsyncWavReader +# Sa.Media -Async and memory-efficient WAV file reader for .NET +Async, memory-efficient WAV file reader for .NET 10+. Designed for Native AOT compatibility with zero allocations on hot paths. -This library provides an asynchronous and memory-optimized way to read and process WAV audio files in .NET. It supports: +## Features -- PCM 16-bit -- IEEE Float 32-bit -- IEEE Double 64-bit -- Streamable chunks -- Time-based trimming +- **Fully asynchronous** — `PipeReader`-based streaming, no blocking I/O +- **Memory efficient** — `ArrayPool`/`MemoryPool` buffer reuse, minimal GC pressure +- **Multi-format support** — PCM 8/16/24/32-bit, IEEE Float 32/64-bit +- **Extensible** — supports `WAVE_FORMAT_EXTENSIBLE` chunks +- **Time-based trimming** — read only the portion you need via `TimeRange` +- **Channel-aware** — per-channel sample enumeration with position tracking +- **Automatic chunk skipping** — `JUNK`, `LIST`, and other metadata chunks are transparently skipped +## Quick Start -## Read WAV Header +### Read header ```csharp using var stream = File.OpenRead("test.wav"); var reader = new AsyncWavReader(stream); var header = await reader.GetHeaderAsync(); -Console.WriteLine($"Sample Rate: {header.SampleRate}, Channels: {header.NumChannels}"); +Console.WriteLine($"{header.NumChannels}ch @ {header.SampleRate}Hz, " + + $"{header.BitsPerSample}-bit {header.AudioFormat}"); ``` -## Read Data +### Read raw samples per channel ```csharp - using var reader = AsyncWavReader.CreateFromFile("test.wav"); +await using var reader = AsyncWavReader.CreateFromFile("test.wav"); - await foreach (var (channel, samples, pos, _) in reader.ReadStreamableChunksAsync( - bufferSize: 1024, - cancellationToken: TestContext.Current.CancellationToken)) - { - Assert.True(samples.Length > 0); - return; - } +await foreach (var packet in reader.ReadSamplesPerChannelAsync( + cancellationToken: ct)) +{ + Console.WriteLine($"Ch#{packet.ChannelId}: {packet.Sample.Length} bytes at pos {packet.Position}"); +} +``` + +### Read normalized double samples [-1.0 … 1.0] + +```csharp +await using var reader = AsyncWavReader.CreateFromFile("test.wav"); + +await foreach (var packet in reader.ReadDoubleSamplesAsync(cancellationToken: ct)) +{ + Console.WriteLine($"Ch#{packet.ChannelId}: {packet.Sample:F4}"); +} +``` + +### Streamable batches (ideal for audio pipelines) + +```csharp +await using var reader = AsyncWavReader.CreateFromFile("test.wav"); + +await foreach (var batch in reader.ReadStreamableChunksAsync( + samplesPerBatch: 4096, + cancellationToken: ct)) +{ + // Each yield produces independent data — safe to process asynchronously +} +``` + +### Trim by time range + +```csharp +await using var reader = AsyncWavReader.CreateFromFile("test.wav"); + +// Read only seconds 5–15 +var range = TimeRange.Seconds(5, 15); +await foreach (var packet in reader.ReadDoubleSamplesAsync(range, cancellationToken: ct)) +{ + // Samples from the trimmed range only +} +``` + +### Convert to different format + +```csharp +await using var reader = AsyncWavReader.CreateFromFile("input.wav"); + +// Convert to 24-bit PCM +await foreach (var packet in reader.ConvertToFormatAsync( + AudioEncoding.Pcm24BitSigned, + cancellationToken: ct)) +{ + // Raw 24-bit PCM bytes per sample +} ``` ## Supported Formats -- ✅ PCM 16 -- ✅ IEEE Float 32 -- ✅ IEEE Double 64 -- ✅ Stereo / Mono -- ✅ Extra chunks like "JUNK", "LIST" Automatically skipped +| Format | Read | Write | +|--------|------|-------| +| PCM 8-bit (unsigned) | ✅ | ✅ | +| PCM 16-bit (signed) | ✅ | ✅ | +| PCM 24-bit (signed) | ✅ | ✅ | +| PCM 32-bit (signed) | ✅ | ✅ | +| IEEE Float 32-bit | ✅ | ✅ | +| IEEE Float 64-bit | ✅ | ✅ | + +All formats support mono and stereo. Unknown chunks (`JUNK`, `LIST`, etc.) are automatically skipped. + +## Public API Reference + +### Core types + +| Type | Description | +|------|-------------| +| `AsyncWavReader` | Main async WAV reader — creates from `Stream` or file path | +| `WavHeader` | Parsed RIFF/WAV header with computed properties (`IsPcm`, `IsStereo`, `Duration`) | +| `AudioPacket` | Record: `(ChannelId, Sample, Position, IsEof)` — raw/conversion bytes | +| `AudioNormalizedPacket` | Record: `(ChannelId, Sample, Position, IsEof)` — normalized double [-1.0, 1.0] | +| `TimeRange` | Record: `(From, To)` — time-based trimming with factory methods | +| `AudioEncoding` | Enum: PCM 8/16/24/32, IEEE Float 32/64 | +| `WaveFormatType` | Enum: `Pcm`, `Adpcm`, `IeeeFloat`, `Extensible` | + +### Key methods on `AsyncWavReader` + +| Method | Returns | Description | +|--------|---------|-------------| +| `Create(Stream)` | `AsyncWavReader` | Factory from stream | +| `CreateFromFile(string)` | `AsyncWavReader` | Factory from file path | +| `GetHeaderAsync()` | `Task` | Thread-safe lazy header parsing | +| `ReadSamplesPerChannelAsync()` | `IAsyncEnumerable` | Raw samples per channel | +| `ReadDoubleSamplesAsync()` | `IAsyncEnumerable` | Normalized double samples | +| `ConvertToFormatAsync()` | `IAsyncEnumerable` | Convert to target encoding | +| `ReadStreamableChunksAsync()` | `IAsyncEnumerable` | Batched samples for pipelines | + +### `TimeRange` factories + +| Method | Example | Description | +|--------|---------|-------------| +| `TimeRange.Create(from, to)` | Basic constructor | From/to TimeSpan | +| `TimeRange.Ms(from, to)` | By milliseconds | Millisecond precision | +| `TimeRange.Seconds(from, to)` | By seconds | Double-second precision | +| `TimeRange.RangeFromDuration(from, dur)` | From start + duration | Build from offset | +| `TimeRange.Default` | `[0, ∞)` | Full file, no trim | + +## Performance Notes + +- `allowBufferReuse=true` (default) reuses pooled buffers across yields — caller must copy before next iteration +- `allowBufferReuse=false` allocates a fresh array per sample — safer for parallel consumers +- `ReadStreamableChunksAsync` forces `allowBufferReuse:false` internally to prevent buffer aliasing +- All internal awaits use `ConfigureAwait(false)` — safe in any synchronization context + +## Project Layout + +``` +src/Sa.Media/ +├── AsyncWavReader.cs # Main reader class +├── AsyncWavWriter.cs # Internal WAV writer +├── AudioEncoding.cs # Format enum +├── AudioEncodingExtensions.cs +├── AudioPacket.cs # Raw sample record +├── AudioNormalizedPacket.cs # Normalized sample record +├── BinaryPipeReader.cs # Little-endian binary reader +├── PipeReaderExtensions.cs # Skip helpers +├── SampleConverter.cs # PCM ↔ double conversion +├── TimeRange.cs # Trimming range +├── TimeRangeExtensions.cs # Expander, merge, sort +├── WavHeader.cs # RIFF header model +├── WavHeaderReader.cs # Header parser +├── WaveFormatType.cs # Format type enum +└── WaveFormatTypeExtensions.cs +``` diff --git a/src/Sa.Media/TimeRangeExtensions.cs b/src/Sa.Media/TimeRangeExtensions.cs index a5fa09e8..2f2edc55 100644 --- a/src/Sa.Media/TimeRangeExtensions.cs +++ b/src/Sa.Media/TimeRangeExtensions.cs @@ -28,40 +28,13 @@ public static IReadOnlyCollection Merge( => MergeCloseRanges(ranges, thresholdMillesecods); - internal ref struct PooledList(int capacity) - { - private T[] _array = ArrayPool.Shared.Rent(capacity); - private int _count = 0; - - public void Add(T item) - { - if (_count >= capacity) - throw new InvalidOperationException("Capacity exceeded"); - - _array[_count++] = item; - } - - public readonly T[] ToArray() - { - var result = new T[_count]; - Array.Copy(_array, result, _count); - return result; - } - - public void Dispose() - { - if (_array != null) - { - ArrayPool.Shared.Return(_array); - _array = null!; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveOptimization)] +#pragma warning disable S3776 +#pragma warning disable S2368 public static TimeRange[][] ExpandTimeRanges( +#pragma warning restore S2368 +#pragma warning restore S3776 TimeRange[][] chunks, int thresholdMillesecods = 300, int gapMilliseconds = 0) diff --git a/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs b/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs index 6cb97691..4f2cbddb 100644 --- a/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs +++ b/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs @@ -240,6 +240,37 @@ await Processor.ConvertToPcmS16Le( Assert.Equal(8000, sampleRate); } + [Theory] + [InlineData("./data/input.wav")] + public async Task ConvertToPcmS16LePreservingFormat_ShouldPreserveOriginalSettings(string inputPath) + { + // Arrange + string outputPath = "./data/output_preserved.wav"; + + if (File.Exists(outputPath)) + File.Delete(outputPath); + + // Получаем исходные настройки + var originalInfo = await IFFProbeExecutor.Default.GetMetaInfo(inputPath, CancellationToken); + var (origChannels, origSampleRate) = await IFFProbeExecutor.Default.GetChannelsAndSampleRate(inputPath, CancellationToken); + + // Act + await Processor.ConvertToPcmS16LePreservingFormat( + inputFileName: inputPath, + outputFileName: outputPath, + isOverwrite: true, + cancellationToken: CancellationToken); + + // Assert + Assert.True(File.Exists(outputPath)); + + var ffprobe = CreateFFProbeExecutor(); + var (outChannels, outSampleRate) = await ffprobe.GetChannelsAndSampleRate(outputPath, cancellationToken: CancellationToken); + + Assert.Equal(origChannels, outChannels); + Assert.Equal(origSampleRate, outSampleRate); + } + private static IFFProbeExecutor CreateFFProbeExecutor() { return IFFProbeExecutor.Default; diff --git a/src/Tests/Sa.MediaTests/TimeRangeExpanderTests.cs b/src/Tests/Sa.MediaTests/TimeRangeExpanderTests.cs index 68d2af53..92f359ba 100644 --- a/src/Tests/Sa.MediaTests/TimeRangeExpanderTests.cs +++ b/src/Tests/Sa.MediaTests/TimeRangeExpanderTests.cs @@ -2,7 +2,7 @@ namespace Sa.MediaTests; -public class TimeRangeExpanderTests +public sealed class TimeRangeExpanderTests { #region Helper Methods From 820b5ff8be21343effdc34ccec71e4030b88a144 Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 26 Jun 2026 13:59:21 +0300 Subject: [PATCH 11/33] improve s3 --- src/Sa.Data.S3/README.md | 131 +++++++++++++----- src/Sa.Data.S3/S3BucketClient.Buckets.cs | 40 +++++- src/Sa.Data.S3/S3BucketClientSetupSettings.cs | 14 ++ src/Sa.Data.S3/S3Stream.cs | 9 +- src/Sa.Data.S3/Setup.cs | 4 +- 5 files changed, 154 insertions(+), 44 deletions(-) diff --git a/src/Sa.Data.S3/README.md b/src/Sa.Data.S3/README.md index d8d3c9c2..5a6e220b 100644 --- a/src/Sa.Data.S3/README.md +++ b/src/Sa.Data.S3/README.md @@ -1,64 +1,119 @@ # Sa.Data.S3 -This is a fork of https://github.com/teoadal/Storage , which is a wrapper around HttpClient for working with S3-compatible storage. It offers performance comparable to MinIO while consuming almost 200 times less memory than the AWS SDK client. +Обёртка над `HttpClient` для работы с S3-совместимыми хранилищами (Minio, AWS S3, DigitalOcean Spaces и др.). Полностью собственная реализация AWS Signature Version 4 — **без зависимостей от AWS SDK или Minio SDK**. -## Forked +## Мотивация -- https://github.com/dundich/Storage; -- https://github.com/teoadal/Storage; +Это форк https://github.com/teoadal/Storage. Мотивация — клиенты [AWS SDK for .NET](https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/welcome.html) (4.x) и [Minio .NET](https://github.com/minio/minio-dotnet) (6.x) потребляли слишком много памяти. Результат: скорость почти как у AWS, а потребление памяти в ~150 раз меньше чем Minio SDK и в ~17 раз меньше AWS SDK. +## Создание клиента -Это обертка над HttpClient для работы с S3 хранилищами. Мотивация создания была простейшей - я не понимал, -почему клиенты [AWS](https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/welcome.html) (4.0.0) -и [Minio](https://github.com/minio/minio-dotnet) (6.0.4) потребляют так много памяти. Результат экспериментов: скорость -почти как у AWS, а потребление памяти почти в 150 раз меньше, чем клиент для Minio (и в 17 для AWS). +### Без DI +```csharp +var client = new S3BucketClient(new HttpClient(), new S3BucketClientSetupSettings +{ + Bucket = "mybucket", + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123" +}); +``` -## Creating a Client - -To interact with an S3-compatible storage system, you need to create a client using the provided configuration. Below is a detailed explanation of the code snippet and its parameters. +### С DI ```csharp - -var client = new S3BucketClient(new HttpClient(), new S3BucketClientSettings +services.AddSaS3BucketClient(new S3BucketClientSetupSettings { - Bucket = "mybucket", - Endpoint = "http://localhost:9000", - AccessKey = "ROOTUSER", - SecretKey = "ChangeMe123" -}; - + Bucket = "mybucket", + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123", + TotalRequestTimeout = TimeSpan.FromSeconds(180), + ConnectionPoolLifetime = TimeSpan.FromMinutes(15), + HandlerLifetime = Timeout.InfiniteTimeSpan // или TimeSpan.FromHours(2) для периодического обновления handler +}); + +// Использование: +var client = serviceProvider.GetRequiredService(); ``` -## IS3BucketClient +## Настройки -```csharp +| Свойство | Описание | По умолчанию | +|---|---|---| +| `AccessKey` | Ключ доступа S3 | *(обязательно)* | +| `SecretKey` | Секретный ключ S3 | *(обязательно)* | +| `Bucket` | Имя бакета | *(обязательно)* | +| `Endpoint` | URL S3-хранилища | *(обязательно)* | +| `Region` | Регион для SigV4 | `"us-east-1"` | +| `Service` | Сервис для SigV4 | `"s3"` | +| `UseHttp2` | Принудительный HTTP/2 | `false` | +| `TotalRequestTimeout` | Таймаут каждого запроса | `180 сек` | +| `ConnectionPoolLifetime` | Время жизни пула соединений | `15 мин` | +| `HandlerLifetime` | Время жизни HttpClient handler | `∞` (бесконечность) | + +## API + +### IBucketOperations +```csharp public interface IBucketOperations { - Task CreateBucket(CancellationToken ct); - Task DeleteBucket(CancellationToken ct); - Task IsBucketExists(CancellationToken ct); + Task CreateBucket(CancellationToken ct); + Task DeleteBucket(CancellationToken ct); + Task DeleteBucket(bool forceDelete, CancellationToken ct); // force: удалить все объекты перед удалением bucket + Task IsBucketExists(CancellationToken ct); } +``` + +### IFileOperations +```csharp public interface IFileOperations { - string BuildFileUrl(string fileName); - string BuildFileUrl(string fileName, TimeSpan expiration); - Task DeleteFile(string fileName, CancellationToken ct); - Task GetFile(string fileName, CancellationToken ct); - Task GetFileStream(string fileName, CancellationToken ct); - Task GetFileUrl(string fileName, TimeSpan expiration, CancellationToken ct); - Task IsFileExists(string fileName, CancellationToken ct); - IAsyncEnumerable List(string? prefix, CancellationToken ct); - Task UploadFile(string fileName, string contentType, byte[] data, CancellationToken ct); - Task UploadFile(string fileName, string contentType, CancellationToken ct); - Task UploadFile(string fileName, string contentType, Stream data, CancellationToken ct); + string BuildFileUrl(string fileName); + string BuildFileUrl(string fileName, TimeSpan expiration); + Task DeleteFile(string fileName, CancellationToken ct); + Task GetFile(string fileName, CancellationToken ct); + Task GetFileStream(string fileName, CancellationToken ct); + Task GetFileUrl(string fileName, TimeSpan expiration, CancellationToken ct); + Task IsFileExists(string fileName, CancellationToken ct); + IAsyncEnumerable List(string? prefix, CancellationToken ct); // с pagination + Task UploadFile(string fileName, string contentType, byte[] data, CancellationToken ct); + Task UploadFile(string fileName, string contentType, CancellationToken ct); // ручной multipart + Task UploadFile(string fileName, string contentType, Stream data, CancellationToken ct); } +``` + +### Rучной Multipart Upload + +Для файлов > 5MB автоматически выбирается multipart upload. Для ручного управления: + +```csharp +using var uploader = await client.UploadFile("large-file.bin", "application/octet-stream", ct); -public interface IS3BucketClient: IBucketOperations, IFileOperations +uploader.AddPart(chunkData, ct); +uploader.AddPart(chunkData, offset, length, ct); // перегрузка с offset +uploader.AddParts(fullDataStream, ct); +uploader.AddParts(fullByteArray, ct); + +if (await uploader.Complete(ct)) +{ + Console.WriteLine($"Uploaded {uploader.Written} bytes"); +} +else { - string Bucket { get; } - Uri Endpoint { get; } + await uploader.Abort(ct); } ``` + +## Особенности реализации + +- **AWS SigV4** — полная ручная реализация подписывания запросов (SHA256 + HMAC-SHA256 chain) +- **ArrayPool.Shared** — пулинг буферов для минимизации GC pressure +- **ref struct ValueStringBuilder** — стек-based строковый билдер без аллокаций +- **stackalloc** — везде где возможно для избежания heap allocation +- **Буферизированный XML парсер** — эффективное чтение ответов S3 (ListObjects, Multipart IDs) +- **Pagination** — автоматическая обработка `IsTruncated` / `NextContinuationToken` в `List()` +- **CancellationToken** — поддерживается во всех async операциях diff --git a/src/Sa.Data.S3/S3BucketClient.Buckets.cs b/src/Sa.Data.S3/S3BucketClient.Buckets.cs index af86336d..426de0ba 100644 --- a/src/Sa.Data.S3/S3BucketClient.Buckets.cs +++ b/src/Sa.Data.S3/S3BucketClient.Buckets.cs @@ -6,7 +6,7 @@ namespace Sa.Data.S3; /// /// Функции управления бакетом /// -public partial class S3BucketClient : IBucketOperations +public sealed partial class S3BucketClient : IBucketOperations { public async Task CreateBucket(CancellationToken ct) { @@ -32,6 +32,21 @@ public async Task CreateBucket(CancellationToken ct) public async Task DeleteBucket(CancellationToken ct) { + return await DeleteBucket(forceDelete: false, ct).ConfigureAwait(false); + } + + /// + /// Удаляет бакет. Если forceDelete=true — сначала удаляет все объекты из бакета. + /// + /// Если true, предварительно удалит все объекты в бакете + /// Токен отмены операции + public async Task DeleteBucket(bool forceDelete, CancellationToken ct) + { + if (forceDelete) + { + await DeleteAllObjects(ct).ConfigureAwait(false); + } + HttpResponseMessage response; using (var request = CreateRequest(HttpMethod.Delete)) { @@ -52,6 +67,29 @@ public async Task DeleteBucket(CancellationToken ct) } } + private async Task DeleteAllObjects(CancellationToken ct) + { + var prefixes = new List(); + await foreach (var key in List(null, ct).ConfigureAwait(false)) + { + prefixes.Add(key); + } + + // S3 SelectObjectCancel требует удаления по ключам, не по prefix + // Перечитаем всё по ключам и удалим + foreach (var key in prefixes) + { + try + { + await DeleteFile(key, ct).ConfigureAwait(false); + } + catch + { + // Ignore individual delete failures during force cleanup + } + } + } + public async Task IsBucketExists(CancellationToken ct) { HttpResponseMessage response; diff --git a/src/Sa.Data.S3/S3BucketClientSetupSettings.cs b/src/Sa.Data.S3/S3BucketClientSetupSettings.cs index 5c58ace9..2030f7d2 100644 --- a/src/Sa.Data.S3/S3BucketClientSetupSettings.cs +++ b/src/Sa.Data.S3/S3BucketClientSetupSettings.cs @@ -2,5 +2,19 @@ public sealed class S3BucketClientSetupSettings : S3BucketSettings { + /// + /// Максимальное время ожидания ответа сервера для каждого запроса. По умолчанию: 180 секунд. + /// public TimeSpan TotalRequestTimeout { get; set; } = TimeSpan.FromSeconds(180); + + /// + /// Время жизни пула соединений в SocketsHttpHandler. По умолчанию: 15 минут. + /// + public TimeSpan ConnectionPoolLifetime { get; set; } = TimeSpan.FromMinutes(15); + + /// + /// Время жизни обработчика HttpClient. По умолчанию: бесконечность (для long-running сервисов). + /// Установите в TimeSpan.FromHours(2) для периодического пересоздания handler и освобождения stale connections. + /// + public TimeSpan HandlerLifetime { get; set; } = Timeout.InfiniteTimeSpan; } diff --git a/src/Sa.Data.S3/S3Stream.cs b/src/Sa.Data.S3/S3Stream.cs index 07795b26..8a379802 100644 --- a/src/Sa.Data.S3/S3Stream.cs +++ b/src/Sa.Data.S3/S3Stream.cs @@ -74,9 +74,12 @@ public override void Write(byte[] buffer, int offset, int count) protected override void Dispose(bool disposing) { - stream.Dispose(); - response.Dispose(); + if (disposing) + { + stream.Dispose(); + response.Dispose(); + } - base.Dispose(true); + base.Dispose(disposing); } } diff --git a/src/Sa.Data.S3/Setup.cs b/src/Sa.Data.S3/Setup.cs index 224c1834..ac065e98 100644 --- a/src/Sa.Data.S3/Setup.cs +++ b/src/Sa.Data.S3/Setup.cs @@ -19,9 +19,9 @@ public static IServiceCollection AddSaS3BucketClient( }) .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler() { - PooledConnectionLifetime = TimeSpan.FromMinutes(15) + PooledConnectionLifetime = settings.ConnectionPoolLifetime }) - .SetHandlerLifetime(Timeout.InfiniteTimeSpan) + .SetHandlerLifetime(settings.HandlerLifetime) .AddStandardResilienceHandler(options => { options.TotalRequestTimeout.Timeout = settings.TotalRequestTimeout; From 3f3e12293e2a58f34584537637a278b58f49ff3b Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 26 Jun 2026 16:15:09 +0300 Subject: [PATCH 12/33] improve Sa.Data.PostgreSql --- README.md | 16 ++ src/Sa.Data.PostgreSql/DbCommandExtensions.cs | 13 +- src/Sa.Data.PostgreSql/IPgDataSource.cs | 88 ++++++- src/Sa.Data.PostgreSql/IPgDistributedLock.cs | 9 - src/Sa.Data.PostgreSql/PgDataSource.cs | 37 ++- src/Sa.Data.PostgreSql/PgDistributedLock.cs | 94 ------- src/Sa.Data.PostgreSql/Readme.md | 229 +++++++++++++----- src/Sa.Data.PostgreSql/Setup.cs | 2 +- src/Sa.Data.S3/Utils/XmlStreamReader.cs | 2 + src/Sa.Media/PipeReaderExtensions.cs | 10 +- src/Sa.Media/WavHeader.cs | 9 +- src/Sa.Schedule/IJobErrorHandlingBuilder.cs | 7 - src/Samples/Schedule.Console/Program.cs | 2 +- .../PgDataSourceBinaryImportTests.cs | 145 +++++++++++ .../PgDataSourceReaderFirstTests.cs | 172 +++++++++++++ .../PgDataSourceReaderListTests.cs | 141 +++++++++++ .../PgDataSourceScalarTests.cs | 117 +++++++++ .../Sa.MediaTests/TimeRangeExpanderTests.cs | 6 - 18 files changed, 893 insertions(+), 206 deletions(-) delete mode 100644 src/Sa.Data.PostgreSql/IPgDistributedLock.cs delete mode 100644 src/Sa.Data.PostgreSql/PgDistributedLock.cs create mode 100644 src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceBinaryImportTests.cs create mode 100644 src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceReaderFirstTests.cs create mode 100644 src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceReaderListTests.cs create mode 100644 src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceScalarTests.cs diff --git a/README.md b/README.md index 0142ee8f..d2f233b1 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,22 @@ dot net10 experimental aot project +## [Sa.Data.PostgreSql](src/Sa.Data.PostgreSql) + +Лёгкая обёртка над Npgsql для типичных операций с PostgreSQL — без ORM overhead, с поддержкой DI, Native AOT и минимальными аллокациями. + +- **ExecuteNonQuery** — INSERT / UPDATE / DELETE / DDL с возвратом числа строк +- **ExecuteScalar / ExecuteScalarTyped** — получение одиночного значения с авто-кастом +- **ExecuteReader** — потоковое чтение строк через callback (без загрузки всего результата в память) +- **ExecuteReaderList** — сборка всех строк в `List` +- **ExecuteReaderFirst** — первое значение первого столбца (Guid, TimeSpan, DateTime, int, long и др.) +- **ExecuteReaderSingle / TryExecuteReaderSingle** — безопасное scalar-значение +- **ExecuteTransactionAsync** — атомарные операции с авто-commit/rollback +- **BeginBinaryImport** — быстрый COPY BINARY для массового импорта +- **PgDistributedLock** — распределённая блокировка на `pg_try_advisory_lock` +- **PgRetryStrategy** — повтор попыток с jitter для transient-ошибок Npgsql +- **DI-интеграция** — `AddSaPostgreSqlDataSource()` регистрирует всё автоматически + ## [Sa.Outbox.PostgreSql](src/Sa.Outbox.PostgreSql) Designed for implementing the Outbox pattern using PostgreSQL, which is used to ensure reliable message delivery in distributed systems. It helps prevent message loss and guarantees that messages will be processed even in the event of failures. diff --git a/src/Sa.Data.PostgreSql/DbCommandExtensions.cs b/src/Sa.Data.PostgreSql/DbCommandExtensions.cs index cf1e0731..ee5ed1fb 100644 --- a/src/Sa.Data.PostgreSql/DbCommandExtensions.cs +++ b/src/Sa.Data.PostgreSql/DbCommandExtensions.cs @@ -7,7 +7,7 @@ namespace Sa.Data.PostgreSql; public static class DbCommandExtensions { /// - /// Добавляет параметр с именем {prefix}{index}, используя минимальные аллокации. + /// Adds a parameter with name {prefix}{index}, using minimal allocations. /// public static NpgsqlCommand AddParameter( this NpgsqlCommand command, @@ -26,17 +26,22 @@ public static NpgsqlCommand AddParameter( return command; } - + /// + /// Adds a parameter — infers the value type from the argument. + /// public static NpgsqlCommand AddParam( this NpgsqlCommand command, string prefix, - T value, + T? value, int index) where TProvider : INamePrefixProvider { var paramName = CachedParamNames.Default.Get(prefix, index); - var param = new NpgsqlParameter(paramName, value); + var param = command.CreateParameter(); + param.ParameterName = paramName; + param.Value = value is null ? DBNull.Value : (object)value!; command.Parameters.Add(param); + return command; } } diff --git a/src/Sa.Data.PostgreSql/IPgDataSource.cs b/src/Sa.Data.PostgreSql/IPgDataSource.cs index 11978fc2..b68b7ffa 100644 --- a/src/Sa.Data.PostgreSql/IPgDataSource.cs +++ b/src/Sa.Data.PostgreSql/IPgDataSource.cs @@ -28,10 +28,70 @@ Task ExecuteNonQuery(string sql, CancellationToken cancellationToken = defa Task ExecuteScalar( string sql, Action? initCommand, CancellationToken cancellationToken = default); + /// + /// Executes a scalar query with no parameters. + /// + Task ExecuteScalar(string sql, CancellationToken cancellationToken = default) + => ExecuteScalar(sql, null, cancellationToken); + + async Task ExecuteScalar( string sql, Action? initCommand, CancellationToken cancellationToken = default) => ((T)(await ExecuteScalar(sql, initCommand, cancellationToken))!); + + /// + /// Executes a typed scalar query. + /// + async Task ExecuteScalarTyped( + string sql, Action? initCommand = null, CancellationToken cancellationToken = default) + { + var result = await ExecuteScalar(sql, initCommand, cancellationToken); + if (result is null || result is DBNull) + return default!; + + var targetType = typeof(T); + + // Handle value types that Npgsql returns boxed + if (result is T typed) + return typed; + + // Direct cast for common types that may not match 'is T' due to nullable annotations + if (targetType == typeof(Guid) && result is Guid g) + return (T)(object)g; + if (targetType == typeof(DateTime) && result is DateTime dt) + return (T)(object)dt; + if (targetType == typeof(DateTimeOffset) && result is DateTimeOffset dto) + return (T)(object)dto; + + // DateOnly -> DateTime conversion (PostgreSQL 'date' type maps to DateOnly in .NET 6+) + if (targetType == typeof(DateTime) && result is DateOnly dateOnly) + return (T)(object)dateOnly.ToDateTime(TimeOnly.MinValue); + + // Fallback: use as-cast for reference/nullable types + if (result is T directCast) + return directCast; + + return (T)Convert.ChangeType(result, typeof(T), null)!; + } + + /// + /// Executes a typed scalar query with parameters. + /// + async Task ExecuteScalarTyped( + string sql, + IReadOnlyCollection parameters, + CancellationToken cancellationToken = default) + { + var result = await ExecuteScalar(sql, cmd => FillParams(cmd, parameters), cancellationToken); + if (result is null || result is DBNull) + return default!; + if (result is T typed) + return typed; + return (T)Convert.ChangeType(result, typeof(T), null)!; + } + + // ExecuteReader Task ExecuteReader( string sql, @@ -91,18 +151,24 @@ async Task ExecuteReaderFirst( await ExecuteReader(sql, (reader, _) => { - value = Type.GetTypeCode(typeof(T)) switch + value = typeof(T) switch { - TypeCode.Char => (T)(object)reader.GetChar(0), - TypeCode.Int64 => (T)(object)reader.GetInt64(0), - TypeCode.Int32 => (T)(object)reader.GetInt32(0), - TypeCode.String => (T)(object)reader.GetString(0), - TypeCode.Boolean => (T)(object)reader.GetBoolean(0), - TypeCode.Double => (T)(object)reader.GetDouble(0), - TypeCode.DateTime => (T)(object)reader.GetDateTime(0), - TypeCode.Decimal => (T)(object)reader.GetDecimal(0), - TypeCode.DBNull => value, - _ => throw new InvalidOperationException($"Unsupported type: {typeof(T)}"), + Type t when t == typeof(Guid) => (T)(object)reader.GetFieldValue(0), + Type t when t == typeof(DateTimeOffset) => (T)(object)reader.GetFieldValue(0), + _ => Type.GetTypeCode(typeof(T)) switch + { + TypeCode.Char => (T)(object)reader.GetChar(0), + TypeCode.Int64 => (T)(object)reader.GetInt64(0), + TypeCode.Int32 => (T)(object)reader.GetInt32(0), + TypeCode.String => (T)(object)reader.GetString(0), + TypeCode.Boolean => (T)(object)reader.GetBoolean(0), + TypeCode.Double => (T)(object)reader.GetDouble(0), + TypeCode.DateTime => (T)(object)reader.GetDateTime(0), + TypeCode.Decimal => (T)(object)reader.GetDecimal(0), + TypeCode.Int16 => (T)(object)reader.GetInt16(0), + TypeCode.DBNull => value, + _ => throw new InvalidOperationException($"Unsupported type: {typeof(T)}"), + } }; } , parameters diff --git a/src/Sa.Data.PostgreSql/IPgDistributedLock.cs b/src/Sa.Data.PostgreSql/IPgDistributedLock.cs deleted file mode 100644 index 1d4eec50..00000000 --- a/src/Sa.Data.PostgreSql/IPgDistributedLock.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Sa.Data.PostgreSql; - -public interface IPgDistributedLock -{ - Task TryExecuteInDistributedLock( - long lockId, - Func exclusiveLockTask, - CancellationToken cancellationToken); -} diff --git a/src/Sa.Data.PostgreSql/PgDataSource.cs b/src/Sa.Data.PostgreSql/PgDataSource.cs index 40fe0980..2667aefa 100644 --- a/src/Sa.Data.PostgreSql/PgDataSource.cs +++ b/src/Sa.Data.PostgreSql/PgDataSource.cs @@ -1,4 +1,5 @@ using Npgsql; +using System.Data; namespace Sa.Data.PostgreSql; @@ -37,19 +38,39 @@ public async ValueTask BeginBinaryImport( Func> write, CancellationToken cancellationToken = default) { - using NpgsqlConnection db = await OpenDbConnection(cancellationToken); - using NpgsqlBinaryImporter writer = await db.BeginBinaryImportAsync(sql, cancellationToken); + await using NpgsqlConnection db = await OpenDbConnection(cancellationToken); + await using NpgsqlBinaryImporter writer = await db.BeginBinaryImportAsync(sql, cancellationToken); ulong result = await write(writer, cancellationToken); return result; } + public async Task ExecuteTransactionAsync( + Func action, + IsolationLevel isolationLevel = IsolationLevel.Unspecified, + CancellationToken cancellationToken = default) + { + await using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); + await connection.OpenAsync(cancellationToken); + await using NpgsqlTransaction transaction = await connection.BeginTransactionAsync(isolationLevel, cancellationToken); + try + { + await action(transaction, cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + catch + { + await transaction.RollbackAsync(cancellationToken); + throw; + } + } + public async Task ExecuteNonQuery( string sql, Action? initCommand, CancellationToken cancellationToken = default) { - using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); - using NpgsqlCommand cmd = new(sql, connection); + await using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); + await using NpgsqlCommand cmd = new(sql, connection); initCommand?.Invoke(cmd); return await cmd.ExecuteNonQueryAsync(cancellationToken); } @@ -59,8 +80,8 @@ public async Task ExecuteNonQuery( Action? initCommand, CancellationToken cancellationToken = default) { - using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); - using NpgsqlCommand cmd = new(sql, connection); + await using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); + await using NpgsqlCommand cmd = new(sql, connection); initCommand?.Invoke(cmd); return await cmd.ExecuteScalarAsync(cancellationToken); } @@ -74,9 +95,9 @@ public async Task ExecuteReader( int rowCount = 0; using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); - using NpgsqlCommand cmd = new(sql, connection); + await using NpgsqlCommand cmd = new(sql, connection); initCommand?.Invoke(cmd); - using NpgsqlDataReader reader = await cmd.ExecuteReaderAsync(cancellationToken); + await using NpgsqlDataReader reader = await cmd.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken) && !cancellationToken.IsCancellationRequested) { read(reader, rowCount); diff --git a/src/Sa.Data.PostgreSql/PgDistributedLock.cs b/src/Sa.Data.PostgreSql/PgDistributedLock.cs deleted file mode 100644 index 3b49dd87..00000000 --- a/src/Sa.Data.PostgreSql/PgDistributedLock.cs +++ /dev/null @@ -1,94 +0,0 @@ -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Npgsql; - -namespace Sa.Data.PostgreSql; - -/// -/// lock by pg_try_advisory_lock -/// -/// -/// -internal sealed partial class PgDistributedLock( - PgDataSourceSettings settings, - ILogger? logger = null) : IPgDistributedLock -{ - private readonly ILogger _logger = logger ?? NullLogger.Instance; - - private readonly NpgsqlConnectionStringBuilder builder = new(settings.ConnectionString); - - public async Task TryExecuteInDistributedLock(long lockId, Func exclusiveLockTask, CancellationToken cancellationToken) - { - LogTryingToAcquireLock(_logger, lockId); - - using var connection = new NpgsqlConnection(builder.ToString()); - await connection.OpenAsync(cancellationToken); - - bool hasLockedAcquired = await TryAcquireLockAsync(lockId, connection, cancellationToken); - - if (!hasLockedAcquired) - { - LogLockRejected(_logger, lockId); - return false; - } - - LogLockAcquired(_logger, lockId); - try - { - if (await TryAcquireLockAsync(lockId, connection, cancellationToken)) - { - await exclusiveLockTask(cancellationToken); - } - } - finally - { - LogReleasingLock(_logger, lockId); - await ReleaseLock(lockId, connection, cancellationToken); - } - return true; - } - - private static async Task TryAcquireLockAsync(long lockId, NpgsqlConnection connection, CancellationToken cancellationToken) - { - string sessionLockCommand = $"SELECT pg_try_advisory_lock({lockId})"; - using var commandQuery = new NpgsqlCommand(sessionLockCommand, connection); - object? result = await commandQuery.ExecuteScalarAsync(cancellationToken); - if (result != null && bool.TryParse(result.ToString(), out var lockAcquired) && lockAcquired) - { - return true; - } - return false; - } - - private static async Task ReleaseLock(long lockId, NpgsqlConnection connection, CancellationToken cancellationToke) - { - string transactionLockCommand = $"SELECT pg_advisory_unlock({lockId})"; - using var commandQuery = new NpgsqlCommand(transactionLockCommand, connection); - await commandQuery.ExecuteScalarAsync(cancellationToke); - } - - - [LoggerMessage( - EventId = 1001, - Level = LogLevel.Trace, - Message = "Trying to acquire session lock for Lock Id {LockId}")] - static partial void LogTryingToAcquireLock(ILogger logger, long lockId); - - [LoggerMessage( - EventId = 1002, - Level = LogLevel.Information, - Message = "Lock {LockId} rejected")] - static partial void LogLockRejected(ILogger logger, long lockId); - - [LoggerMessage( - EventId = 1003, - Level = LogLevel.Information, - Message = "Lock {LockId} acquired")] - static partial void LogLockAcquired(ILogger logger, long lockId); - - [LoggerMessage( - EventId = 1004, - Level = LogLevel.Information, - Message = "Releasing session lock for {LockId}")] - static partial void LogReleasingLock(ILogger logger, long lockId); -} diff --git a/src/Sa.Data.PostgreSql/Readme.md b/src/Sa.Data.PostgreSql/Readme.md index 63e70695..46c1d088 100644 --- a/src/Sa.Data.PostgreSql/Readme.md +++ b/src/Sa.Data.PostgreSql/Readme.md @@ -1,87 +1,200 @@ -# IPgDataSource +# Sa.Data.PostgreSql -Provides a lightweight (minimal) abstraction for working with PostgreSQL databases in .NET applications. +Лёгкая обёртка над Npgsql для типичных операций с PostgreSQL — без ORM overhead, с поддержкой DI, AOT и минимальными аллокациями. + +## Быстрый старт + +```csharp +// Вариант 1: прямой создание +var dataSource = IPgDataSource.Create("Host=db;Database=mydb;Username=usr;Password=pwd"); + +// Вариант 2: через DI +services.AddSaPostgreSqlDataSource(b => b.WithConnectionString("Host=db;Database=mydb;Username=usr;Password=pwd")); +// или с factory (например, из IConfiguration): +services.AddSaPostgreSqlDataSource(b => b.WithConnectionString(sp => + sp.GetRequiredService().GetConnectionString("Default"))); +``` ## ExecuteNonQuery -Executes an SQL query that does not return data (e.g., INSERT, UPDATE, DELETE) and returns the number of affected rows. +Выполняет SQL-команду, которая не возвращает данные (INSERT / UPDATE / DELETE / DDL), и возвращает число затронутых строк. + +```csharp +// Простой запрос +int affected = await dataSource.ExecuteNonQuery("DELETE FROM sessions WHERE expired = true"); + +// С параметрами +int affected = await dataSource.ExecuteNonQuery(""" + INSERT INTO users (name, age) VALUES (@p0, @p1); + """, [ + new NpgsqlParameter { ParameterName = "p0", Value = "Tom" }, + new NpgsqlParameter { ParameterName = "p1", Value = 18 } + ]); +``` + +## ExecuteScalar / ExecuteScalarTyped + +Возвращает первое значение первой строки результата. `ExecuteScalarTyped` автоматически кастует результат, включая поддержку `Guid`, `DateTime`, `DateTimeOffset` и `DateOnly → DateTime`. ```csharp -var dataSource = new PgDataSource(new PgDataSourceSettings("YourConnectionString")); -int affectedRows = await dataSource.ExecuteNonQuery("SELECT 2"); -Console.WriteLine($"Affected Rows: {affectedRows}"); +// object? overload +object? count = await dataSource.ExecuteScalar("SELECT COUNT(*) FROM users"); + +// Typed overload +int count = await dataSource.ExecuteScalarTyped("SELECT COUNT(*) FROM users"); +long id = await dataSource.ExecuteScalarTyped("SELECT nextval('users_id_seq')"); +Guid tenantId = await dataSource.ExecuteScalarTyped("SELECT tenant_uuid FROM tenants LIMIT 1"); +``` + +## ExecuteReader -var parameters = new[] +Потоковое чтение строк с callback'ом — идеально для обработки больших результатов без загрузки в память. + +```csharp +int processed = 0; +await dataSource.ExecuteReader("SELECT id, name FROM users", (reader, rowIndex) => { - new NpgsqlParameter("p1", "Tom"), - new NpgsqlParameter("p2", 18) -}; + int id = reader.GetInt32(0); + string name = reader.GetString(1); + Console.WriteLine($"{rowIndex}: {id} → {name}"); + processed++; +}); +Console.WriteLine($"Processed {processed} rows"); +``` -int affectedRows = await dataSource.ExecuteNonQuery(""" - CREATE TABLE IF NOT EXISTS users ( - name text, - age int - ); +## ExecuteReaderList - INSERT INTO users (name, age) VALUES (@p1, @p2); - """, parameters); +Читает все строки и собирает их в `List`. + +```csharp +// Простая проекция +var names = await dataSource.ExecuteReaderList( + "SELECT name FROM users ORDER BY name", + reader => reader.GetString(0)); + +// С параметрами +var activeUsers = await dataSource.ExecuteReaderList<(int Id, string Name)>( + """SELECT id, name FROM users WHERE active = @active ORDER BY name""", + reader => (reader.GetInt32(0), reader.GetString(1)), + [new NpgsqlParameter { ParameterName = "active", Value = true }]); +``` + +## ExecuteReaderFirst + +Возвращает первое значение из первого столбца первой строки. Возвращает `default(T)` если результат пуст. + +Поддерживаемые типы: `int`, `long`, `short`, `bool`, `double`, `decimal`, `char`, `string`, `DateTime`, `Guid`, `DateTimeOffset`. + +```csharp +// Вернёт 0 если таблица пуста +int errorCount = await dataSource.ExecuteReaderFirst( + "SELECT COUNT(*) FROM outbox_errors"); -Console.WriteLine($"Affected Rows: {affectedRows}"); +// Guid — работает автоматически +Guid firstTenantId = await dataSource.ExecuteReaderFirst( + "SELECT tenant_id FROM tenants LIMIT 1"); ``` -## ExecuteReader +## ExecuteTransactionAsync -Reading Data +Атомарная транзакция с автоматическим rollback при ошибке. ```csharp -int actual = 0; -await dataSource.ExecuteReader("SELECT 1", (reader, i) => actual = reader.GetInt32(0)); -Console.WriteLine($"Value from Database: {actual}"); +await dataSource.ExecuteTransactionAsync(async (transaction, ct) => +{ + // Все команды внутри используют одну транзакцию + await dataSource.ExecuteNonQuery( + "INSERT INTO accounts (balance) VALUES (0)", ct); + + await dataSource.ExecuteNonQuery( + "INSERT INTO transactions (account_id, amount) VALUES (1, 100)", ct); -// get first value -int errCount = await fixture.DataSource.ExecuteReaderFirst("select count(error_id) from outbox__$error"); + // При успехе — авто-commit +}, IsolationLevel.ReadCommitted, ct); +// При любом исключении — авто-rollback +try +{ + await dataSource.ExecuteTransactionAsync(async (tx, ct) => + { + throw new InvalidOperationException("Oops"); + }, ct); +} +catch (InvalidOperationException) +{ + // Транзакция откатилась автоматически +} ``` ## BeginBinaryImport -Binary Import +Быстрый бинарный импорт данных через COPY — в разы быстрее поштучных INSERT'ов. ```csharp -public async ValueTask BulkWrite(ReadOnlyMemory> messages CancellationToken cancellationToken){ - // Start binary import - ulong result = await dataSource.BeginBinaryImport(sqlTemplate, async (writer, t) => +ulong imported = await dataSource.BeginBinaryImport( + "COPY bulk_data (id, payload, created_at) FROM STDIN BINARY", + async (writer, ct) => { - // Write rows to import - WriteRows(writer, typeCode, messages); - return await writer.CompleteAsync(t); - }, cancellationToken); + foreach (var item in items) + { + writer.StartRow(); + writer.Write(item.Id, NpgsqlDbType.Integer); + writer.Write(item.Payload, NpgsqlDbType.Bytea); + writer.Write(item.CreatedAt.ToUnixTimeMilliseconds(), NpgsqlDbType.Timestamp); + } + return await writer.CompleteAsync(ct); + }, + cancellationToken); + +Console.WriteLine($"Imported {imported} rows"); +``` - return result; -} -private void WriteRows(NpgsqlBinaryImporter writer, ReadOnlyMemory> messages) -{ - foreach (OutboxMessage message in messages.Span) +## PgRetryStrategy + +Повтор попыток с jitter для transient-ошибок Npgsql. + +```csharp +using Sa.Data.PostgreSql; + +// Автоматически повторяет при transient-ошибках (connection reset, timeout и т.п.) +var result = await PgRetryStrategy.ExecuteWithRetry( + async ct => { - // Generate a unique identifier for the message - string id = idGenerator.GenId(message.PartInfo.CreatedAt); - - // Start a new row for writing - writer.StartRow(); - - // Write data to the row - writer.Write(id, NpgsqlDbType.Char); // id - writer.Write(message.PartInfo.TenantId, NpgsqlDbType.Integer); // tenant - writer.Write(message.PartInfo.Part, NpgsqlDbType.Text); // part - - // Serialize and write the payload - using RecyclableMemoryStream stream = streamManager.GetStream(); - serializer.Serialize(stream, message.Payload); - stream.Position = 0; - writer.Write(stream, NpgsqlDbType.Bytea); // payload - writer.Write(stream.Length, NpgsqlDbType.Integer); // payload_size - writer.Write(message.PartInfo.CreatedAt.ToUnixTimeSeconds(), NpgsqlDbType.Bigint); // created_at - } + using var conn = await dataSource.OpenDbConnection(ct); + return await conn.OpenAsync(ct); + }, + retryCount: 5, + initialDelay: 530); +``` + +## DbCommandExtensions + INamePrefixProvider + +Оптимизированный API для добавления параметризованных команд с пред-кэшированными именами параметров (минимальные аллокации). + +```csharp +// Объявите провайдер префиксов +public class UserParams : INamePrefixProvider +{ + public static string[] GetPrefixes() => ["name", "age", "email"]; + public static int MaxIndex => 10; } -``` \ No newline at end of file + +// Используйте — имена генерируются как @name0, @name1, ..., @age0, ... +var cmd = new NpgsqlCommand("SELECT * FROM users WHERE name = @name0 AND age > @age0") + .AddParam("name", "Tom", 0) + .AddParam("age", 18, 0); +``` + +## Сравнение методов + +| Метод | Возврат | Когда использовать | +|---|---|---| +| `ExecuteNonQuery` | `int` (строки) | INSERT / UPDATE / DELETE / DDL | +| `ExecuteScalar` | `object?` | Одно значение, нужна ручная конвертация | +| `ExecuteScalarTyped` | `T` | Одно значение с авто-кастом (Guid, DateTime, DateTimeOffset, DateOnly) | +| `ExecuteReader` | `int` (строки) | Потоковая обработка, много строк | +| `ExecuteReaderList` | `List` | Небольшой результат, нужно собрать всё | +| `ExecuteReaderFirst` | `T` | Одна строка одного столбца | +| `BeginBinaryImport` | `ulong` (строки) | Массовый импорт COPY BINARY | +| `ExecuteTransactionAsync` | `void` | Атомарные операции с rollback | diff --git a/src/Sa.Data.PostgreSql/Setup.cs b/src/Sa.Data.PostgreSql/Setup.cs index 66e5ce5c..ec50b64a 100644 --- a/src/Sa.Data.PostgreSql/Setup.cs +++ b/src/Sa.Data.PostgreSql/Setup.cs @@ -28,7 +28,7 @@ public static IServiceCollection AddSaPostgreSqlDataSource( return new PgDataSource(settings); }); - services.TryAddSingleton(); + return services; } } diff --git a/src/Sa.Data.S3/Utils/XmlStreamReader.cs b/src/Sa.Data.S3/Utils/XmlStreamReader.cs index 13b3792a..fd392dcb 100644 --- a/src/Sa.Data.S3/Utils/XmlStreamReader.cs +++ b/src/Sa.Data.S3/Utils/XmlStreamReader.cs @@ -12,7 +12,9 @@ public static string ReadString(Stream stream, ReadOnlySpan elementName, i : buffer[..written].ToString(); } +#pragma warning disable S3776 private static int ReadTo(Stream stream, ReadOnlySpan elementName, ref Span valueBuffer) +#pragma warning restore S3776 { var expectedIndex = 0; var propertyLength = elementName.Length; diff --git a/src/Sa.Media/PipeReaderExtensions.cs b/src/Sa.Media/PipeReaderExtensions.cs index d18f650d..289819ea 100644 --- a/src/Sa.Media/PipeReaderExtensions.cs +++ b/src/Sa.Media/PipeReaderExtensions.cs @@ -17,14 +17,14 @@ public static async ValueTask SkipAsync(this PipeReader reader, long count if (result.Buffer.IsEmpty && result.IsCompleted) break; // Недостаточно данных - var toConsume = Math.Min(remaining, (long)result.Buffer.Length); + var toConsume = Math.Min(remaining, result.Buffer.Length); var consumed = result.Buffer.GetPosition(toConsume); reader.AdvanceTo(consumed, consumed); remaining -= toConsume; - // Если буфер маленький, но нам нужно больше — продолжаем читать - if ((long)result.Buffer.Length <= toConsume && !result.IsCompleted) - continue; + //// Если буфер маленький, но нам нужно больше — продолжаем читать + //if (result.Buffer.Length <= toConsume && !result.IsCompleted) + // continue; } return count - remaining; } @@ -42,7 +42,7 @@ public static async ValueTask SkipFullSegmentsAsync(this PipeReader reader, long if (result.Buffer.IsEmpty && result.IsCompleted) return; - var toConsume = Math.Min(remaining, (long)result.Buffer.Length); + var toConsume = Math.Min(remaining, result.Buffer.Length); var consumed = result.Buffer.GetPosition(toConsume); // Продвигаем Buffer до consumed, frontier тоже diff --git a/src/Sa.Media/WavHeader.cs b/src/Sa.Media/WavHeader.cs index 581f0abe..6409c82b 100644 --- a/src/Sa.Media/WavHeader.cs +++ b/src/Sa.Media/WavHeader.cs @@ -72,8 +72,10 @@ public void Validate() if (SampleRate == 0) throw new InvalidDataException("Sample rate is zero"); +#pragma warning disable S3236 ArgumentOutOfRangeException.ThrowIfNegativeOrZero(BlockAlign, nameof(BlockAlign)); ArgumentOutOfRangeException.ThrowIfNegative(DataSize, nameof(DataSize)); +#pragma warning restore S3236 } public double GetDurationInSeconds(long? fileSize = default) @@ -122,11 +124,14 @@ public override string ToString() _ => AudioFormat.ToString() }; + var isStereo = IsStereo ? "(Stereo)" : ""; + var stereoOrMono = IsMono ? "(Mono)" : isStereo; + return $$""" [WAV Header] Format: {{format}} - Channels: {{NumChannels}} {{(IsMono ? "(Mono)" : IsStereo ? "(Stereo)" : "")}} + Channels: {{NumChannels}} {{(stereoOrMono)}} Sample Rate: {{SampleRate:N0}} Hz Bit Depth: {{BitsPerSample}}-bit Byte Rate: {{GetBytesPerSecond():N0}} bytes/sec @@ -154,7 +159,7 @@ [WAV Header] long dataEnd = HasDataSize ? dataOffset + DataSize - : (fileSize ?? (long)TimeSpan.MaxValue.TotalSeconds * bytesPerSecond);// throw new InvalidOperationException("fileSize required for streaming data")); + : (fileSize ?? (long)TimeSpan.MaxValue.TotalSeconds * bytesPerSecond); long fromOffset = dataOffset + (long)(range.From.TotalSeconds * bytesPerSecond); diff --git a/src/Sa.Schedule/IJobErrorHandlingBuilder.cs b/src/Sa.Schedule/IJobErrorHandlingBuilder.cs index 175a0bca..391f1a0e 100644 --- a/src/Sa.Schedule/IJobErrorHandlingBuilder.cs +++ b/src/Sa.Schedule/IJobErrorHandlingBuilder.cs @@ -27,13 +27,6 @@ public interface IJobErrorHandlingBuilder /// The current IJobErrorHandlingBuilder instance. IJobErrorHandlingBuilder ThenStopAllJobs(); - /// - /// Specifies that the current job should be stopped if an error occurs. - /// - /// The current IJobErrorHandlingBuilder instance. - [Obsolete("Use ThenAbortJob instead. This method will be removed in a future version.")] - IJobErrorHandlingBuilder ThenStopJob() => ThenAbortJob(); - /// /// Specifies a custom error suppression policy. /// diff --git a/src/Samples/Schedule.Console/Program.cs b/src/Samples/Schedule.Console/Program.cs index 3285c3d1..e2d195ac 100644 --- a/src/Samples/Schedule.Console/Program.cs +++ b/src/Samples/Schedule.Console/Program.cs @@ -23,7 +23,7 @@ builder.AddJob() .EverySeconds(2) .WithName("Some 2") - .ConfigureErrorHandling(c => c.IfErrorRetry(2).ThenStopJob()) + .ConfigureErrorHandling(c => c.IfErrorRetry(2).ThenAbortJob()) ; builder.AddInterceptor(); diff --git a/src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceBinaryImportTests.cs b/src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceBinaryImportTests.cs new file mode 100644 index 00000000..ea4a3b5c --- /dev/null +++ b/src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceBinaryImportTests.cs @@ -0,0 +1,145 @@ +using Npgsql; +using NpgsqlTypes; +using Sa.Data.PostgreSql; +using Sa.Data.PostgreSql.Fixture; +using System.Linq; + +namespace Sa.Data.PostgreSqlTests; + +[Collection(nameof(PgDataSourceFixture))] +public class PgDataSourceBinaryImportTests(PgDataSourceFixture fixture) : IClassFixture +{ + [Fact()] + public async Task BeginBinaryImport_ImportsRows() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS binary_import (id int, name text, active bool); + DELETE FROM binary_import; + """, TestContext.Current.CancellationToken); + + var imported = await fixture.DataSource.BeginBinaryImport( + "COPY binary_import (id, name, active) FROM STDIN BINARY", + async (writer, ct) => + { + for (int i = 1; i <= 50; i++) + { + writer.StartRow(); + writer.Write(i, NpgsqlDbType.Integer); + writer.Write($"item{i}", NpgsqlDbType.Varchar); + writer.Write(i % 2 == 0, NpgsqlDbType.Boolean); + } + return await writer.CompleteAsync(ct); + }, + TestContext.Current.CancellationToken); + + Assert.Equal(50UL, imported); + + var count = (int)(long)(await fixture.DataSource.ExecuteScalar("SELECT COUNT(*) FROM binary_import", TestContext.Current.CancellationToken))!; + Assert.Equal(50, count); + + // verify data integrity + var first = await fixture.DataSource.ExecuteReaderFirst( + "SELECT name FROM binary_import ORDER BY id LIMIT 1", + TestContext.Current.CancellationToken); + Assert.Equal("item1", first); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS binary_import;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task BeginBinaryImport_EmptyImport_ReturnsZero() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS binary_import_empty (id int); + DELETE FROM binary_import_empty; + """, TestContext.Current.CancellationToken); + + var imported = await fixture.DataSource.BeginBinaryImport( + "COPY binary_import_empty (id) FROM STDIN BINARY", + async (writer, ct) => + { + return await writer.CompleteAsync(ct); + }, + TestContext.Current.CancellationToken); + + Assert.Equal(0UL, imported); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS binary_import_empty;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task BeginBinaryImport_GuidAndTimestamp() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS binary_import_types ( + guid_val uuid, + ts_val timestamptz, + payload bytea + ); + DELETE FROM binary_import_types; + """, TestContext.Current.CancellationToken); + + var guid1 = Guid.NewGuid(); + var guid2 = Guid.NewGuid(); + var ts = new DateTimeOffset(2024, 6, 15, 12, 0, 0, TimeSpan.Zero); + var payload = new byte[] { 0xDE, 0xAD, 0xBE, 0xEF }; + + var imported = await fixture.DataSource.BeginBinaryImport( + "COPY binary_import_types (guid_val, ts_val, payload) FROM STDIN BINARY", + async (writer, ct) => + { + writer.StartRow(); + writer.Write(guid1, NpgsqlDbType.Uuid); + writer.Write(ts.ToUnixTimeMilliseconds(), NpgsqlDbType.Timestamp); + writer.Write(payload, NpgsqlDbType.Bytea); + + writer.StartRow(); + writer.Write(guid2, NpgsqlDbType.Uuid); + writer.Write(ts.AddMinutes(1).ToUnixTimeMilliseconds(), NpgsqlDbType.Timestamp); + writer.Write(payload.Reverse().ToArray(), NpgsqlDbType.Bytea); + + return await writer.CompleteAsync(ct); + }, + TestContext.Current.CancellationToken); + + Assert.Equal(2UL, imported); + + var count = (int)(long)(await fixture.DataSource.ExecuteScalar("SELECT COUNT(*) FROM binary_import_types", TestContext.Current.CancellationToken))!; + Assert.Equal(2, count); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS binary_import_types;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task BeginBinaryImport_LargeBatch() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS binary_import_large (seq int, data text); + DELETE FROM binary_import_large; + """, TestContext.Current.CancellationToken); + + const int batchSize = 1000; + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var imported = await fixture.DataSource.BeginBinaryImport( + "COPY binary_import_large (seq, data) FROM STDIN BINARY", + async (writer, ct) => + { + for (int i = 0; i < batchSize; i++) + { + writer.StartRow(); + writer.Write(i, NpgsqlDbType.Integer); + writer.Write($"data-{i}-{new string('x', 100)}", NpgsqlDbType.Text); + } + return await writer.CompleteAsync(ct); + }, + TestContext.Current.CancellationToken); + + sw.Stop(); + + Assert.Equal(batchSize, (int)imported); + Assert.True(sw.ElapsedMilliseconds < 5000, $"Large batch took too long: {sw.ElapsedMilliseconds}ms"); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS binary_import_large;", TestContext.Current.CancellationToken); + } +} diff --git a/src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceReaderFirstTests.cs b/src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceReaderFirstTests.cs new file mode 100644 index 00000000..1a36ae5a --- /dev/null +++ b/src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceReaderFirstTests.cs @@ -0,0 +1,172 @@ +using Npgsql; +using NpgsqlTypes; +using Sa.Data.PostgreSql; +using Sa.Data.PostgreSql.Fixture; + +namespace Sa.Data.PostgreSqlTests; + +[Collection(nameof(PgDataSourceFixture))] +public class PgDataSourceReaderFirstTests(PgDataSourceFixture fixture) : IClassFixture +{ + [Fact()] + public async Task ExecuteReaderFirst_Int() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_first_int (val int); + DELETE FROM reader_first_int; + INSERT INTO reader_first_int (val) VALUES (10), (20), (30); + """, TestContext.Current.CancellationToken); + + var first = await fixture.DataSource.ExecuteReaderFirst( + "SELECT val FROM reader_first_int ORDER BY val LIMIT 1", + TestContext.Current.CancellationToken); + + Assert.Equal(10, first); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS reader_first_int;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteReaderFirst_String() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_first_str (name text); + DELETE FROM reader_first_str; + INSERT INTO reader_first_str (name) VALUES ('Zebra'), ('Apple'), ('Mango'); + """, TestContext.Current.CancellationToken); + + var first = await fixture.DataSource.ExecuteReaderFirst( + "SELECT name FROM reader_first_str ORDER BY name LIMIT 1", + TestContext.Current.CancellationToken); + + Assert.Equal("Apple", first); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS reader_first_str;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteReaderFirst_Guid() + { + var expected = Guid.NewGuid(); + + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_first_guid (guid_val uuid); + DELETE FROM reader_first_guid; + INSERT INTO reader_first_guid (guid_val) VALUES (@g); + """, + [new NpgsqlParameter { ParameterName = "g", Value = expected, NpgsqlDbType = NpgsqlDbType.Uuid }], + TestContext.Current.CancellationToken); + + var actual = await fixture.DataSource.ExecuteReaderFirst( + "SELECT guid_val FROM reader_first_guid", + TestContext.Current.CancellationToken); + + Assert.Equal(expected, actual); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS reader_first_guid;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteReaderFirst_WithParameters() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_first_param (name text, age int); + DELETE FROM reader_first_param; + INSERT INTO reader_first_param (name, age) VALUES ('Tom', 18), ('Jerry', 5); + """, TestContext.Current.CancellationToken); + + var age = await fixture.DataSource.ExecuteReaderFirst( + "SELECT age FROM reader_first_param WHERE name = @name", + [new NpgsqlParameter { ParameterName = "name", Value = "Tom" }], + TestContext.Current.CancellationToken); + + Assert.Equal(18, age); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS reader_first_param;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteReaderFirst_Bool() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_first_bool (flag bool); + DELETE FROM reader_first_bool; + INSERT INTO reader_first_bool (flag) VALUES (true); + """, TestContext.Current.CancellationToken); + + var flag = await fixture.DataSource.ExecuteReaderFirst( + "SELECT flag FROM reader_first_bool", + TestContext.Current.CancellationToken); + + Assert.True(flag); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS reader_first_bool;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteReaderFirst_Double() + { + var expected = 2.71828; + var actual = await fixture.DataSource.ExecuteReaderFirst( + "SELECT 2.71828", + TestContext.Current.CancellationToken); + + Assert.True(Math.Abs(actual - expected) < 0.001); + } + + [Fact()] + public async Task ExecuteReaderFirst_Char() + { + var actual = await fixture.DataSource.ExecuteReaderFirst( + "SELECT 'A'::char", + TestContext.Current.CancellationToken); + + Assert.Equal('A', actual); + } + + [Fact()] + public async Task ExecuteReaderFirst_Decimal() + { + var expected = 19.99m; + var actual = await fixture.DataSource.ExecuteReaderFirst( + "SELECT 19.99::numeric(10,2)", + TestContext.Current.CancellationToken); + + Assert.Equal(expected, actual); + } + + [Fact()] + public async Task ExecuteReaderFirst_CountReturnsZeroWhenEmpty() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_first_empty (val int); + DELETE FROM reader_first_empty; + """, TestContext.Current.CancellationToken); + + var count = await fixture.DataSource.ExecuteReaderFirst( + "SELECT COUNT(*) FROM reader_first_empty", + TestContext.Current.CancellationToken); + + Assert.Equal(0, count); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS reader_first_empty;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteReaderFirst_AggregationFunction() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_first_agg (val int); + DELETE FROM reader_first_agg; + INSERT INTO reader_first_agg (val) VALUES (10), (20), (30); + """, TestContext.Current.CancellationToken); + + var avg = await fixture.DataSource.ExecuteReaderFirst( + "SELECT AVG(val)::double precision FROM reader_first_agg", + TestContext.Current.CancellationToken); + + Assert.True(Math.Abs(avg - 20.0) < 0.001); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS reader_first_agg;", TestContext.Current.CancellationToken); + } +} diff --git a/src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceReaderListTests.cs b/src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceReaderListTests.cs new file mode 100644 index 00000000..b2eb2fde --- /dev/null +++ b/src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceReaderListTests.cs @@ -0,0 +1,141 @@ +using Npgsql; +using NpgsqlTypes; +using Sa.Data.PostgreSql; +using Sa.Data.PostgreSql.Fixture; + +namespace Sa.Data.PostgreSqlTests; + +[Collection(nameof(PgDataSourceFixture))] +public class PgDataSourceReaderListTests(PgDataSourceFixture fixture) : IClassFixture +{ + [Fact()] + public async Task ExecuteReaderList_ReturnsAllRows() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_list_test (id int, name text); + DELETE FROM reader_list_test; + INSERT INTO reader_list_test (id, name) VALUES (1, 'Alice'), (2, 'Bob'), (3, 'Charlie'); + """, TestContext.Current.CancellationToken); + + var names = await fixture.DataSource.ExecuteReaderList( + "SELECT name FROM reader_list_test ORDER BY id", + reader => reader.GetString(0), + TestContext.Current.CancellationToken); + + Assert.Equal(3, names.Count); + Assert.Contains("Alice", names); + Assert.Contains("Bob", names); + Assert.Contains("Charlie", names); + + await fixture.DataSource.ExecuteNonQuery("DELETE FROM reader_list_test;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteReaderList_WithParameters() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_list_params (name text, active bool); + DELETE FROM reader_list_params; + INSERT INTO reader_list_params (name, active) VALUES ('Alice', true), ('Bob', false), ('Charlie', true); + """, TestContext.Current.CancellationToken); + + var activeNames = await fixture.DataSource.ExecuteReaderList( + "SELECT name FROM reader_list_params WHERE active = @active ORDER BY name", + reader => reader.GetString(0), + [new NpgsqlParameter { ParameterName = "active", Value = true }], + TestContext.Current.CancellationToken); + + Assert.Equal(2, activeNames.Count); + Assert.Contains("Alice", activeNames); + Assert.Contains("Charlie", activeNames); + + await fixture.DataSource.ExecuteNonQuery("DELETE FROM reader_list_params;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteReaderList_ReturnsEmpty_ForNoData() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_list_empty (id int); + DELETE FROM reader_list_empty; + """, TestContext.Current.CancellationToken); + + var result = await fixture.DataSource.ExecuteReaderList( + "SELECT id FROM reader_list_empty", + reader => reader.GetInt32(0), + TestContext.Current.CancellationToken); + + Assert.Empty(result); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS reader_list_empty;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteReaderList_TupleProjection() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_list_tuple (id int, name text); + DELETE FROM reader_list_tuple; + INSERT INTO reader_list_tuple (id, name) VALUES (1, 'First'), (2, 'Second'); + """, TestContext.Current.CancellationToken); + + var tuples = await fixture.DataSource.ExecuteReaderList<(int Id, string Name)>( + "SELECT id, name FROM reader_list_tuple ORDER BY id", + reader => (reader.GetInt32(0), reader.GetString(1)), + TestContext.Current.CancellationToken); + + Assert.Equal(2, tuples.Count); + Assert.Equal(1, tuples[0].Id); + Assert.Equal("First", tuples[0].Name); + Assert.Equal(2, tuples[1].Id); + Assert.Equal("Second", tuples[1].Name); + + await fixture.DataSource.ExecuteNonQuery("DELETE FROM reader_list_tuple;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteReaderList_Guid() + { + var guid1 = Guid.NewGuid(); + var guid2 = Guid.NewGuid(); + + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_list_guid (guid_val uuid); + DELETE FROM reader_list_guid; + INSERT INTO reader_list_guid (guid_val) VALUES (@g1), (@g2); + """, + [new NpgsqlParameter { ParameterName = "g1", Value = guid1, NpgsqlDbType = NpgsqlDbType.Uuid }, + new NpgsqlParameter { ParameterName = "g2", Value = guid2, NpgsqlDbType = NpgsqlDbType.Uuid }], + TestContext.Current.CancellationToken); + + var guids = await fixture.DataSource.ExecuteReaderList( + "SELECT guid_val FROM reader_list_guid ORDER BY guid_val", + reader => reader.GetFieldValue(0), + TestContext.Current.CancellationToken); + + Assert.Equal(2, guids.Count); + Assert.Contains(guid1, guids); + Assert.Contains(guid2, guids); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS reader_list_guid;", TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteReaderList_CountMatchesRowCount() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS reader_list_count (val int); + DELETE FROM reader_list_count; + INSERT INTO reader_list_count (val) SELECT generate_series(1, 100); + """, TestContext.Current.CancellationToken); + + var count = await fixture.DataSource.ExecuteReaderList( + "SELECT val FROM reader_list_count ORDER BY val", + reader => reader.GetInt32(0), + TestContext.Current.CancellationToken); + + Assert.Equal(100, count.Count); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS reader_list_count;", TestContext.Current.CancellationToken); + } +} diff --git a/src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceScalarTests.cs b/src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceScalarTests.cs new file mode 100644 index 00000000..0100966a --- /dev/null +++ b/src/Tests/Sa.Data.PostgreSqlTests/PgDataSourceScalarTests.cs @@ -0,0 +1,117 @@ +using Npgsql; +using Sa.Data.PostgreSql.Fixture; + +namespace Sa.Data.PostgreSqlTests; + +[Collection(nameof(PgDataSourceFixture))] +public class PgDataSourceScalarTests(PgDataSourceFixture fixture) : IClassFixture +{ + [Fact()] + public async Task ExecuteScalar_ReturnsFirstValue() + { + var actual = await fixture.DataSource.ExecuteScalar("SELECT 42", TestContext.Current.CancellationToken); + Assert.Equal(42, actual); + } + + [Fact()] + public async Task ExecuteScalar_ReturnsNull_ForEmptyResult() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS empty_table (id int); + DELETE FROM empty_table; + """, TestContext.Current.CancellationToken); + + var actual = await fixture.DataSource.ExecuteScalar("SELECT id FROM empty_table", TestContext.Current.CancellationToken); + Assert.Null(actual); + } + + [Fact()] + public async Task ExecuteScalar_Typed_ReturnsInt() + { + var actual = await fixture.DataSource.ExecuteScalarTyped("SELECT 123", cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(123, actual); + } + + [Fact()] + public async Task ExecuteScalar_Typed_ReturnsString() + { + const string expected = "hello"; + var actual = await fixture.DataSource.ExecuteScalarTyped("SELECT 'hello'", cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(expected, actual); + } + + [Fact()] + public async Task ExecuteScalar_Typed_ReturnsGuid() + { + var expected = Guid.NewGuid(); + var actual = await fixture.DataSource.ExecuteScalarTyped($"SELECT '{expected}'::uuid", cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(expected, actual); + } + + [Fact()] + public async Task ExecuteScalar_Typed_NumericAggregation() + { + await fixture.DataSource.ExecuteNonQuery(""" + INSERT INTO empty_table (id) VALUES (1), (2), (3); + """, cancellationToken: TestContext.Current.CancellationToken); + + var sum = await fixture.DataSource.ExecuteScalarTyped("SELECT SUM(id)::bigint FROM empty_table", cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(6L, sum); + + // cleanup + await fixture.DataSource.ExecuteNonQuery("DELETE FROM empty_table;", cancellationToken: TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteScalar_Typed_WithParameters() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS param_test_table (name text, value int); + DELETE FROM param_test_table; + """, cancellationToken: TestContext.Current.CancellationToken); + + var val = await fixture.DataSource.ExecuteScalarTyped( + """INSERT INTO param_test_table (name, value) VALUES (@p0, @p1) RETURNING value""", + [new NpgsqlParameter { ParameterName = "p0", Value = "test" }, new NpgsqlParameter { ParameterName = "p1", Value = 99 }] + , cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(99, val); + + await fixture.DataSource.ExecuteNonQuery("DELETE FROM param_test_table;", cancellationToken: TestContext.Current.CancellationToken); + } + + [Fact()] + public async Task ExecuteScalar_Typed_Bool() + { + var actual = await fixture.DataSource.ExecuteScalarTyped("SELECT true", cancellationToken: TestContext.Current.CancellationToken); + Assert.True(actual); + } + + [Fact()] + public async Task ExecuteScalar_Typed_Double() + { + var actual = await fixture.DataSource.ExecuteScalarTyped("SELECT 3.14", cancellationToken: TestContext.Current.CancellationToken); + Assert.True(Math.Abs(actual - 3.14) < 0.001); + } + + [Fact()] + public async Task ExecuteScalar_Typed_DateTime() + { + var expected = DateTime.UtcNow.Date; + var actual = await fixture.DataSource.ExecuteScalarTyped("SELECT CURRENT_DATE", cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(expected.Date, actual.Date); + } + + [Fact()] + public async Task ExecuteScalar_Typed_DefaultForEmpty() + { + await fixture.DataSource.ExecuteNonQuery(""" + CREATE TABLE IF NOT EXISTS empty_scalar (val int); + DELETE FROM empty_scalar; + """, cancellationToken: TestContext.Current.CancellationToken); + + var actual = await fixture.DataSource.ExecuteScalar("SELECT val FROM empty_scalar WHERE val > 0", cancellationToken: TestContext.Current.CancellationToken); + Assert.Null(actual); + + await fixture.DataSource.ExecuteNonQuery("DROP TABLE IF EXISTS empty_scalar;", cancellationToken: TestContext.Current.CancellationToken); + } +} diff --git a/src/Tests/Sa.MediaTests/TimeRangeExpanderTests.cs b/src/Tests/Sa.MediaTests/TimeRangeExpanderTests.cs index 92f359ba..2e4a75de 100644 --- a/src/Tests/Sa.MediaTests/TimeRangeExpanderTests.cs +++ b/src/Tests/Sa.MediaTests/TimeRangeExpanderTests.cs @@ -41,12 +41,6 @@ private static void AssertChunksEqual(TimeRange[][] expected, TimeRange[][] actu } } - private static void AssertApproxEqual(long expected, long actual, long tolerance = 1) - { - var diff = Math.Abs(expected - actual); - Assert.True(diff <= tolerance); - } - #endregion From f1294b0e948a9f62c811268cc851cf4fb29cd807 Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 26 Jun 2026 16:58:57 +0300 Subject: [PATCH 13/33] improve part --- README.md | 17 +- src/Sa.Partitional.PostgreSql/ApiReference.md | 160 ++++++++ .../Cache/PartCache.cs | 5 +- .../Cache/PartCacheSettings.cs | 8 + .../Classes/Enumeration.cs | 54 ++- .../Classes/StrOrNum.cs | 95 ++++- .../Cleaning/PartCleanupService.cs | 3 +- .../Configuration/Builder/ISchemaBuilder.cs | 20 + .../Configuration/Builder/ISettingsBuilder.cs | 23 ++ .../Configuration/Builder/ITableBuilder.cs | 84 +++- .../Configuration/Builder/TableBuilder.cs | 2 +- .../Configuration/IPartConfiguration.cs | 33 ++ src/Sa.Partitional.PostgreSql/Guide.md | 363 ++++++++++++++++++ .../Migration/IMigrationService.cs | 26 ++ .../Migration/MigrationJobConstance.cs | 10 + .../Migration/MigrationScheduleSettings.cs | 24 ++ .../Migration/PartMigrationService.cs | 13 +- src/Sa.Partitional.PostgreSql/Part.cs | 12 + .../Partitional/IPartRepository.cs | 65 +++- .../Partitional/PartRepository.cs | 26 +- src/Sa.Partitional.PostgreSql/PgErrorCodes.cs | 34 ++ src/Sa.Partitional.PostgreSql/PgPartBy.cs | 31 ++ src/Sa.Partitional.PostgreSql/Readme.md | 225 ++--------- .../Settings/IPartTableMigrationSupport.cs | 9 + .../Settings/ITableSettings.cs | 36 +- .../Settings/ITableSettingsStorage.cs | 10 + src/Sa.Partitional.PostgreSql/Setup.cs | 13 + .../SqlBuilder/ISqlTableBuilder.cs | 5 + .../SqlBuilder/SqlTableBuilder.cs | 13 + 29 files changed, 1156 insertions(+), 263 deletions(-) create mode 100644 src/Sa.Partitional.PostgreSql/ApiReference.md create mode 100644 src/Sa.Partitional.PostgreSql/Guide.md create mode 100644 src/Sa.Partitional.PostgreSql/PgErrorCodes.cs diff --git a/README.md b/README.md index d2f233b1..a1e45b79 100644 --- a/README.md +++ b/README.md @@ -31,13 +31,16 @@ Designed for implementing the Outbox pattern using PostgreSQL, which is used to ## [Sa.Partitional.PostgreSql](src/Sa.Partitional.PostgreSql) -A library designed for managing table partitioning in PostgreSQL with the aim of improving performance and manageability for large volumes of data. - -- Declaratively describe a partitioned table by time (day, month, year). -- Define partitions based on lists of keys for rows or numbers. -- Set a schedule for migrations to create new partitions. -- Set a schedule for deleting old partitions. -- Manage partitions. +Declarative PostgreSQL table partitioning for .NET 10 — range (day/month/year) and list partitioning with automated migration, cleanup scheduling, and in-memory caching. + +- **Range partitioning** by day, month, or year with automatic timestamp-based naming +- **List partitioning** by string or numeric keys with hierarchical child partitions +- **Fluent builder API** for declaring tables, tuning fillfactor, custom constraints, and migrations +- **Automated migration** — pre-create future partitions as a background job +- **Automated cleanup** — drop old partitions past a configurable retention window +- **In-memory cache** — avoids repeated catalog queries; auto-invalidates on runtime changes +- **StrOrNum** discriminated union for type-safe partition key values +- See the full [Guide](src/Sa.Partitional.PostgreSql/Guide.md) and [API Reference](src/Sa.Partitional.PostgreSql/ApiReference.md). ## [Sa.Schedule](src/Sa.Schedule) diff --git a/src/Sa.Partitional.PostgreSql/ApiReference.md b/src/Sa.Partitional.PostgreSql/ApiReference.md new file mode 100644 index 00000000..1bb46c1e --- /dev/null +++ b/src/Sa.Partitional.PostgreSql/ApiReference.md @@ -0,0 +1,160 @@ +# API Reference + +## Architecture Diagram + +``` +┌──────────────────────────────────────────────────────┐ +│ AddSaPartitional(configure) │ +│ │ +│ ┌─────────────┐ ┌──────────────┐ ┌────────────┐ │ +│ │ Table Builder│ │ Part Cache │ │ Migration │ │ +│ │ (ISettings) │ │ (PartCache) │ │ Schedule │ │ +│ └──────┬──────┘ └──────┬───────┘ └─────┬──────┘ │ +│ │ │ │ │ +│ ┌──────▼────────────────▼────────────────▼──────┐ │ +│ │ IPartitionManager │ │ +│ │ • Migrate() │ │ +│ │ • EnsureParts() │ │ +│ └───────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────┘ + │ │ │ + ▼ ▼ ▼ + ┌───────────┐ ┌────────────┐ ┌──────────────┐ + │ Repository│ │ SQL Builder│ │ Cleanup Job │ + │ (DDL exec)│ │ (template) │ │ (DROP parts) │ + └───────────┘ └────────────┘ └──────────────┘ +``` + +## Key Interfaces + +### `IPartitionManager` + +Entry point for programmatic partition management. Registered as singleton via DI. + +```csharp +public interface IPartitionManager +{ + /// Create any missing partitions across all configured tables. + Task Migrate(CancellationToken cancellationToken = default); + + /// Create partitions only for the specified dates. + Task Migrate(DateTimeOffset[] dates, CancellationToken cancellationToken = default); + + /// + /// Ensure a specific partition exists; create it if absent. + /// Checks the in-memory cache first, falls back to direct DDL if needed. + /// + Task EnsureParts( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default); +} +``` + +**Usage:** + +```csharp +public class MyService +{ + private readonly IPartitionManager _parts; + + public MyService(IPartitionManager parts) => _parts = parts; + + public async Task OnDataReceived(string tenant, DateTime now) + { + // Ensures partition for this tenant/date exists before writing data + await _parts.EnsureParts("events", now, new StrOrNum[] { tenant }); + } +} +``` + +### `IPartRepository` + +Low-level DDL executor — creates, queries, and drops partitions directly against PostgreSQL. + +```csharp +public interface IPartRepository +{ + /// Creates a single partition (child table) for the given table, date, and values. + Task CreatePart( + string tableName, DateTimeOffset date, StrOrNum[] partValues, + CancellationToken cancellationToken = default); + + /// Ensures all tables have partitions covering each date in . + Task Migrate(DateTimeOffset[] dates, CancellationToken cancellationToken = default); + + /// Same as above, but resolves list-partition values dynamically via . + Task Migrate( + DateTimeOffset[] dates, + Func> resolve, + CancellationToken cancellationToken = default); + + /// Retrieves all range partitions starting from . + Task> GetPartsFromDate( + string tableName, DateTimeOffset fromDate, + CancellationToken cancellationToken = default); + + /// Retrieves all range partitions up to and including . + Task> GetPartsToDate( + string tableName, DateTimeOffset toDate, + CancellationToken cancellationToken = default); + + /// Drops all partitions whose FromDate ≤ . + Task DropPartsToDate( + string tableName, DateTimeOffset toDate, + CancellationToken cancellationToken = default); +} +``` + +### `IMigrationService` + +Automated pre-creation of future partitions. Runs on schedule or manually. + +```csharp +public interface IMigrationService +{ + /// Triggered after a successful migration cycle completes. + CancellationToken OnMigrated { get; } + + /// Migrates all tables for the configured forward-days window. + Task Migrate(CancellationToken cancellationToken = default); + + /// Migrates only for the specified dates. + Task Migrate(DateTimeOffset[] dates, CancellationToken cancellationToken = default); + + /// + /// Waits synchronously (up to timeout) for an in-flight migration to complete. + /// Returns immediately if migration has already finished. + /// + Task WaitMigration(TimeSpan timeout, CancellationToken cancellationToken = default); +} +``` + +### `IPartCleanupService` + +Automatic removal of old partitions past the retention window. + +```csharp +public interface IPartCleanupService +{ + /// Drops partitions older than the configured retention period. + Task Clean(CancellationToken cancellationToken); + + /// Drops all partitions with FromDate ≤ . + Task Clean(DateTimeOffset toDate, CancellationToken cancellationToken); +} +``` + +## Internal Components + +| Component | File | Role | +|---|---|---| +| `PartitionManager` | `PartitionManager.cs` | Orchestrates cache + migration for `IPartitionManager` | +| `PartCache` | `Cache/PartCache.cs` | In-memory cache of partition metadata per table | +| `PartMigrationService` | `Migration/PartMigrationService.cs` | Scheduled migration executor with dedup guard | +| `PartCleanupService` | `Cleaning/PartCleanupService.cs` | Drops old partitions based on retention settings | +| `MigrationJob` | `Migration/MigrationJob.cs` | `IJob` wrapper for scheduled migrations | +| `PartCleanupJob` | `Cleaning/PartCleanupJob.cs` | `IJob` wrapper for scheduled cleanup | +| `SqlBuilder` | `SqlBuilder/SqlBuilder.cs` | Generates DDL templates from `ITableSettings` | +| `PartRepository` | `Partitional/PartRepository.cs` | Executes DDL via Npgsql | diff --git a/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs b/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs index cdd46239..521eac3a 100644 --- a/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs +++ b/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs @@ -9,8 +9,7 @@ internal sealed class PartCache( IPartRepository repository , ISqlBuilder sqlBuilder , PartCacheSettings settings - , TimeProvider? timeProvider = null -) : IPartCache + , TimeProvider? timeProvider = null) : IPartCache { private readonly ConcurrentDictionary>> _cache = new(); @@ -44,7 +43,7 @@ private async Task> SelectPartsInDb(string tableName, Canc List list = await repository.GetPartsFromDate(tableName, from, cancellationToken); return list; } - catch (Npgsql.PostgresException ex) when (ex.SqlState == Npgsql.PostgresErrorCodes.UndefinedTable) + catch (Npgsql.PostgresException ex) when (PgErrorCodes.IsUndefinedTable(ex)) { return []; } diff --git a/src/Sa.Partitional.PostgreSql/Cache/PartCacheSettings.cs b/src/Sa.Partitional.PostgreSql/Cache/PartCacheSettings.cs index d4e0e9a8..dc160230 100644 --- a/src/Sa.Partitional.PostgreSql/Cache/PartCacheSettings.cs +++ b/src/Sa.Partitional.PostgreSql/Cache/PartCacheSettings.cs @@ -1,6 +1,14 @@ namespace Sa.Partitional.PostgreSql; +/// +/// Settings that control the partition cache window. +/// The cache preloads partition metadata for a configurable period into the future. +/// public sealed class PartCacheSettings { + /// + /// Gets or sets how far ahead (from the current time) the cache should preload partitions. + /// Default is 1 day. + /// public TimeSpan CachedFromDate { get; set; } = TimeSpan.FromDays(1); } diff --git a/src/Sa.Partitional.PostgreSql/Classes/Enumeration.cs b/src/Sa.Partitional.PostgreSql/Classes/Enumeration.cs index 520b8fe9..aac4ca51 100644 --- a/src/Sa.Partitional.PostgreSql/Classes/Enumeration.cs +++ b/src/Sa.Partitional.PostgreSql/Classes/Enumeration.cs @@ -5,9 +5,15 @@ namespace Sa.Partitional.PostgreSql.Classes; /// -/// https://josef.codes/enumeration-class-in-c-sharp-using-records/ +/// A base record that implements a type-safe enumeration pattern using reflection-discovered static fields. +/// Inspired by Josef Bihl's article. /// -/// +/// +/// The derived enumeration type. Enforced via the recursive generic constraint where T : Enumeration{T}. +/// All enum variants must be declared as public static readonly T fields. +/// +/// Unique integer identifier used for database storage and comparison. +/// Human-readable display name. public record Enumeration<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicFields)] T>(int Id, string Name) : IComparable where T : Enumeration { @@ -32,35 +38,79 @@ public record Enumeration<[DynamicallyAccessedMembers(DynamicallyAccessedMemberT return items; }); + /// + /// Returns all registered enumeration values discovered via reflection. + /// [DebuggerStepThrough] public static IEnumerable GetAll() => AllItems.Value.Values; + /// + /// Computes the absolute difference between the integer IDs of two enumeration values. + /// Useful for determining adjacency (difference of 1) or distance between variants. + /// + /// The first enumeration value. + /// The second enumeration value. + /// The absolute difference between their values. [DebuggerStepThrough] public static int DiffId(Enumeration firstId, Enumeration secondId) => Math.Abs(firstId.Id - secondId.Id); + /// + /// Looks up an enumeration value by its integer . + /// Throws when the id is not found. + /// + /// The integer identifier to look up. + /// The matching enumeration value. + /// Thrown when no variant has the specified id. [DebuggerStepThrough] public static T FromId(int id) => TryFromId(id, out var matchingItem) ? matchingItem : throw new InvalidOperationException($"'{id}' is not a valid value in {typeof(T)}"); + /// + /// Looks up an enumeration value by its display . + /// Throws when the name is not found. + /// + /// The display name to look up. + /// The matching enumeration value. + /// Thrown when no variant has the specified name. [DebuggerStepThrough] public static T FromName(string name) => (TryFromName(name, out var matchingItem)) ? matchingItem : throw new InvalidOperationException($"'{name}' is not a valid display name in {typeof(T)}"); + /// + /// Attempts to look up an enumeration value by its display . + /// + /// The display name to look up. + /// The matching enumeration value, or null if not found. + /// true if a matching variant was found; otherwise false. [DebuggerStepThrough] public static bool TryFromName(string name, [MaybeNullWhen(false)] out T item) => AllItemsByName.Value.TryGetValue(name, out item); + /// + /// Attempts to look up an enumeration value by its integer . + /// + /// The integer identifier to look up. + /// The matching enumeration value, or null if not found. + /// true if a matching variant was found; otherwise false. [DebuggerStepThrough] public static bool TryFromId(int id, [MaybeNullWhen(false)] out T item) => AllItems.Value.TryGetValue(id, out item); + /// + /// Compares this instance to another enumeration value by their integer . + /// + /// The other enumeration value to compare with. + /// A signed integer indicating the relative order. [DebuggerStepThrough] public int CompareTo(T? other) => Id.CompareTo(other!.Id); + /// + /// Returns the display of this enumeration value. + /// public override string ToString() => Name; } diff --git a/src/Sa.Partitional.PostgreSql/Classes/StrOrNum.cs b/src/Sa.Partitional.PostgreSql/Classes/StrOrNum.cs index a731e6fb..aa9029d5 100644 --- a/src/Sa.Partitional.PostgreSql/Classes/StrOrNum.cs +++ b/src/Sa.Partitional.PostgreSql/Classes/StrOrNum.cs @@ -4,46 +4,95 @@ namespace Sa.Partitional.PostgreSql.Classes; -/// -///StrOrNum +/// +/// A discriminated union that represents either a (string) or a (64-bit integer). +/// Used throughout Sa.Partitional.PostgreSql for partition key values that may be either text labels or numeric identifiers. +/// /// /// -/// StrOrNum val = 10; -/// StrOrNum val_1 = "привет"; +/// StrOrNum val = 10; +/// StrOrNum val_1 = "hello"; /// string v = val.Match( /// onChoiceNum: item => $"long: {item}", /// onChoiceStr: item => $"string: {item}" /// ); /// /// -/// -/// +/// [JsonConverter(typeof(StrOrNumConverter))] public abstract record StrOrNum { + /// + /// String variant of the discriminated union. + /// + /// The string value. public record ChoiceStr(string Item) : StrOrNum { public override string ToString() => Item; } + /// + /// Numeric variant of the discriminated union. + /// + /// The value. public record ChoiceNum(long Item) : StrOrNum { public override string ToString() => $"{Item}"; } + /// + /// Dispatches to either or depending on the active variant. + /// + /// The return type shared by both branches. + /// Callback invoked when this is a . + /// Callback invoked when this is a . + /// The result of the invoked callback. public U Match(Func onChoiceStr, Func onChoiceNum) => Match(onChoiceStr, onChoiceNum, this); + /// + /// Implicitly converts a to a wrapping . + /// public static implicit operator StrOrNum(string item) => new ChoiceStr(item); + /// + /// Implicitly converts an to a wrapping . + /// public static implicit operator StrOrNum(int item) => new ChoiceNum(item); + + /// + /// Implicitly converts a to a wrapping . + /// public static implicit operator StrOrNum(long item) => new ChoiceNum(item); + + /// + /// Implicitly converts a to a wrapping . + /// public static implicit operator StrOrNum(short item) => new ChoiceNum(item); + /// + /// Explicitly extracts the underlying from a , + /// or parses a back to its string representation. + /// public static explicit operator string(StrOrNum choice) => choice.Match(c1 => c1, c2 => c2.ToString()); + + /// + /// Explicitly extracts the underlying from a , + /// or attempts to parse a as a number (returns 0 on failure). + /// public static explicit operator long(StrOrNum choice) => choice.Match(c1 => StrToLong(c1) ?? 0, c2 => c2); + + /// + /// Explicitly extracts the underlying from a , + /// or attempts to parse a as a number (returns 0 on failure). + /// public static explicit operator int(StrOrNum choice) => choice.Match(c1 => StrToInt(c1) ?? 0, c2 => (int)c2); + + /// + /// Explicitly extracts the underlying from a , + /// or attempts to parse a as a number (returns 0 on failure). + /// public static explicit operator short(StrOrNum choice) => choice.Match(c1 => StrToShort(c1) ?? 0, c2 => (short)c2); private static U Match(Func onChoiceStr, Func onChoiceNum, StrOrNum choice) @@ -58,10 +107,24 @@ private static U Match(Func onChoiceStr, Func onChoiceNum return result; } + /// + /// Returns the contained value as a human-readable string. + /// public override string ToString() => Match(str => str, num => $"{num}"); + /// + /// Returns a formatted serialisation string prefixed with the variant kind + /// (s:<value> for string, n:<value> for number). + /// This format is used by and the JSON converter. + /// public string ToFmtString() => Match(str => $"s:{str}", num => $"n:{num}"); + /// + /// Parses a formatted string produced by back into a . + /// Strings without a prefix are treated as . + /// + /// The formatted input string. + /// A instance matching the original value. public static StrOrNum FromFmtStr(string? fmtInput) { if (string.IsNullOrEmpty(fmtInput)) return new ChoiceStr(string.Empty); @@ -82,18 +145,34 @@ public static StrOrNum FromFmtStr(string? fmtInput) private StrOrNum() { } - static int? StrToInt(ReadOnlySpan str) => int.TryParse(str, CultureInfo.InvariantCulture, out int result) ? result : null; - static short? StrToShort(ReadOnlySpan str) => short.TryParse(str, CultureInfo.InvariantCulture, out short result) ? result : null; + private static int? StrToInt(ReadOnlySpan str) => int.TryParse(str, CultureInfo.InvariantCulture, out int result) ? result : null; + private static short? StrToShort(ReadOnlySpan str) => short.TryParse(str, CultureInfo.InvariantCulture, out short result) ? result : null; + + /// + /// Safely parses a of characters into a . + /// Uses to avoid culture-dependent parsing issues. + /// + /// The character span to parse. + /// The parsed , or null if parsing fails. public static long? StrToLong(ReadOnlySpan str) => long.TryParse(str, CultureInfo.InvariantCulture, out long result) ? result : null; } +/// +/// Serialises to/from JSON using the formatted s:/n: protocol. +/// public class StrOrNumConverter : JsonConverter { + /// + /// Reads a JSON string and deserialises it into a via . + /// public override StrOrNum Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => StrOrNum.FromFmtStr(reader.GetString()); + /// + /// Writes a as a JSON string using . + /// public override void Write(Utf8JsonWriter writer, StrOrNum value, JsonSerializerOptions options) => writer.WriteStringValue(value.ToFmtString()); } diff --git a/src/Sa.Partitional.PostgreSql/Cleaning/PartCleanupService.cs b/src/Sa.Partitional.PostgreSql/Cleaning/PartCleanupService.cs index dc324f45..817721b2 100644 --- a/src/Sa.Partitional.PostgreSql/Cleaning/PartCleanupService.cs +++ b/src/Sa.Partitional.PostgreSql/Cleaning/PartCleanupService.cs @@ -4,8 +4,7 @@ internal sealed class PartCleanupService( IPartRepository repository , PartCleanupScheduleSettings settings , ISqlBuilder sqlBuilder - , TimeProvider? timeProvider = null -) : IPartCleanupService + , TimeProvider? timeProvider = null) : IPartCleanupService { public async Task Clean(DateTimeOffset toDate, CancellationToken cancellationToken) { diff --git a/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISchemaBuilder.cs b/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISchemaBuilder.cs index d6fab81f..59d96585 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISchemaBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISchemaBuilder.cs @@ -1,8 +1,28 @@ namespace Sa.Partitional.PostgreSql; +/// +/// Builder for declaring one or more database schemas and the tables they contain. +/// public interface ISchemaBuilder { + /// + /// Creates a new table definition builder using the schema's default column set (an auto-generated id column). + /// + /// The logical table name (without schema prefix). + /// A fluent for further configuration. ITableBuilder CreateTable(string tableName); + + /// + /// Creates a new table definition builder with explicitly provided SQL field definitions. + /// + /// The logical table name (without schema prefix). + /// Raw SQL column definitions, e.g. "created_at timestamptz NOT NULL". + /// A fluent for further configuration. ITableBuilder AddTable(string tableName, params string[] sqlFields); + + /// + /// Validates and materialises all configured tables into instances. + /// + /// An array of immutable table settings ready for runtime use. ITableSettings[] Build(); } diff --git a/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISettingsBuilder.cs b/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISettingsBuilder.cs index cdf86766..32c7a642 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISettingsBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISettingsBuilder.cs @@ -1,11 +1,34 @@ namespace Sa.Partitional.PostgreSql; +/// +/// Top-level builder for partitioned-table configuration. Allows declaring schemas (default + named) and their tables. +/// public interface ISettingsBuilder { + /// + /// Gets the name of the default schema that is applied to tables without an explicit schema. + /// string DefaultSchema { get; } + /// + /// Adds a schema using the default column set (id bigint PRIMARY KEY). + /// + /// An action that configures tables within this schema. + /// The same for chaining. ISettingsBuilder AddSchema(Action schemaBuilder); + + /// + /// Adds a named schema using the default column set. + /// + /// The PostgreSQL schema name. + /// An action that configures tables within this schema. + /// The same for chaining. ISettingsBuilder AddSchema(string schemaName, Action schemaBuilder); + + /// + /// Validates and materialises all configured schemas and tables into an immutable . + /// + /// The settings storage ready for registration in the DI container. ITableSettingsStorage Build(); } diff --git a/src/Sa.Partitional.PostgreSql/Configuration/Builder/ITableBuilder.cs b/src/Sa.Partitional.PostgreSql/Configuration/Builder/ITableBuilder.cs index 3572f731..b1d4d349 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/Builder/ITableBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/Builder/ITableBuilder.cs @@ -2,31 +2,113 @@ namespace Sa.Partitional.PostgreSql; +/// +/// Fluent builder for configuring a single partitioned PostgreSQL table. +/// Chains method calls to declare fields, partitioning strategy, migration behaviour, and tuning knobs. +/// public interface ITableBuilder { + /// + /// Appends raw SQL field definitions to the table (e.g. "created_at timestamptz NOT NULL"). + /// + /// One or more column definitions. + /// The same for chaining. ITableBuilder AddFields(params string[] sqlFields); + + /// + /// Configures the table for list partitioning on the specified columns. + /// All listed fields must share the same type and participate in partition key resolution. + /// + /// Column names used as partition keys. + /// The same for chaining. ITableBuilder PartByList(params string[] fieldNames); + + /// + /// Configures the table for range partitioning using a timestamp/timestamptz column. + /// + /// The partitioning granularity — day, month, or year. + /// + /// The column to partition on. When null, the first timestamptz-typed column is used automatically. + /// + /// The same for chaining. ITableBuilder PartByRange(PgPartBy partBy, string? timestampFieldName = null); + /// + /// Overrides the auto-detected timestamp field name used for range partitioning. + /// + /// The column name to use as the partition key. + /// The same for chaining. ITableBuilder TimestampAs(string timestampFieldName); + /// + /// Sets the separator character used between schema and table names in generated SQL (default: _). + /// + /// The separator character(s). + /// The same for chaining. ITableBuilder WithPartSeparator(string partSeparator); + + /// + /// Sets the fillfactor storage parameter for both root and child tables. + /// Lower values leave free space for future HOT updates or dynamic partition growth. + /// + /// An integer between 1 and 100. + /// The same for chaining. ITableBuilder WithFillFactor(int fillFactor); + /// + /// Sets the postfix appended to child/partition table names (default: __part). + /// + /// The suffix string. + /// The same for chaining. ITableBuilder WithPartTablePostfix(string postfix); + /// + /// Registers a callback that produces extra SQL to run after the root-table CREATE TABLE statement. + /// + /// A factory producing the SQL fragment. + /// The same for chaining. ITableBuilder AddPostSql(Func postSql); - ITableBuilder AddConstraintPkSql(Func pkSql); + /// + /// Registers a callback that produces custom CHECK / PRIMARY KEY constraint SQL. + /// + /// A factory producing the constraint definition. + /// The same for chaining. + ITableBuilder AddConstraintPkSql(Func pkSql); + /// + /// Finalises the builder and returns an immutable snapshot. + /// + /// The validated table settings. ITableSettings Build(); + /// + /// Attaches a custom migration provider that supplies list-partition values at runtime. + /// + /// The migration support implementation. + /// The same for chaining. ITableBuilder AddMigration(IPartTableMigrationSupport migrationSupport); + /// + /// Attaches a lazy migration callback that resolves partition values asynchronously. + /// + /// A function that returns partition values when triggered. + /// The same for chaining. ITableBuilder AddMigration(Func> getPartValues); + /// + /// Declares static list-partition values to create eagerly at startup. + /// + /// One or more partition values (strings or numbers). + /// The same for chaining. ITableBuilder AddMigration(params StrOrNum[] partValues); + /// + /// Declares a parent-child hierarchy of list-partition values. + /// + /// The parent partition value. + /// Child partition values nested under the parent. + /// The same for chaining. ITableBuilder AddMigration(StrOrNum parent, StrOrNum[] childs) { foreach (StrOrNum child in childs) AddMigration(parent, child); diff --git a/src/Sa.Partitional.PostgreSql/Configuration/Builder/TableBuilder.cs b/src/Sa.Partitional.PostgreSql/Configuration/Builder/TableBuilder.cs index bba5ae9c..bbe14be3 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/Builder/TableBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/Builder/TableBuilder.cs @@ -69,7 +69,7 @@ public ITableBuilder WithPartSeparator(string partSeparator) public ITableBuilder WithFillFactor(int fillFactor) { - _fillFactor = Math.Max(100, fillFactor); + _fillFactor = Math.Clamp(fillFactor, 10, 100); return this; } diff --git a/src/Sa.Partitional.PostgreSql/Configuration/IPartConfiguration.cs b/src/Sa.Partitional.PostgreSql/Configuration/IPartConfiguration.cs index 67888699..a0fe5fc1 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/IPartConfiguration.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/IPartConfiguration.cs @@ -2,12 +2,45 @@ namespace Sa.Partitional.PostgreSql; +/// +/// Fluent configuration surface returned by . +/// Use this interface to optionally wire up partitioned tables, caching, migration scheduling, cleanup scheduling, and the data source. +/// public interface IPartConfiguration { + /// + /// Registers the partitioned-table schema definition. + /// + /// An action that receives an for declaring schemas and tables. + /// The same for chaining. IPartConfiguration AddPartTables(Action configure); + + /// + /// Enables the partition cache with optional custom settings. + /// When is null, the default cache window (1 day ahead) is used. + /// + /// Optional action to tweak . + /// The same for chaining. IPartConfiguration AddPartCache(Action? configure = null); + + /// + /// Configures the automated partition-migration schedule (pre-creates future partitions). + /// + /// Optional action to tweak . + /// The same for chaining. IPartConfiguration AddPartMigrationSchedule(Action? configure = null); + + /// + /// Configures the automated partition-cleanup schedule (drops old partitions past the retention window). + /// + /// Optional action to tweak . + /// The same for chaining. IPartConfiguration AddPartCleanupSchedule(Action? configure = null); + /// + /// Configures the PostgreSQL data source via . + /// + /// Optional action to set connection strings, pooling, retries, etc. + /// The same for chaining. IPartConfiguration AddDataSource(Action? configure = null); } diff --git a/src/Sa.Partitional.PostgreSql/Guide.md b/src/Sa.Partitional.PostgreSql/Guide.md new file mode 100644 index 00000000..fedef84f --- /dev/null +++ b/src/Sa.Partitional.PostgreSql/Guide.md @@ -0,0 +1,363 @@ +# Sa.Partitional.PostgreSql — Guide + +Detailed usage guide for configuring, tuning, and understanding Sa.Partitional.PostgreSql. + +--- + +## Detailed Setup + +### Range-Partitioned Table + +Simplest case: a single table partitioned by day: + +```csharp +builder.Services.AddSaPartitional((sp, builder) => +{ + builder.AddSchema("public", schema => + { + schema.CreateTable("events") + .PartByRange(PgPartBy.Day) + .WithFillFactor(90); + }); +}) +.AddPartMigrationSchedule((sp, opts) => +{ + opts.AsBackgroundJob = true; + opts.ForwardDays = 2; +}); +``` + +This generates: +- `events` — root table (range-partitioned on `created_at`) +- `events_y2026m06d26`, `events_y2026m06d27`, … — leaf partitions +- `events__part$` — metadata tracking table + +### List + Hierarchical Partitioning + +More complex scenario: list-partitioned root with range-partitioned children: + +```csharp +builder.Services.AddSaPartitional((sp, builder) => +{ + builder.AddSchema("public", schema => + { + schema.AddTable("customer", + "id uuid DEFAULT gen_random_uuid()", + "country text NOT NULL", + "city text NOT NULL") + .WithPartSeparator("_") + .PartByList("country", "city") + .AddMigration("RU", "Moscow", "Samara") + .AddMigration("USA", "Alabama", "New York") + .AddMigration("FR", "Paris", "Lyon"); + }); +}) +.AddPartMigrationSchedule((sp, opts) => +{ + opts.AsBackgroundJob = true; + opts.ForwardDays = 2; +}) +.AddPartCleanupSchedule((sp, opts) => +{ + opts.AsBackgroundJob = true; + opts.DropPartsAfterRetention = TimeSpan.FromDays(30); +}); +``` + +This generates a multi-level hierarchy: +``` +customer (root, LIST on country) +├── customer_FR (LIST on city) +│ ├── customer_FR_Bordeaux (RANGE on created_at) +│ │ ├── customer_FR_Bordeaux_y2026m06d26 +│ │ └── customer_FR_Bordeaux_y2026m06d27 +│ ├── customer_FR_Lyon_y2026m06d26 +│ └── customer_FR_Paris_y2026m06d26 +├── customer_RU (LIST on city) +│ ├── customer_RU_Moscow_y2026m06d26 +│ └── customer_RU_Samara_y2026m06d26 +└── customer_USA (...) +``` + +--- + +## Configuration Options + +### Migration Schedule (`MigrationScheduleSettings`) + +| Property | Default | Description | +|---|---|---| +| `ForwardDays` | `2` | How many days ahead to pre-create partitions on each run | +| `AsBackgroundJob` | `false` | Run as a hosted background service | +| `ExecutionInterval` | `4h + jitter` | Interval between runs (adds 1–59 min random jitter) | +| `WaitMigrationTimeout` | `3s` | Max wait time when two callers race for the same migration | +| `MigrationJobName` | `"Migration job"` | Display name in the job scheduler | + +### Cleanup Schedule (`PartCleanupScheduleSettings`) + +| Property | Default | Description | +|---|---|---| +| `DropPartsAfterRetention` | `30 days` | Partitions older than this are dropped | +| `AsBackgroundJob` | `false` | Run as a hosted background service | +| `ExecutionInterval` | `4h + jitter` | Interval between cleanup runs | +| `InitialDelay` | `1 min` | Delay before first execution | + +### Cache Settings (`PartCacheSettings`) + +| Property | Default | Description | +|---|---|---| +| `CachedFromDate` | `1 day` | How far ahead from now the cache preloads partition metadata | + +The cache avoids repeated catalog queries (`pg_class`, `pg_partitioned_table`). When a partition is created at runtime via `EnsureParts`, the cache is invalidated and reloaded automatically. + +--- + +## Fluent Table Builder + +The `ITableBuilder` interface supports a fluent DSL: + +```csharp +schema.CreateTable("orders") + // 1. Choose partitioning strategy + .PartByRange(PgPartBy.Month) // range by month (default granularity) + // or + .PartByList("tenant_id", "region") // list by column values + + // 2. Override timestamp column (auto-detected by default) + .TimestampAs("shipped_at") + + // 3. Tuning knobs + .WithFillFactor(85) // WITH (fillfactor = 85) for HOT updates + .WithPartSeparator("_") // separator in partition names (default: _) + .WithPartTablePostfix("__part") // child table suffix (default: __part) + + // 4. Custom SQL hooks + .AddPostSql(() => "INCLUDE (extra_column)") + .AddConstraintPkSql(() => $"CONSTRAINT pk_orders PRIMARY KEY (id, tenant_id, shipped_at)") + + // 5. Declare partition migrations + .AddMigration(new StrOrNum[] { "US", "EU", "APAC" }) // static values + .AddMigration(myMigrationProvider) // IPartTableMigrationSupport + .AddMigration(async ct => // lazy async resolver + { + var rows = await QueryDb(ct); + return rows.Select(r => new StrOrNum[] { r.Key }).ToArray(); + }) + + // 6. Finalise + .Build(); +``` + +### Migration Variants + +Three ways to declare which partitions to create: + +| Variant | Signature | When to use | +|---|---|---| +| **Static** | `.AddMigration(params StrOrNum[] partValues)` | Fixed set known at compile time | +| **Parent+Children** | `.AddMigration(StrOrNum parent, StrOrNum[] childs)` | Hierarchical list partitions | +| **Dynamic** | `.AddMigration(Func> getPartValues)` | Values come from another table / API | +| **Interface** | `.AddMigration(IPartTableMigrationSupport support)` | Reusable migration provider class | + +--- + +## Type-Safe Partition Keys: `StrOrNum` + +Partition values may be strings (e.g. country codes) or numbers (e.g. tenant IDs). The `StrOrNum` discriminated union handles both: + +```csharp +// Implicit conversion from string or numeric types +StrOrNum country = "RU"; // ChoiceStr +StrOrNum tenant = 42L; // ChoiceNum + +// Pattern-match on the active variant +string description = tenant.Match( + onChoiceStr: s => $"text: {s}", + onChoiceNum: n => $"numeric: {n}" +); + +// Round-trip serialization (used by JSON converter) +StrOrNum parsed = StrOrNum.FromFmtStr("s:hello"); // ChoiceStr +StrOrNum number = StrOrNum.FromFmtStr("n:123"); // ChoiceNum + +// Parse raw spans safely (culture-independent) +long? value = StrOrNum.StrToLong("42".AsSpan()); // 42 +``` + +### JSON Serialization + +`StrOrNumConverter` serialises values using a prefixed format: +- Strings → `"s:value"` +- Numbers → `"n:123"` + +This ensures type fidelity across JSON round-trips. + +--- + +## Partition Naming Convention + +PostgreSQL limits table names to **63 characters**. To stay within this limit, every partition name ends with an `int64` Unix-timestamp segment: + +| Strategy | Timestamp Format | Example | +|---|---|---| +| Day | `yYYYYmmDD` | `y2026m06d26` | +| Month | `yYYYYmm` | `y2026m06` | +| Year | `yYYYY` | `y2026` | + +Full partition name pattern: `{schema}{separator}{tableName}{postfix}{separator}{key}_{timestamp}` + +For example: `public.customer__RU_Yokohama_y2026m06d26` + +### Name Length Limits + +When combined, the full name must fit within 63 characters: + +``` +public._customer__RU_Yokohama_y2026m06d26 + ^6 _^8 ^^4 ^^8 ^^^^^^^^^^ + 6 + 1 + 8 + 1 + 4 + 1 + 8 + 1 + 10 = 40 chars ✓ +``` + +If your keys are long, consider reducing `WithPartSeparator` length or using shorter key values. + +--- + +## Date Ranges + +Each partition covers an inclusive-exclusive `[from, to)` interval computed from the `PgPartBy` strategy: + +``` +Day: [2026-06-26 00:00 UTC, 2026-06-27 00:00 UTC) +Month: [2026-06-01 00:00 UTC, 2026-07-01 00:00 UTC) +Year: [2026-01-01 00:00 UTC, 2027-01-01 00:00 UTC) +``` + +All computations use UTC. The `PgPartBy` record stores three delegates: +- `GetRange` — computes the `LimSection` for a given date +- `Fmt` — formats a date into a partition name string +- `ParseFmt` — parses a partition name back into a `DateTimeOffset` + +--- + +## Generated DDL Example + +Given this configuration: + +```csharp +schema.AddTable("events", + "tenant_id text NOT NULL", + "created_at timestamptz NOT NULL") + .PartByList("tenant_id") + .PartByRange(PgPartBy.Day, "created_at") + .AddMigration("analytics", "api", "web"); +``` + +The library generates: + +```sql +-- Root table (list-partitioned on tenant_id) +CREATE TABLE public.events ( + id uuid DEFAULT gen_random_uuid(), + tenant_id text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT pk_events PRIMARY KEY (id, tenant_id, created_at) +) PARTITION BY LIST (tenant_id); + +-- First-level children (list partition by tenant) +CREATE TABLE public."events_analytics" PARTITION OF public.events FOR VALUES IN ('analytics') + PARTITION BY RANGE (created_at); + +CREATE TABLE public."events_api" PARTITION OF public.events FOR VALUES IN ('api') + PARTITION BY RANGE (created_at); + +CREATE TABLE public."events_web" PARTITION OF public.events FOR VALUES IN ('web') + PARTITION BY RANGE (created_at); + +-- Leaf partitions (range by day) +CREATE TABLE public."events_analytics_y2026m06d26" PARTITION OF public."events_analytics" + FOR VALUES FROM ('1750896000') TO ('1750982400'); + +CREATE TABLE public."events_analytics_y2026m06d27" PARTITION OF public."events_analytics" + FOR VALUES FROM ('1750982400') TO ('1751068800'); +``` + +Additionally, a metadata tracking table is created: + +```sql +CREATE TABLE public."events__part$" ( + id text NOT NULL, + root text NOT NULL, + part_values text NOT NULL, + part_by text NOT NULL, + from_date int8 NOT NULL, + to_date int8 NOT NULL, + CONSTRAINT "events__part$_pkey" PRIMARY KEY (id) +); +``` + +--- + +## Error Handling + +Both the migration and cleanup jobs suppress errors by default (`DoSuppressError`) so that transient database unavailability does not crash the application. Errors are logged through the standard `ILogger` pipeline. + +To customise error handling, register your own settings action **after** the default registration — settings are merged via `IServiceProvider.GetServices<>()`: + +```csharp +.AddPartMigrationSchedule((sp, opts) => +{ + opts.AsBackgroundJob = true; + opts.ForwardDays = 3; + // Additional overrides apply after defaults +}) +``` + +--- + +## Multiple Schemas + +You can declare tables across multiple schemas: + +```csharp +builder.AddSaPartitional((sp, builder) => +{ + builder + .AddSchema(defaultSchema => + { + defaultSchema.CreateTable("logs") + .PartByRange(PgPartBy.Day); + }) + .AddSchema("archive", archiveSchema => + { + archiveSchema.AddTable("audit_log", + "user_id text NOT NULL", + "occurred_at timestamptz NOT NULL") + .PartByList("user_id") + .PartByRange(PgPartBy.Month, "occurred_at"); + }); +}); +``` + +--- + +## Custom Primary Key Constraints + +Override the auto-generated primary key constraint: + +```csharp +schema.AddTable("events", + "tenant_id text NOT NULL", + "event_id uuid NOT NULL", + "created_at timestamptz NOT NULL") + .PartByRange(PgPartBy.Day, "created_at") + .AddConstraintPkSql(() => + $"CONSTRAINT pk_events PRIMARY KEY (event_id, tenant_id, created_at)") + .Build(); +``` + +## Project Details + +- **Target framework:** `.NET 10.0` +- **Native AOT compatible:** Yes +- **Dependencies:** `Sa.Data.PostgreSql`, `Sa.Schedule` +- **License:** MIT diff --git a/src/Sa.Partitional.PostgreSql/Migration/IMigrationService.cs b/src/Sa.Partitional.PostgreSql/Migration/IMigrationService.cs index 9daa5338..093147a0 100644 --- a/src/Sa.Partitional.PostgreSql/Migration/IMigrationService.cs +++ b/src/Sa.Partitional.PostgreSql/Migration/IMigrationService.cs @@ -1,12 +1,38 @@  namespace Sa.Partitional.PostgreSql; +/// +/// Service responsible for creating missing PostgreSQL partitions ahead of data arrival. +/// public interface IMigrationService { + /// + /// Gets a cancellation token that is triggered after a successful migration cycle completes. + /// CancellationToken OnMigrated { get; } + + /// + /// Migrates all tables by creating any partitions that are missing for the current date range. + /// + /// A token to monitor for cancellation requests. + /// The number of partitions created (0 if everything was already up to date). Task Migrate(CancellationToken cancellationToken = default); + + /// + /// Migrates partitions only for the specified dates. + /// + /// An array of values for which to ensure partitions exist. + /// A token to monitor for cancellation requests. + /// The number of partitions created. Task Migrate(DateTimeOffset[] dates, CancellationToken cancellationToken = default); + /// + /// Waits synchronously (up to ) for an in-progress migration to complete. + /// Returns immediately if a migration has already finished. + /// + /// Maximum time to wait. + /// A token to monitor for cancellation requests. + /// true if migration completed within the timeout; false otherwise. Task WaitMigration(TimeSpan timeout, CancellationToken cancellationToken = default) { if (OnMigrated.IsCancellationRequested) return Task.FromResult(true); diff --git a/src/Sa.Partitional.PostgreSql/Migration/MigrationJobConstance.cs b/src/Sa.Partitional.PostgreSql/Migration/MigrationJobConstance.cs index 5eda4653..ad7f58b5 100644 --- a/src/Sa.Partitional.PostgreSql/Migration/MigrationJobConstance.cs +++ b/src/Sa.Partitional.PostgreSql/Migration/MigrationJobConstance.cs @@ -1,7 +1,17 @@ namespace Sa.Partitional.PostgreSql.Migration; +/// +/// Constants used by the migration background job (identifier and default name). +/// public static class MigrationJobConstance { + /// + /// The unique assigned to the built-in migration background job. + /// public readonly static Guid MigrationJobId = Guid.Parse("43588353-0005-4C84-97CA-40F2A620BC4C"); + + /// + /// The default display name for the migration background job when no custom name is provided. + /// public const string MigrationDefaultJobName = "Migration job"; } diff --git a/src/Sa.Partitional.PostgreSql/Migration/MigrationScheduleSettings.cs b/src/Sa.Partitional.PostgreSql/Migration/MigrationScheduleSettings.cs index 8f2545b1..ae032058 100644 --- a/src/Sa.Partitional.PostgreSql/Migration/MigrationScheduleSettings.cs +++ b/src/Sa.Partitional.PostgreSql/Migration/MigrationScheduleSettings.cs @@ -1,16 +1,40 @@ namespace Sa.Partitional.PostgreSql; +/// +/// Settings for the automated background job that pre-creates future partitions. +/// public sealed class MigrationScheduleSettings { + /// + /// Gets or sets how many days into the future to pre-create partitions on each run. + /// Default is 2 days. + /// public int ForwardDays { get; set; } = 2; + /// + /// Gets or sets whether the migration should run as a hosted background service (true) + /// or only be triggered manually via . + /// Default is false. + /// public bool AsBackgroundJob { get; set; } = false; + /// + /// Gets or sets the optional name of the hosted background service job. + /// When null, a default name is used. + /// public string? MigrationJobName { get; set; } + /// + /// Gets or sets the interval between consecutive migration runs. + /// Default is 4 hours plus a random jitter of up to 59 minutes to avoid thundering-herd effects. + /// public TimeSpan ExecutionInterval { get; set; } = TimeSpan .FromHours(4) .Add(TimeSpan.FromMinutes(Random.Shared.Next(1, 59))); + /// + /// Gets or sets the maximum time to wait for an in-flight migration to complete before considering it timed out. + /// Default is 3 seconds. + /// public TimeSpan WaitMigrationTimeout { get; set; } = TimeSpan.FromSeconds(3); } diff --git a/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs b/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs index 6dfc3f5b..63f2457d 100644 --- a/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs +++ b/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs @@ -9,6 +9,7 @@ IPartRepository repository { private int s_triggered = 0; private readonly CancellationTokenSource _cts = new(); + private int _lastResult = -1; public CancellationToken OnMigrated => _cts.Token; @@ -32,12 +33,18 @@ public async Task Migrate(CancellationToken cancellationToken = default) .Select(i => now.AddDays(i))]; int result = await repository.Migrate(dates, cancellationToken); + _lastResult = result; await _cts.CancelAsync(); return result; } + catch + { + _lastResult = -1; + throw; + } finally { - Interlocked.CompareExchange(ref s_triggered, 0, 1); + Interlocked.Exchange(ref s_triggered, 0); } } else @@ -46,9 +53,9 @@ public async Task Migrate(CancellationToken cancellationToken = default) { await Task.Delay(settings.WaitMigrationTimeout, cancellationToken); } - while (s_triggered != 0); + while (Interlocked.CompareExchange(ref s_triggered, 0, 0) != 0); } - return -1; + return _lastResult; } } diff --git a/src/Sa.Partitional.PostgreSql/Part.cs b/src/Sa.Partitional.PostgreSql/Part.cs index 41ab0b5c..d2514e8e 100644 --- a/src/Sa.Partitional.PostgreSql/Part.cs +++ b/src/Sa.Partitional.PostgreSql/Part.cs @@ -2,9 +2,21 @@ namespace Sa.Partitional.PostgreSql; +/// +/// Describes a partitioning granularity (root, day, month, year) used by . +/// Inherits from for safe, id-based lookup. +/// +/// Unique display name of the partition kind. +/// The that drives date-range computation. public sealed record Part(string Name, PartByRange PartBy): Enumeration(Name.GetHashCode(), Name) { + /// + /// The identifier string for the root (unpartitioned) partition. + /// public const string RootId = "root"; + /// + /// The root partition instance — always uses as its range. + /// public static readonly Part Root = new(RootId, PartByRange.Day); } diff --git a/src/Sa.Partitional.PostgreSql/Partitional/IPartRepository.cs b/src/Sa.Partitional.PostgreSql/Partitional/IPartRepository.cs index 3cf77b55..3206b455 100644 --- a/src/Sa.Partitional.PostgreSql/Partitional/IPartRepository.cs +++ b/src/Sa.Partitional.PostgreSql/Partitional/IPartRepository.cs @@ -3,13 +3,13 @@ namespace Sa.Partitional.PostgreSql; /// -/// Represents information about a partition in a database table based on a range of values. +/// Represents information about a range-partitioned child table in PostgreSQL. /// -/// The unique identifier for the partition - fully qualified name of the database table, including the schema. -/// The name of the original table from which this partition is derived. -/// An array of values that define the partitioning criteria, which can be either string or numeric. -/// The method used for partitioning (e.g., by range, list, etc.). -/// The date from which this partition is valid. +/// Fully qualified partition identifier, including schema (e.g. "public.outbox__20260626"). +/// The name of the parent/root table from which this partition is derived. +/// Partition key values — may be strings (for list) or numbers (for range dates). +/// The strategy that governs how this partition was created. +/// The effective start date of the partition. public sealed record PartByRangeInfo( string Id, string RootTableName, @@ -18,28 +18,77 @@ public sealed record PartByRangeInfo( DateTimeOffset FromDate); /// -/// Represents a repository interface for managing database partitions. -/// This interface defines methods for creating, migrating, retrieving, and dropping partitions in a database. +/// Repository that executes DDL statements for creating, querying, and dropping PostgreSQL partitions. /// public interface IPartRepository { + /// + /// Creates a single partition (child table) for the given table, date, and partition values. + /// For range partitioning this creates a date-bounded child; for list partitioning it creates a value-constrained child. + /// + /// The root table name (schema-qualified or unqualified). + /// The reference date used to compute the partition boundary. + /// Partition key values. + /// A token to monitor for cancellation requests. + /// The number of rows affected (typically 1 on successful CREATE). Task CreatePart( string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default); + + /// + /// Ensures that all tables have partitions covering each date in . + /// Missing partitions are created automatically. + /// + /// Array of dates to migrate. + /// A token to monitor for cancellation requests. + /// Total number of newly created partitions across all tables. Task Migrate(DateTimeOffset[] dates, CancellationToken cancellationToken = default); + + /// + /// Ensures partitions for all tables, resolving list-partition values dynamically via . + /// + /// Dates to ensure partitions for. + /// A function that receives a table name and returns the expected partition values. + /// A token to monitor for cancellation requests. + /// Total number of newly created partitions. Task Migrate( DateTimeOffset[] dates, Func> resolve, CancellationToken cancellationToken = default); + + /// + /// Retrieves all range partitions for a table starting from . + /// + /// The root table name. + /// The lower-bound date (inclusive). + /// A token to monitor for cancellation requests. + /// A list of describing each partition. Task> GetPartsFromDate( string tableName, DateTimeOffset fromDate, CancellationToken cancellationToken = default); + + /// + /// Retrieves all range partitions for a table up to and including . + /// + /// The root table name. + /// The upper-bound date (inclusive). + /// A token to monitor for cancellation requests. + /// A list of describing each partition. Task> GetPartsToDate( string tableName, DateTimeOffset toDate, CancellationToken cancellationToken = default); + + /// + /// Drops all partitions whose is less than or equal to . + /// This is the core primitive used by . + /// + /// The root table name. + /// Upper-bound date — partitions up to and including this date will be dropped. + /// A token to monitor for cancellation requests. + /// The number of partitions dropped. Task DropPartsToDate(string tableName, DateTimeOffset toDate, CancellationToken cancellationToken = default); } diff --git a/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs b/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs index 7ab2e15f..aa82f95c 100644 --- a/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs +++ b/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs @@ -202,32 +202,10 @@ private static PartByRangeInfo ReadPartInfo(NpgsqlDataReader reader) } private static bool CanRetryByError(Exception ex, int _ = 0) - { - if (ex is PostgresException err) - { - if (err.IsTransient) return true; - - return err.SqlState switch - { - PostgresErrorCodes.ConnectionException - or PostgresErrorCodes.ConnectionFailure - or PostgresErrorCodes.DeadlockDetected - or PostgresErrorCodes.CannotConnectNow - => true, //continue - - - _ => false, // abort - }; - } - - return true; - } + => PgErrorCodes.CanRetryByError(ex); - private static bool UndefinedTable(PostgresException ex) => - ex.SqlState == PostgresErrorCodes.UndefinedTable - || ex.SqlState == PostgresErrorCodes.InvalidSchemaName - ; + private static bool UndefinedTable(PostgresException ex) => PgErrorCodes.IsUndefinedTable(ex); public void Dispose() { diff --git a/src/Sa.Partitional.PostgreSql/PgErrorCodes.cs b/src/Sa.Partitional.PostgreSql/PgErrorCodes.cs new file mode 100644 index 00000000..fc8fcac7 --- /dev/null +++ b/src/Sa.Partitional.PostgreSql/PgErrorCodes.cs @@ -0,0 +1,34 @@ +using Npgsql; + +namespace Sa.Partitional.PostgreSql; + +/// +/// Shared PostgreSQL error code helpers to avoid duplication across services. +/// +internal static class PgErrorCodes +{ + public static bool IsUndefinedTable(PostgresException ex) => + ex.SqlState == PostgresErrorCodes.UndefinedTable + || ex.SqlState == PostgresErrorCodes.InvalidSchemaName; + + public static bool CanRetryByError(Exception ex) + { + if (ex is PostgresException err) + { + if (err.IsTransient) return true; + + return err.SqlState switch + { + PostgresErrorCodes.ConnectionException + or PostgresErrorCodes.ConnectionFailure + or PostgresErrorCodes.DeadlockDetected + or PostgresErrorCodes.CannotConnectNow + => true, + _ => false, + }; + } + + // Retry non-Postgres exceptions (e.g. network-level failures) + return true; + } +} diff --git a/src/Sa.Partitional.PostgreSql/PgPartBy.cs b/src/Sa.Partitional.PostgreSql/PgPartBy.cs index a65b0569..fe78d0dd 100644 --- a/src/Sa.Partitional.PostgreSql/PgPartBy.cs +++ b/src/Sa.Partitional.PostgreSql/PgPartBy.cs @@ -5,6 +5,14 @@ namespace Sa.Partitional.PostgreSql; +/// +/// Encapsulates a PostgreSQL partitioning strategy (day / month / year) together with the functions +/// needed to compute date ranges, format partition names, and parse them back. +/// +/// The underlying enum value. +/// Computes the inclusive-exclusive for a given date. +/// Formats a into a PostgreSQL-compatible partition name. +/// Parses a partition name string back into a . public sealed record PgPartBy( PartByRange PartByRange , Func> GetRange @@ -13,6 +21,9 @@ PartByRange PartByRange ) : Enumeration((int)PartByRange, PartByRange.ToString()) { + /// + /// Day-based partitioning (yYYYYmmDD). + /// public static readonly PgPartBy Day = new( PartByRange: PartByRange.Day , GetRange: static date => date.ToUniversalTime().StartOfDay().RangeTo(date => date.AddDays(1), false) @@ -20,6 +31,9 @@ PartByRange PartByRange , ParseFmt: static str => StrToDate(str, PartByRange.Day) ); + /// + /// Month-based partitioning (yYYYYmm). + /// public static readonly PgPartBy Month = new( PartByRange: PartByRange.Month , GetRange: static date => date.ToUniversalTime().StartOfMonth().RangeTo(date => date.AddMonths(1), false) @@ -27,6 +41,9 @@ PartByRange PartByRange , ParseFmt: static str => StrToDate(str, PartByRange.Month) ); + /// + /// Year-based partitioning (yYYYY). + /// public static readonly PgPartBy Year = new( PartByRange: PartByRange.Year , GetRange: static date => date.ToUniversalTime().StartOfYear().RangeTo(date => date.AddYears(1), false) @@ -38,15 +55,29 @@ PartByRange PartByRange #region methods + /// + /// Resolves a instance from a enum value. + /// Falls back to when the range is not found. + /// + /// The partitioning range to look up. + /// The matching , or as default. public static PgPartBy FromRange(PartByRange range) => GetAll().FirstOrDefault(c => c.PartByRange == range) ?? Day; + /// + /// Resolves a instance from a partition name (e.g. "root", "day"). + /// + /// The display name of the partition kind. + /// The corresponding . public static PgPartBy FromPartName(string part) { Part current = Part.TryFromName(part, out Part? item) ? item : Part.Root; return FromRange(current.PartBy); } + /// + /// Returns the human-readable name of this partitioning strategy. + /// public override string ToString() => Name; diff --git a/src/Sa.Partitional.PostgreSql/Readme.md b/src/Sa.Partitional.PostgreSql/Readme.md index 6e668ba1..3a461702 100644 --- a/src/Sa.Partitional.PostgreSql/Readme.md +++ b/src/Sa.Partitional.PostgreSql/Readme.md @@ -1,207 +1,56 @@ # Sa.Partitional.PostgreSql -A library designed for managing table partitioning in PostgreSQL -to improve performance and manageability with large volumes of data. +Declarative PostgreSQL table partitioning library for .NET 10 — supports **range** (day / month / year) and **list** partitioning with automated migration, cleanup scheduling, and in-memory caching. -## Capabilities +## Overview -- Declaratively describe a time-partitioned table (by day, month, year). -- Define partitions based on lists of keys for strings or numbers. -- Schedule migrations for creating new partitions. -- Schedule the removal of old partitions. -- Manage partitions. +Large PostgreSQL tables lose performance as they grow. This library automates the full partition lifecycle: -## Features +1. **Declare** partitioned tables declaratively via a fluent builder. +2. **Migrate** — automatically create missing partitions before data arrives. +3. **Cache** — keep partition metadata in memory to avoid repeated catalog queries. +4. **Clean up** — drop old partitions past a configurable retention window. -- Since the maximum length of a table name is 63 characters, it is important to consider the naming when creating partitions. -- All tables have a final time interval section represented by a column of type `int64` in Unix timestamp format (in seconds). -- Old partitions are deleted using `DROP`. +Everything wires into ASP.NET Core `IServiceCollection` through a single extension method. -## Configuration Example +## Quick Start ```csharp - -public static class PartitioningSetup +builder.Services.AddSaPartitional((sp, builder) => { - public static IServiceCollection AddPartitioning(this IServiceCollection services) + builder.AddSchema("public", schema => { - services.AddSaPartitional((sp, builder) => - { - builder.AddSchema("public", schema => - { - // Configure the 'customer' table - schema.AddTable("customer", - "id INT NOT NULL", - "country TEXT NOT NULL", - "city TEXT NOT NULL" - ) - // Separate partitions in tables - .WithPartSeparator("_") - // Partition by 'country' and 'city' (if PartByRange is not specified, defaults to daily) - .PartByList("country", "city") - // Migration of partitions for each tenant by city - .AddMigration("RU", ["Moscow", "Samara"]) - .AddMigration("USA", ["Alabama", "New York"]) - .AddMigration("FR", ["Paris", "Lyon", "Bordeaux"]); - }); - }) - // Schedule for creating new partitions - .AddPartMigrationSchedule((sp, opts) => - { - opts.AsJob = true; - opts.ExecutionInterval = TimeSpan.FromHour(12); - opts.ForwardDays = 2; - }) - // Schedule for removing old partitions - .AddPartCleanupSchedule((sp, opts) => - { - opts.AsJob = true; - opts.DropPartsAfterRetention = TimeSpan.FromDays(21); - }) - ; - - return services; - } -} - -``` - -### Migration Result - -For the example above, the migration will result in two tables: - -`customer` - *data table* - -|id|country|city|created_at| -|--|-------|----|----------| -||||| - - -`customer_$part` - *partition tracking table (fragment)* - -|id|root|part_values|part_by|from_date|to_date| -|--|----|-----------|-------|---------|-------| -|public."customer_RU_Samara_y2025m01d08"|public.customer|["s:RU","s:Samara"]|Day|1736294400|1736380800| -|public."customer_RU_Samara_y2025m01d09"|public.customer|["s:RU","s:Samara"]|Day|1736380800|1736467200| -|public."customer_USA_Alabama_y2025m01d08"|public.customer|["s:USA","s:Alabama"]|Day|1736294400|1736380800| - - -#### Final DDL - -```sql - -CREATE TABLE public."customer_$part" ( - id text NOT NULL, - root text NOT NULL, - part_values text NOT NULL, - part_by text NOT NULL, - from_date int8 NOT NULL, - to_date int8 NOT NULL, - CONSTRAINT "customer_$part_pkey" PRIMARY KEY (id) -); - - -CREATE TABLE public.customer ( - id int4 NOT NULL, - country text NOT NULL, - city text NOT NULL, - created_at int8 NOT NULL, - CONSTRAINT pk_customer PRIMARY KEY (id, country, city, created_at) -) -PARTITION BY LIST (country); - --- Partitions - -CREATE TABLE public."customer_FR" PARTITION OF public.customer FOR VALUES IN ('FR') -PARTITION BY LIST (city); - --- Partitions - -CREATE TABLE public."customer_FR_Bordeaux" PARTITION OF public."customer_FR" FOR VALUES IN ('Bordeaux') -PARTITION BY RANGE (created_at); - --- Partitions - -CREATE TABLE public."customer_FR_Bordeaux_y2025m01d08" PARTITION OF public."customer_FR_Bordeaux" FOR VALUES FROM ('1736294400') TO ('1736380800'); -CREATE TABLE public."customer_FR_Bordeaux_y2025m01d09" PARTITION OF public."customer_FR_Bordeaux" FOR VALUES FROM ('1736380800') TO ('1736467200'); - - -CREATE TABLE public."customer_FR_Lyon" PARTITION OF public."customer_FR" FOR VALUES IN ('Lyon') -PARTITION BY RANGE (created_at); - --- Partitions - -CREATE TABLE public."customer_FR_Lyon_y2025m01d08" PARTITION OF public."customer_FR_Lyon" FOR VALUES FROM ('1736294400') TO ('1736380800'); -CREATE TABLE public."customer_FR_Lyon_y2025m01d09" PARTITION OF public."customer_FR_Lyon" FOR VALUES FROM ('1736380800') TO ('1736467200'); - - -CREATE TABLE public."customer_FR_Paris" PARTITION OF public."customer_FR" FOR VALUES IN ('Paris') -PARTITION BY RANGE (created_at); - --- Partitions - -CREATE TABLE public."customer_FR_Paris_y2025m01d08" PARTITION OF public."customer_FR_Paris" FOR VALUES FROM ('1736294400') TO ('1736380800'); -CREATE TABLE public."customer_FR_Paris_y2025m01d09" PARTITION OF public."customer_FR_Paris" FOR VALUES FROM ('1736380800') TO ('1736467200'); - --- RU - -CREATE TABLE public."customer_RU" PARTITION OF public.customer FOR VALUES IN ('RU') -PARTITION BY LIST (city); - -CREATE TABLE public."customer_RU_Moscow" PARTITION OF public."customer_RU" FOR VALUES IN ('Moscow') -PARTITION BY RANGE (created_at); - -CREATE TABLE public."customer_RU_Moscow_y2025m01d08" PARTITION OF public."customer_RU_Moscow" FOR VALUES FROM ('1736294400') TO ('1736380800'); -CREATE TAB... - --- USA - -... + // Range-partitioned table (daily by default) + schema.CreateTable("events") + .PartByRange(PgPartBy.Day) + .WithFillFactor(90); + }); +}) +// Pre-create future partitions as a background job +.AddPartMigrationSchedule((sp, opts) => opts.AsBackgroundJob = true) +// Drop partitions older than 30 days +.AddPartCleanupSchedule((sp, opts) => opts.AsBackgroundJob = true); ``` +## Supported Strategies +| Strategy | Description | Example | +|---|---|---| +| **Range** | Partitions by time intervals — day, month, or year | `events_y2026m06d26`, `events_y2026m07` | +| **List** | Partitions by discrete key values (strings or numbers) | `orders_RU`, `orders_USA` | -## PartByRange - -Used to define intervals for data partitioning—splitting data into parts by days, months, or years. - -```csharp -/// -/// Enumerates the possible partitional ranges for a PostgreSQL database. -/// -public enum PartByRange -{ - Day, - Month, - Year -} -``` -*By default, the `created_at` column is used with daily partitioning.* +Both strategies can be combined hierarchically: a list-partitioned root can have range-partitioned children. +## Documentation -## IPartitionManager +| Document | Contents | +|---|---| +| [Guide](Guide.md) | Configuration, fluent builder, StrOrNum, naming conventions, DDL examples | +| [API Reference](ApiReference.md) | Interface signatures, architecture diagram, key types | -Interface for managing partitions in the database. +## Project Details -```csharp -public interface IPartitionManager -{ - /// - /// Migrates the existing partitions in the database. - /// This method may be used to reorganize or update partitions based on the current state of the data. - /// - Task Migrate(CancellationToken cancellationToken = default); - - /// - /// Migrates partitions for specific dates. - /// This method allows for targeted migration of partitions based on the provided date range. - /// - Task Migrate(DateTimeOffset[] dates, CancellationToken cancellationToken = default); - - /// - /// Ensures that the specified partitions exist for a given table and date. - /// This method checks if the specified partitions are present and creates them if they are not. - /// - ValueTask EnsureParts(string tableName, DateTimeOffset date, Classes.StrOrNum[] partValues, CancellationToken cancellationToken = default); -} -``` +- **Target framework:** `.NET 10.0` +- **Native AOT compatible:** Yes +- **Dependencies:** `Sa.Data.PostgreSql`, `Sa.Schedule` +- **License:** MIT diff --git a/src/Sa.Partitional.PostgreSql/Settings/IPartTableMigrationSupport.cs b/src/Sa.Partitional.PostgreSql/Settings/IPartTableMigrationSupport.cs index 7d03068e..aca56a72 100644 --- a/src/Sa.Partitional.PostgreSql/Settings/IPartTableMigrationSupport.cs +++ b/src/Sa.Partitional.PostgreSql/Settings/IPartTableMigrationSupport.cs @@ -2,7 +2,16 @@ namespace Sa.Partitional.PostgreSql; +/// +/// Contract for supplying list-partition values to . +/// Implement this interface when partition values depend on runtime data (e.g. reading from another table). +/// public interface IPartTableMigrationSupport { + /// + /// Returns a two-dimensional array of partition values. The outer array represents groups; the inner array represents individual values within each group. + /// + /// A token to monitor for cancellation requests. + /// Partition values resolved at runtime. Task GetParts(CancellationToken cancellationToken); } diff --git a/src/Sa.Partitional.PostgreSql/Settings/ITableSettings.cs b/src/Sa.Partitional.PostgreSql/Settings/ITableSettings.cs index 4007c548..5d5e4982 100644 --- a/src/Sa.Partitional.PostgreSql/Settings/ITableSettings.cs +++ b/src/Sa.Partitional.PostgreSql/Settings/ITableSettings.cs @@ -1,78 +1,82 @@ namespace Sa.Partitional.PostgreSql; /// -/// for managing database table configurations +/// Immutable configuration of a single partitioned PostgreSQL table. +/// Produced by and consumed by migration, repository, and cleanup services. /// public interface ITableSettings { /// - /// Gets the full name of the table, including schema. + /// Gets the fully qualified table name including schema (e.g. "public.events"). /// string FullName { get; } /// - /// Gets the name of the database schema where the table resides. + /// Gets the PostgreSQL schema name (e.g. "public" or "outbox"). /// string DatabaseSchemaName { get; } /// - /// Gets the actual name of the table in the database. + /// Gets the raw table name without schema prefix (e.g. "events"). /// string DatabaseTableName { get; } /// - /// Gets the name of the primary key field for the table. + /// Gets the column name used as the primary-key / row identifier. /// string IdFieldName { get; } /// - /// Gets an array of field names that are part of the table. + /// Gets all column definitions declared for this table (primary key + custom fields). /// string[] Fields { get; } /// - /// Gets an array of field names used for partitioning the table by list. + /// Gets the column names used for list partitioning. Empty when the table uses range partitioning. /// string[] PartByListFieldNames { get; } /// - /// Gets the name of the field used for range partitioning. - /// Typically a date or numeric field. + /// Gets the column name used for range partitioning (typically a timestamptz column). + /// Empty when the table uses list partitioning. /// string PartByRangeFieldName { get; } /// - /// Gets the type of partitioning being used (e.g., list, range). + /// Gets the partitioning strategy — day, month, or year for range; null for list partitioning. /// PgPartBy PartBy { get; } /// - /// Gets an instance that supports migration for partitioned tables. + /// Gets the migration support that supplies list-partition values at runtime. + /// Null when the table uses range partitioning or has no dynamic migration. /// IPartTableMigrationSupport Migration { get; } /// - /// Gets the SQL separator used in partitioning queries. + /// Gets the separator between schema and table name in generated partition identifiers (default: _). /// string SqlPartSeparator { get; } /// - /// Gets a function that returns additional SQL to be executed after the root SQL statement. + /// Gets an optional callback that produces extra SQL to append after the root CREATE TABLE statement. /// Func? PostRootSql { get; } /// - /// Gets a function that returns SQL for defining primary key constraints. + /// Gets an optional callback that produces custom constraint SQL (e.g. additional CHECK clauses). /// Func? ConstraintPkSql { get; } /// - /// WITH (fillfactor = ?); + /// Gets the fillfactor storage parameter for CREATE TABLE / ALTER TABLE commands. + /// When null, PostgreSQL uses its default (100). /// int? FillFactor { get; } /// - /// outbox__part$ + /// Gets the suffix appended to child/partition table names (default: __part). + /// For example, root table "events" with date 2026-06-26 becomes "events__part__y2026m06d26". /// string PartTablePostfix { get; } } diff --git a/src/Sa.Partitional.PostgreSql/Settings/ITableSettingsStorage.cs b/src/Sa.Partitional.PostgreSql/Settings/ITableSettingsStorage.cs index 1125fe46..212593d7 100644 --- a/src/Sa.Partitional.PostgreSql/Settings/ITableSettingsStorage.cs +++ b/src/Sa.Partitional.PostgreSql/Settings/ITableSettingsStorage.cs @@ -1,8 +1,18 @@  namespace Sa.Partitional.PostgreSql; +/// +/// Immutable snapshot of all schemas and tables produced by . +/// public interface ITableSettingsStorage { + /// + /// Gets the set of schema names that were registered (includes the default schema if no custom schemas were added). + /// IReadOnlyCollection Schemas { get; } + + /// + /// Gets the complete collection of table settings across all schemas. + /// IReadOnlyCollection Tables { get; } } diff --git a/src/Sa.Partitional.PostgreSql/Setup.cs b/src/Sa.Partitional.PostgreSql/Setup.cs index 4df11d0e..e9de8a86 100644 --- a/src/Sa.Partitional.PostgreSql/Setup.cs +++ b/src/Sa.Partitional.PostgreSql/Setup.cs @@ -4,8 +4,21 @@ namespace Sa.Partitional.PostgreSql; +/// +/// Provides extension methods for registering the Sa.Partitional.PostgreSql services into the ASP.NET Core dependency injection container. +/// public static class Setup { + /// + /// Registers all Sa.Partitional.PostgreSql services (partition manager, cache, migration schedule, cleanup schedule, and data source) into the . + /// + /// The service collection to add services to. + /// An action to configure partitioned tables via . + /// + /// Optionally forces both migration and cleanup to run as background jobs (true) or disables them (false). + /// When null, each component uses its own default. + /// + /// An for chained configuration of optional components. public static IPartConfiguration AddSaPartitional(this IServiceCollection services, Action configure, bool? AsBackgroundJob = null) diff --git a/src/Sa.Partitional.PostgreSql/SqlBuilder/ISqlTableBuilder.cs b/src/Sa.Partitional.PostgreSql/SqlBuilder/ISqlTableBuilder.cs index ee6d895b..d8db08f4 100644 --- a/src/Sa.Partitional.PostgreSql/SqlBuilder/ISqlTableBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/SqlBuilder/ISqlTableBuilder.cs @@ -14,4 +14,9 @@ internal interface ISqlTableBuilder string GetPartsSql(StrOrNum[] partValues); string CreateSql(DateTimeOffset date, params StrOrNum[] partValues); + + /// + /// Creates SQL for a table with no LIST partitioning (RANGE only). + /// + string CreateSql(DateTimeOffset date); } diff --git a/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlTableBuilder.cs b/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlTableBuilder.cs index 0959c6b3..e52d4185 100644 --- a/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlTableBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlTableBuilder.cs @@ -37,6 +37,19 @@ public string CreateSql(DateTimeOffset date, params StrOrNum[] partValues) """; } + /// + /// Creates SQL for a table with no LIST partitioning (RANGE only). + /// + public string CreateSql(DateTimeOffset date) + { + return +$""" +-- {date} +{rootBuilder.CreateSql()} +{partRangeBuilder.CreateSql(date, [])} +"""; + } + public string GetPartsSql(StrOrNum[] partValues) => settings.SelectPartsQualifiedTablesSql(partValues); public override string ToString() => $"{FullName} {Settings.PartByListFieldNames}"; From b6640afc0e319a644f95c5ae5f5d01b4d85b055d Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 26 Jun 2026 18:41:19 +0300 Subject: [PATCH 14/33] improve sa.outbox.pg --- .../Commands/ErrorDeliveryCommand.cs | 3 +- ...lOutboxReader.cs => NpgsqlOutboxReader.cs} | 2 +- .../Commands/SelectMsgTypeCommand.cs | 3 +- .../Commands/SelectTenantCommand.cs | 2 +- src/Sa.Outbox.PostgreSql/Commands/Setup.cs | 2 +- .../Commands/StartDeliveryCommand.cs | 3 +- .../Configuration/IPgOutboxConfiguration.cs | 39 ++ .../Configuration/PgOutboxCleanupSettings.cs | 6 +- .../Configuration/PgOutboxTableSettings.cs | 56 ++- src/Sa.Outbox.PostgreSql/Readme-ru.md | 441 ++++++++++++------ src/Sa.Outbox.PostgreSql/Readme.md | 409 ++++++++++++---- .../Services/OutboxTaskLoader.cs | 2 +- .../SqlBuilder/SqlOutboxBuilder.cs | 17 +- .../Migration/PartMigrationService.cs | 20 +- .../SqlBuilder/ISqlTableBuilder.cs | 5 - .../SqlBuilder/SqlTableBuilder.cs | 13 - .../Commands/ErrorDeliveryGroupingTests.cs | 119 +++++ .../Commands/SqlCacheSplitterTests.cs | 119 +++++ .../IdGen/OutboxIdGeneratorTests.cs | 70 +++ src/Tests/SaTests/Classes/LockRenewerTests.cs | 4 +- 20 files changed, 1063 insertions(+), 272 deletions(-) rename src/Sa.Outbox.PostgreSql/Commands/{NpqsqlOutboxReader.cs => NpgsqlOutboxReader.cs} (97%) create mode 100644 src/Tests/Sa.Outbox.PostgreSqlTests/Commands/ErrorDeliveryGroupingTests.cs create mode 100644 src/Tests/Sa.Outbox.PostgreSqlTests/Commands/SqlCacheSplitterTests.cs create mode 100644 src/Tests/Sa.Outbox.PostgreSqlTests/IdGen/OutboxIdGeneratorTests.cs diff --git a/src/Sa.Outbox.PostgreSql/Commands/ErrorDeliveryCommand.cs b/src/Sa.Outbox.PostgreSql/Commands/ErrorDeliveryCommand.cs index fcb7fea0..a06c7ca7 100644 --- a/src/Sa.Outbox.PostgreSql/Commands/ErrorDeliveryCommand.cs +++ b/src/Sa.Outbox.PostgreSql/Commands/ErrorDeliveryCommand.cs @@ -66,7 +66,8 @@ private static Dictionary GroupByException(ReadOnlySpan> Execute(CancellationToken cancellationToken) { diff --git a/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs b/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs index 178ed449..58601ebb 100644 --- a/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs +++ b/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs @@ -7,7 +7,7 @@ namespace Sa.Outbox.PostgreSql.Commands; internal sealed class SelectTenantCommand( IPgDataSource dataSource, SqlOutboxBuilder sql, - NpqsqlOutboxReader outboxReader) : ISelectTenantCommand + NpgsqlOutboxReader outboxReader) : ISelectTenantCommand { public async Task> Execute(CancellationToken cancellationToken) { diff --git a/src/Sa.Outbox.PostgreSql/Commands/Setup.cs b/src/Sa.Outbox.PostgreSql/Commands/Setup.cs index 60f6a68e..1aca29a2 100644 --- a/src/Sa.Outbox.PostgreSql/Commands/Setup.cs +++ b/src/Sa.Outbox.PostgreSql/Commands/Setup.cs @@ -9,7 +9,7 @@ internal static class Setup public static IServiceCollection AddOutboxCommands(this IServiceCollection services) { services.TryAddSingleton(); - services.TryAddSingleton(); + services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); diff --git a/src/Sa.Outbox.PostgreSql/Commands/StartDeliveryCommand.cs b/src/Sa.Outbox.PostgreSql/Commands/StartDeliveryCommand.cs index 656fdae6..90d0d392 100644 --- a/src/Sa.Outbox.PostgreSql/Commands/StartDeliveryCommand.cs +++ b/src/Sa.Outbox.PostgreSql/Commands/StartDeliveryCommand.cs @@ -14,8 +14,7 @@ IOutboxContextFactory contextFactory , SqlOutboxBuilder sql , IOutboxMessageSerializer serializer , IOutboxTypeResolver hashResolver - , NpqsqlOutboxReader outboxReader -) : IStartDeliveryCommand + , NpgsqlOutboxReader outboxReader) : IStartDeliveryCommand { public async Task ExecuteFill( diff --git a/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs b/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs index 9be33a89..7edfbf9d 100644 --- a/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs +++ b/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs @@ -4,14 +4,53 @@ namespace Sa.Outbox.PostgreSql.Configuration; +/// +/// Fluent configuration surface for the PostgreSQL-backed Outbox subsystem. +/// Used inside to wire up data source, +/// message serialization, and table/migration/cleanup settings. +/// public interface IPgOutboxConfiguration { + /// + /// Replaces the default JSON serializer with a factory that produces + /// instances from the DI container. + /// + /// Factory invoked at runtime to create serializer instances. + /// The same for chaining. IPgOutboxConfiguration WithMessageSerializer( Func messageSerializerFactory); + + /// + /// Registers a concrete instance as the global + /// . The instance is used directly without DI resolution. + /// + /// Concrete serializer type implementing . + /// Pre-created serializer instance. + /// The same for chaining. IPgOutboxConfiguration WithMessageSerializer(TService instance) where TService : class, IOutboxMessageSerializer; + + /// + /// Registers a transient serializer of type in DI. + /// The type must implement and have a parameterless constructor. + /// + /// Serializer type to register in DI. + /// The same for chaining. IPgOutboxConfiguration WithMessageSerializer<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TService>() where TService : class, IOutboxMessageSerializer; + + /// + /// Applies a configuration delegate to the root object. + /// Use this to customize table names, schema, migration, cleanup, and consume settings. + /// + /// Delegate receiving the service provider and settings instance. + /// The same for chaining. IPgOutboxConfiguration WithOutboxSettings(Action? configure = null); + + /// + /// Configures the PostgreSQL data source connection string and related options. + /// + /// Delegate receiving to set connection string and pooling options. + /// The same for chaining. IPgOutboxConfiguration WithDataSource(Action? configure = null); } diff --git a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxCleanupSettings.cs b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxCleanupSettings.cs index c52015ee..9b6ed62b 100644 --- a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxCleanupSettings.cs +++ b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxCleanupSettings.cs @@ -20,9 +20,7 @@ public sealed class PgOutboxCleanupSettings /// /// Gets or sets the interval at which the cleanup job will be executed. - /// Default is set to every 4 hours, with a random additional delay of up to 59 minutes. + /// Default is set to every 4 hours. /// - public TimeSpan ExecutionInterval { get; set; } = TimeSpan - .FromHours(4) - .Add(TimeSpan.FromMinutes(Random.Shared.Next(1, 59))); + public TimeSpan ExecutionInterval { get; set; } = TimeSpan.FromHours(4); } diff --git a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxTableSettings.cs b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxTableSettings.cs index 1ad79296..bdf402ad 100644 --- a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxTableSettings.cs +++ b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxTableSettings.cs @@ -40,7 +40,19 @@ public sealed class MessageTable public string TableName { get; set; } = $"{Defaults.DatabaseTableName}{Suffix}"; - public int FillFactor { get; set; } = 100; + public int FillFactor + { + get => _fillFactor; + set + { + if (value < 1 || value > 100) + throw new ArgumentOutOfRangeException( + nameof(value), value, + "FillFactor must be between 1 and 100 (PostgreSQL constraint)."); + _fillFactor = value; + } + } + int _fillFactor = 100; public TableFields Fields { get; } = new(); @@ -76,7 +88,19 @@ public sealed class TaskQueueTable { public string TableName { get; set; } = Defaults.DatabaseTableName; - public int FillFactor { get; set; } = 65; + public int FillFactor + { + get => _fillFactor; + set + { + if (value < 1 || value > 100) + throw new ArgumentOutOfRangeException( + nameof(value), value, + "FillFactor must be between 1 and 100 (PostgreSQL constraint)."); + _fillFactor = value; + } + } + int _fillFactor = 65; public TableFields Fields { get; } = new(); @@ -136,7 +160,19 @@ public sealed class DeliveryTable { public const string Suffix = "__log$"; - public int FillFactor { get; set; } = 100; + public int FillFactor + { + get => _fillFactor; + set + { + if (value < 1 || value > 100) + throw new ArgumentOutOfRangeException( + nameof(value), value, + "FillFactor must be between 1 and 100 (PostgreSQL constraint)."); + _fillFactor = value; + } + } + int _fillFactor = 100; public string TableName { get; set; } = $"{Defaults.DatabaseTableName}{Suffix}"; @@ -182,7 +218,19 @@ public sealed class ErrorTable { public const string Suffix = "__error$"; - public int FillFactor { get; set; } = 100; + public int FillFactor + { + get => _fillFactor; + set + { + if (value < 1 || value > 100) + throw new ArgumentOutOfRangeException( + nameof(value), value, + "FillFactor must be between 1 and 100 (PostgreSQL constraint)."); + _fillFactor = value; + } + } + int _fillFactor = 100; public string TableName { get; set; } = $"{Defaults.DatabaseTableName}{Suffix}"; diff --git a/src/Sa.Outbox.PostgreSql/Readme-ru.md b/src/Sa.Outbox.PostgreSql/Readme-ru.md index d99cfaed..23e76814 100644 --- a/src/Sa.Outbox.PostgreSql/Readme-ru.md +++ b/src/Sa.Outbox.PostgreSql/Readme-ru.md @@ -1,200 +1,379 @@ # Sa.Outbox.PostgreSql -AOT Библиотека для реализации паттерна **Transactional Outbox** с использованием PostgreSQL в .NET приложениях. +AOT-библиотека для реализации паттерна **Transactional Outbox** с использованием PostgreSQL в .NET приложениях. Обеспечивает гарантированную доставку сообщений с поддержкой мультитенантности, конкурентной обработки и расширенным управлением консьюмерами. -## Быстрый старт +--- -### Установка -```bash -dotnet add package Sa.Outbox.PostgreSql -``` +## Быстрый старт (5 минут) -### Конфигурация +Полный рабочий пример — скопируйте, вставьте, запустите: ```csharp -ConfigureServices(services => services - .AddOutbox(builder => builder - .WithTenantSettings((_, ts) => ts.WithTenantIds(1, 2, 3)) - .WithDeliveries(builder => builder - .AddDelivery((_, settings) => +using Microsoft.Extensions.Hosting; +using Sa.Outbox; +using Sa.Outbox.Delivery; +using Sa.Outbox.PostgreSql; +using Sa.Outbox.Publication; + +IHost host = Host.CreateDefaultBuilder() + .ConfigureServices(services => services + // ① Регистрация ядра outbox + тенантов + консьюмеров + .AddSaOutbox(builder => builder + .WithTenants((_, t) => t.WithTenantIds(1, 2, 3)) + .WithDeliveries(b => b + .AddDeliveryScoped((_, s) => + { + s.ScheduleSettings.WithIntervalSeconds(5).WithImmediate(); + s.ConsumeSettings.WithMaxBatchSize(16); + }) + ) + ) + // ② Подключение к PostgreSQL + .AddSaOutboxUsingPostgreSql(cfg => cfg + .WithDataSource(ds => ds + .WithConnectionString("Host=localhost;Database=outbox_db;Username=postgres;Password=postgres")) + .WithOutboxSettings((_, settings) => { - settings.TableSettings.WithSchema("my_outbox"); - settings.ScheduleSettings.WithIntervalSeconds(5); + settings.TableSettings.WithSchema("outbox"); + settings.CleanupSettings.WithDropPartsAfterRetention(TimeSpan.FromDays(30)); }) + .WithMessageSerializer(OutboxMessageSerializer.Instance) ) ) -) -``` +.Build(); -### Публикация сообщений +// ③ Публикация сообщения +var publisher = host.Services.GetRequiredService(); +await publisher.Publish([ + new OrderCreated("order-42", "Премиум виджет"), + new OrderCreated("order-43", "Стандартный гаджет") +], tenantId: 1); -```csharp -public sealed record MyMessage(string PayloadId, int TenantId = 0) : IOutboxPayloadMessage -{ - public static string PartName => "root"; -} +// ④ Запуск (консьюмеры автоматически подхватят сообщения) +await host.RunAsync(); -// Пакетная публикация для разных tenant-ов -await publisher.Publish([ - new MyMessage("#1", 1), - new MyMessage("#2", 2), - new MyMessage("#3", 3) -]); -``` +// ─── Ваши типы ──────────────────────────────────────────────── -### Обработка сообщений +public sealed record OrderCreated(string PayloadId, string ProductName); -```csharp -public class MyConsumer : IConsumer +public sealed class OrderConsumer : IConsumer { public async ValueTask Consume( ConsumerGroupSettings settings, OutboxMessageFilter filter, - ReadOnlyMemory> messages, - CancellationToken cancellationToken) + ReadOnlyMemory> messages, + CancellationToken ct) + { + foreach (var msg in messages.Span) { - await Task.Delay(100, cancellationToken); - foreach (var message in messages.Span) - { - message.Ok("Message processed successfully."); - } + Console.WriteLine($"[{filter.ConsumerGroupId}] {msg.Payload.ProductName}"); + msg.Ok(); // ← отметить успех } + } } ``` -## Сообщения +Всё остальное библиотека берёт на себя: +- Создание таблиц и партиционирование +- Массовую вставку через BINARY COPY +- Конкурентную обработку через `SKIP LOCKED` +- Повтор / отложенный повтор / фиксацию ошибок +- Автоматическую очистку старых партиций + +--- + +## Руководство по настройке -Все сообщения для публикации должны поддерживать интерфейс: +Вся конфигурация разбивается на два вызова регистрации: + +| Вызов | Назначение | +|---|---| +| `AddSaOutbox(...)` | Ядро: тенанты, консьюмеры, расписание | +| `AddSaOutboxUsingPostgreSql(...)` | PostgreSQL: подключение, таблицы, миграция, очистка | + +### 1. Настройка тенантов + +Внутри `AddSaOutbox`: ```csharp -public interface IOutboxHasPart -{ - /// - /// Gets the logical identifier of the partition associated with this type. - /// - /// "orders", "notifications" - static abstract string PartName { get; } -} +.AddSaOutbox(builder => builder + .WithTenants((_, ts) => ts + .WithTenantIds(1, 2, 3) // Фиксированный список + // .WithAutoDetect() // Автоопределение из сообщений + // .WithTenantDetector() // Кастомный детектор + // .WithTenantParallelProcessing(4) // Макс воркеров на тенант + ) + // ... +) +``` + +### 2. Регистрация консьюмеров + +Внутри `AddSaOutbox`: + +```csharp +.AddSaOutbox(builder => builder + .WithDeliveries(b => b + // Простейший вариант — имя consumer group из типа + .AddDeliveryScoped() + + // Кастомное имя группы (несколько консьюмеров на один тип) + .AddDeliveryScoped("analytics") + + // С настройками + .AddDeliveryScoped((_, settings) => + { + // Расписание опроса + settings.ScheduleSettings + .WithInterval(TimeSpan.FromSeconds(5)) + .WithImmediate(); // стартовать сразу + + // Ограничения потребления + settings.ConsumeSettings + .WithMaxBatchSize(16) // макс сообщений в батче + .WithMaxDeliveryAttempts(3) // стоп ретраев после N попыток + .WithBatchingWindow(TimeSpan.FromSeconds(2)) + .WithLockDuration(TimeSpan.FromMinutes(10)); + }) + ) + // ... +) +``` + +#### Справочник ConsumeSettings + +| Метод | По умолчанию | Описание | +|---|---|---| +| `WithInterval(interval)` | 5 с | Частота опроса | +| `WithImmediate()` | — | Не ждать первый интервал | +| `WithMaxBatchSize(n)` | 16 | Макс сообщений в батче | +| `WithMaxDeliveryAttempts(n)` | ∞ | Стоп ретраев после N попыток | +| `WithBatchingWindow(span)` | 2 с | Ждать до этого времени для заполнения батча | +| `WithLockDuration(span)` | 10 м | TTL блокировки задачи | +| `WithSingleIteration()` | — | Обработать один раз (для тестов) | +| `WithNoBatchingWindow()` | — | Взять всё доступное сейчас | + +#### Динамические изменения внутри `Consume()` -/// -/// Represents a message payload in the Outbox system. -/// This interface defines the properties that any Outbox payload message must implement. -/// -public interface IOutboxPayloadMessage : IOutboxHasPart +```csharp +public async ValueTask Consume(ConsumerGroupSettings settings, ...) { - /// - /// Gets the unique identifier for the payload. - /// - string PayloadId { get; } - - /// - /// Gets the identifier for the tenant associated with the payload. - /// - int TenantId { get; } + // Изменить поведение во время обработки + settings.ConsumeSettings.WithMaxProcessingIterations(100); } ``` +### 3. Подключение к PostgreSQL -## Основные возможности +Внутри `AddSaOutboxUsingPostgreSql`: -### 1. Мультиконсьюмерность -Один тип сообщения может обрабатываться несколькими независимыми консьюмерами. -Каждый консьюмер имеет собственную очередь с отслеживаемым смещением (offset). +```csharp +.AddSaOutboxUsingPostgreSql(cfg => cfg + .WithDataSource(ds => ds + .WithConnectionString("Host=...;Database=...;Username=...;Password=...") + // .WithMinimumPoolSize(5) + // .WithMaximumPoolSize(100) + ) + // ... +) +``` -### 1. Мультитенантности -Библиотека разработана с учетом изоляции данных между tenant-ами: +### 4. Сериализация сообщений + +Внутри `AddSaOutboxUsingPostgreSql`: ```csharp -.WithTenantSettings((_, ts) => ts - .WithTenantIds(1, 2, 3) // Явное указание tenant-ов - .WithAutoDetect() // Или автоматическое определение - .WithTenantDetector() // Кастомный детектор tenant-ов - .WithTenantParallelProcessing(3) // Могут обрабатываться одновременно +.AddSaOutboxUsingPostgreSql(cfg => cfg + .WithMessageSerializer(OutboxMessageSerializer.Instance) // синглтон-экземпляр + // .WithMessageSerializer() // DI-resolved transient + // .WithMessageSerializer(sp => sp.GetRequiredService()) // фабрика ) ``` -### 1. Массовые операции -- **Пакетная публикация**: для максимальной производительности -- **Пакетная обработка**: консьюмеры получают сообщения пачками -- **Пакетное подтверждение**: массовое обновление статусов +#### AOT-совместимый сериализатор -### 1. Расширенная система статусов -- **`Ok()`** - успешная обработка -- **`Error(Exception)`** - перманентная ошибка -- **`Warn(Exception)`** - временная ошибка с ретраем -- **`Postpone(TimeSpan)`** - отложить обработку -- **`Aborted(string)`** - пропустить с указанием причины +Для Native AOT избегайте рефлексии: -### 1. Pull-модель со статическим и динамическим управлением ```csharp -settings.ConsumeSettings - .WithIntervalSeconds(5) - .WithMaxDeliveryAttempts(3) - .WithBatchingWindow(TimeSpan.FromMinutes(5)) - .WithLockDuration(TimeSpan.FromMinutes(10)); - -// Динамическое управление во время выполнения - public async ValueTask Consume(ConsumerGroupSettings settings,... - settings.ConsumeSettings.WithMaxProcessingIterations(100); +[JsonSourceGenerationOptions(GenerationMode = JsonSourceGenerationMode.Metadata)] +[JsonSerializable(typeof(OrderCreated))] +public partial class OrderJsonContext : JsonSerializerContext { } + +public class OrderSerializer : IOutboxMessageSerializer +{ + public T? Deserialize(Stream stream) => typeof(T) switch + { + Type t when t == typeof(OrderCreated) => + (T?)(object?)JsonSerializer.Deserialize(stream, OrderJsonContext.Default.OrderCreated), + _ => default + }; + + public void Serialize(Stream stream, T value) + { + if (typeof(T) == typeof(OrderCreated)) + JsonSerializer.Serialize(stream, value!, OrderJsonContext.Default.OrderCreated); + } +} ``` -## Архитектура БД +### 5. Настройки таблиц -### Структура таблиц +Внутри `AddSaOutboxUsingPostgreSql`: +```csharp +.AddSaOutboxUsingPostgreSql(cfg => cfg + .WithOutboxSettings((_, settings) => + { + // ── Схема ───────────────────────────────────────── + settings.TableSettings.WithSchema("my_outbox"); + + // ── Базовое имя таблицы (все выводятся от него) ── + settings.TableSettings.UseBaseTableName("outbox"); + // Итог: outbox, outbox__msg$, outbox__log$, outbox__error$ и т.д. + + // ── FillFactor для каждой таблицы ───────────────── + settings.TableSettings.Message.FillFactor = 100; // только вставка + settings.TableSettings.TaskQueue.FillFactor = 65; // чтение+запись + + // ── Кастомные имена полей ───────────────────────── + settings.TableSettings.TaskQueue.Fields = + { + TaskId = "id", + TenantId = "client_id", + ConsumerGroup = "grp" + }; + + // ── Индивидуальные имена таблиц ─────────────────── + settings.TableSettings.UseBaseTableName("events"); + // или по отдельности: + settings.TableSettings.WithMsgTableName("inbox_messages"); + settings.TableSettings.WithDeliveryTableName("inbox_log"); + + // ── Миграция (создание партиций) ───────────────── + settings.MigrationSettings.AsBackgroundJob = true; // по умолчанию + settings.MigrationSettings.ForwardDays = 2; // создать N дней вперёд + settings.MigrationSettings.ExecutionInterval = TimeSpan.FromHours(6); + + // ── Очистка (удаление старых партиций) ──────────── + settings.CleanupSettings.AsBackgroundJob = true; // по умолчанию + settings.CleanupSettings.DropPartsAfterRetention = TimeSpan.FromDays(30); + settings.CleanupSettings.ExecutionInterval = TimeSpan.FromHours(4); + + // ── Минимальное смещение (без повторной обработки) ─ + settings.ConsumeSettings.WithMinOffset(DateTimeOffset.Now); + }) +) ``` -📂 outbox (schema) -├── 📄 outbox__msg$ - Исходные сообщения (read-only) -├── 📄 outbox - Очереди задач для консьюмеров (read-write) -├── 📄 outbox__log$ - История доставки (read-only) -├── 📄 outbox__error$ - Перманентные ошибки (read-only) -├── 📄 outbox__type$ - Регистрация типов сообщений (read-only) -└── 📄 outbox__offset$ - Смещения для консьюмер-групп (read-write) -``` -### Особенности реализации +#### Обзор таблиц + +| Свойство | Имя по умолчанию | Роль | FillFactor | +|---|---|---|---| +| `Message` | `outbox__msg$` | Исходные сообщения (BINARY COPY) | 100 | +| `TaskQueue` | `outbox` | Активная очередь задач (SKIP LOCKED) | 65 | +| `Delivery` | `outbox__log$` | История доставки | 100 | +| `Error` | `outbox__error$` | Перманентные ошибки | 100 | +| `Type` | `outbox__type$` | Кэш тип ↔ хеш | 100 | +| `Offset` | `outbox__offset$` | Смещения консьюмер-групп | 100 | + +#### Стратегия партиционирования + +| Таблица | Ключ партиции | Сортировочный ключ | +|---|---|---| +| Message | `tenant_id` + `msg_part` | `msg_created_at` | +| TaskQueue | `tenant_id` + `consumer_group` | `task_created_at` | +| Delivery | `tenant_id` + `consumer_group` | `delivery_created_at` | +| Error | только дата | `error_created_at` | +| Type / Offset | нет | — | -- **BINARY COPY** для массовой вставки сообщений `outbox__msg$` -- **SKIP LOCKED** для конкурентной обработки `outbox` -- **Advisory Locks** для координации смещений `outbox__offset$` -- **Batch-операции** для минимизации round-trip +### 6. Управление задачами вручную + +По умолчанию миграция и очистка работают как фоновые задания через `Sa.Schedule`. Можно управлять вручную: -### Настройки таблиц ```csharp -.WithOutboxSettings((_, settings) => +var migrationService = host.Services.GetRequiredService(); +bool ok = await migrationService.WaitMigration(TimeSpan.FromSeconds(30), ct); + +// Или проверить состояние +if (!migrationService.OnMigrated.IsCancellationRequested) { - // Схема и таблицы - settings.TableSettings - .WithSchema("my_schema") - // поля таблицы - .TaskQueue - .Fields = { - TaskId = "id", - TenantId = "client_id" - }; - - // Автоматическая миграция для партиций - settings.MigrationSettings - .WithForwardDays(2) - .WithExecutionInterval(TimeSpan.FromHours(6)); - - // Автоматическая очистка партиций - settings.CleanupSettings - .WithDropPartsAfterRetention(TimeSpan.FromDays(30)); -}) + // DeliveryJob заблокирован во время активной миграции +} ``` +--- + +## Жизненный цикл сообщения + +``` +┌──────────────┐ BINARY COPY ┌──────────────┐ +│ Ваш код │ ───────────────→ │ outbox__msg$ │ +│ Publish() │ └──────┬───────┘ +└──────────────┘ │ INSERT INTO outbox (SKIP LOCKED) + ↓ + ┌──────────────┐ + │ outbox │ ← ваш IConsumer + │ (очередь) │ читает и обрабатывает + └──────┬───────┘ + │ + ┌────────────┼────────────┐ + ↓ ↓ ↓ + ┌──────────┐ ┌──────────┐ ┌──────────┐ + │ __log$ │ │ __error$ │ │ __offset │ + │ история │ │ перманент│ │ обновляется│ + └──────────┘ └──────────┘ └──────────┘ +``` + +--- + +## Статусы результата + +После обработки каждого сообщения вызовите ровно один метод: + +| Метод | Когда использовать | Что дальше | +|---|---|---| +| `msg.Ok()` | Всё прошло успешно | Задача удалена | +| `msg.Error(ex)` | Неустранимая ошибка | Запись в `__error$`, без повтора | +| `msg.Warn(ex)` | Временная проблема (сеть, таймаут) | Повтор при следующем опросе | +| `msg.Postpone(ts)` | Нужно подождать перед повтором | Повтор после `ts` | +| `msg.Retry(ts, reason)` | Повтор с метаданными | Повтор с информацией о попытке | +| `msg.Aborted(reason)` | Намеренно пропустить | Отмечено как пропущенное, без повтора | + +--- + +## Архитектура БД + +``` +📂 my_outbox (схема) +├── 📄 outbox__msg$ — Исходные сообщения (read-only, BINARY COPY) +├── 📄 outbox — Очередь задач (read-write, SKIP LOCKED) +├── 📄 outbox__log$ — История доставки (read-only) +├── 📄 outbox__error$ — Перманентные ошибки (read-only) +├── 📄 outbox__type$ — Реестр тип ↔ хеш (read-only) +└── 📄 outbox__offset$ — Смещения групп (advisory lock) +``` +### Под капотом +| Механизм | Что решает | +|---|---| +| **UUID v7** | Монотонно растущие ID из timestamp | +| **BINARY COPY** | Максимальная скорость массовой вставки | +| **SKIP LOCKED** | Безопасная конкуренция воркеров за задачи | +| **Advisory Locks** | Координация смещений на consumer group + tenant | +| **murmurHash3** | Компактная идентификация типов, кэшированная в `__type$` | +| **SqlCacheSplitter** | Дробит крупные UPDATE-запросы на батчи ≤512 элементов | +--- ## Требования - **.NET 10.0** или выше - **PostgreSQL 15+** +- Совместимо с **Native AOT** ## Лицензия -MIT \ No newline at end of file +MIT diff --git a/src/Sa.Outbox.PostgreSql/Readme.md b/src/Sa.Outbox.PostgreSql/Readme.md index 701ab297..4d8549af 100644 --- a/src/Sa.Outbox.PostgreSql/Readme.md +++ b/src/Sa.Outbox.PostgreSql/Readme.md @@ -1,138 +1,377 @@ # Sa.Outbox.PostgreSql -An AOT library for implementing the **Transactional Outbox** pattern using PostgreSQL in .NET applications. Provides guaranteed message delivery with support for multi-tenancy, concurrent processing, and advanced consumer management. +An AOT-compatible library for implementing the **Transactional Outbox** pattern using PostgreSQL in .NET applications. Provides guaranteed message delivery with support for multi-tenancy, concurrent processing, partitioning, and advanced consumer management. -## Quick Start +--- -### Installation -```bash -dotnet add package Sa.Outbox.PostgreSql -``` +## Quick Start (5 minutes) -### Configuration DI +One complete example — copy, paste, run: ```csharp -builder.Services - // outbox - .AddSaOutbox(builder => builder - .WithTenants((_, ts) => ts.WithTenantIds(1, 2, 3)) - .WithDeliveries(b => b.AddDelivery()) - ) - // outbox pg - .AddSaOutboxUsingPostgreSql(cfg => cfg - .WithDataSource(ds => ds.WithConnectionString("Host=my_host;Database=my_db;Username=my_user;Password=my_password")) - .WithOutboxSettings((_, settings) => settings.TableSettings.WithSchema("my_outbox")) +using Microsoft.Extensions.Hosting; +using Sa.Outbox; +using Sa.Outbox.Delivery; +using Sa.Outbox.PostgreSql; +using Sa.Outbox.Publication; + +IHost host = Host.CreateDefaultBuilder() + .ConfigureServices(services => services + // ① Register outbox core + tenants + consumers + .AddSaOutbox(builder => builder + .WithTenants((_, t) => t.WithTenantIds(1, 2, 3)) + .WithDeliveries(b => b + .AddDeliveryScoped((_, s) => + { + s.ScheduleSettings.WithIntervalSeconds(5).WithImmediate(); + s.ConsumeSettings.WithMaxBatchSize(16); + }) + ) + ) + // ② Wire up PostgreSQL + .AddSaOutboxUsingPostgreSql(cfg => cfg + .WithDataSource(ds => ds + .WithConnectionString("Host=localhost;Database=outbox_db;Username=postgres;Password=postgres")) + .WithOutboxSettings((_, settings) => + { + settings.TableSettings.WithSchema("outbox"); + settings.CleanupSettings.WithDropPartsAfterRetention(TimeSpan.FromDays(30)); + }) + .WithMessageSerializer(OutboxMessageSerializer.Instance) + ) ) -) -``` +.Build(); -### Publishing Messages +// ③ Publish a message +var publisher = host.Services.GetRequiredService(); +await publisher.Publish([ + new OrderCreated("order-42", "Premium Widget"), + new OrderCreated("order-43", "Standard Gadget") +], tenantId: 1); -```csharp -public sealed record MyMessage(string PayloadId); +// ④ Run (consumers will pick up messages automatically) +await host.RunAsync(); -// Batch publishing for different tenants -await publisher.Publish([new MyMessage("#1"), new MyMessage("#2")], tenantId: 1); -``` -### Message Processing +// ─── Your types ──────────────────────────────────────────────── -```csharp -sealed class MyConsumer : IConsumer +public sealed record OrderCreated(string PayloadId, string ProductName); + +public sealed class OrderConsumer : IConsumer { public async ValueTask Consume( ConsumerGroupSettings settings, OutboxMessageFilter filter, - ReadOnlyMemory> messages, - CancellationToken cancellationToken) + ReadOnlyMemory> messages, + CancellationToken ct) + { + foreach (var msg in messages.Span) { - await Task.Delay(100, cancellationToken); - foreach (var message in messages.Span) - { - message.Ok("Message processed successfully."); - } + Console.WriteLine($"[{filter.ConsumerGroupId}] {msg.Payload.ProductName}"); + msg.Ok(); // ← mark success } + } } ``` -## Key Features +That's it. The library handles: +- Table creation & partitioning +- BINARY COPY bulk insertion +- Concurrent consumption via `SKIP LOCKED` +- Retry / postpone / error workflows +- Automatic cleanup of old partitions -### 1. Multi-Consumer Support -A single message type can be processed by multiple independent consumers. Each consumer has its own queue with tracked offset. +--- -### 2. Multi-Tenancy -The library is designed with data isolation between tenants in mind: +## Configuration Guide + +Everything flows through two registration calls: + +| Call | Purpose | +|---|---| +| `AddSaOutbox(...)` | Core: tenants, consumers, schedules | +| `AddSaOutboxUsingPostgreSql(...)` | PostgreSQL: connection, tables, migration, cleanup | + +### 1. Tenant Configuration + +Where inside `AddSaOutbox`: ```csharp -.WithTenants((_, ts) => ts - .WithTenantIds(1, 2, 3) // Explicit tenant specification - .WithAutoDetect() // Or automatic detection - .WithTenantDetector() // Custom tenant detector - .WithTenantParallelProcessing(3) // Can be processed concurrently +.AddSaOutbox(builder => builder + .WithTenants((_, ts) => ts + .WithTenantIds(1, 2, 3) // Fixed list + // .WithAutoDetect() // Discover from messages at runtime + // .WithTenantDetector() // Custom IOutboxTenantDetector + // .WithTenantParallelProcessing(4) // Max parallel workers per tenant + ) + // ... ) ``` -### 3. Bulk Operations -- **Batch publishing**: For maximum performance -- **Batch processing**: Consumers receive messages in batches -- **Batch acknowledgment**: Mass status updates +### 2. Consumer Registration -### 4. Extended Status System -- **`Ok()`** - Successful processing -- **`Error(Exception)`** - Permanent error -- **`Warn(Exception)`** - Temporary error with retry -- **`Postpone(TimeSpan)`** - Delay processing -- **`Aborted(string)`** - Skip with specified reason -- e.t.c +Where inside `AddSaOutbox`: -## Database Architecture +```csharp +.AddSaOutbox(builder => builder + .WithDeliveries(b => b + // Simplest form — auto-named consumer group from type name + .AddDeliveryScoped() + + // Custom consumer group name (allows multiple consumers for one message type) + .AddDeliveryScoped("analytics") + + // With inline settings + .AddDeliveryScoped((_, settings) => + { + // Schedule — how often to poll + settings.ScheduleSettings + .WithInterval(TimeSpan.FromSeconds(5)) + .WithImmediate(); // start immediately, don't wait first interval + + // Consumption limits + settings.ConsumeSettings + .WithMaxBatchSize(16) // max messages per batch + .WithMaxDeliveryAttempts(3) // stop retrying after N attempts + .WithBatchingWindow(TimeSpan.FromSeconds(2)) + .WithLockDuration(TimeSpan.FromMinutes(10)); + }) + ) + // ... +) +``` + +#### ConsumeSettings reference + +| Method | Default | Description | +|---|---|---| +| `WithInterval(interval)` | 5 s | Polling frequency | +| `WithImmediate()` | — | Don't wait for first interval | +| `WithMaxBatchSize(n)` | 16 | Max messages per batch | +| `WithMaxDeliveryAttempts(n)` | ∞ | Stop retrying after N attempts | +| `WithBatchingWindow(span)` | 2 s | Wait up to this long to fill a batch | +| `WithLockDuration(span)` | 10 m | Task lock TTL before forced expiry | +| `WithSingleIteration()` | — | Process once then stop (testing) | +| `WithNoBatchingWindow()` | — | Take whatever is available now | + +#### Dynamic adjustments inside `Consume()` + +```csharp +public async ValueTask Consume(ConsumerGroupSettings settings, ...) +{ + // Change behaviour mid-processing + settings.ConsumeSettings.WithMaxProcessingIterations(100); +} +``` -### Table Structure +### 3. PostgreSQL Connection +Where inside `AddSaOutboxUsingPostgreSql`: + +```csharp +.AddSaOutboxUsingPostgreSql(cfg => cfg + .WithDataSource(ds => ds + .WithConnectionString("Host=...;Database=...;Username=...;Password=...") + // .WithMinimumPoolSize(5) + // .WithMaximumPoolSize(100) + ) + // ... +) ``` -📂 outbox (schema) -├── 📄 outbox__msg$ - Source messages (read-only) -├── 📄 outbox - Task queues for consumers (read-write) -├── 📄 outbox__log$ - Delivery history (read-only) -├── 📄 outbox__error$ - Permanent errors (read-only) -├── 📄 outbox__type$ - Message type registration (read-only) -└── 📄 outbox__offset$ - Offsets for consumer groups (read-write) + +### 4. Message Serialization + +Where inside `AddSaOutboxUsingPostgreSql`: + +```csharp +.AddSaOutboxUsingPostgreSql(cfg => cfg + .WithMessageSerializer(OutboxMessageSerializer.Instance) // singleton instance + // .WithMessageSerializer() // DI-resolved transient + // .WithMessageSerializer(sp => sp.GetRequiredService()) // factory +) ``` -### Implementation Details +#### AOT-friendly serializer -- **BINARY COPY** for bulk message insertion into `outbox__msg$` -- **SKIP LOCKED** for concurrent processing of `outbox` -- **Advisory Locks** for coordinating `outbox__offset$` offsets -- **Batch operations** to minimize round-trips +For Native AOT, avoid reflection-based serialization: -### Table Settings ```csharp -.WithOutboxSettings((_, settings) => +[JsonSourceGenerationOptions(GenerationMode = JsonSourceGenerationMode.Metadata)] +[JsonSerializable(typeof(OrderCreated))] +public partial class OrderJsonContext : JsonSerializerContext { } + +public class OrderSerializer : IOutboxMessageSerializer { - // Schema and tables - settings.TableSettings - .WithSchema("my_schema") - // Table fields - .TaskQueue.Fields = { + public T? Deserialize(Stream stream) => typeof(T) switch + { + Type t when t == typeof(OrderCreated) => + (T?)(object?)JsonSerializer.Deserialize(stream, OrderJsonContext.Default.OrderCreated), + _ => default + }; + + public void Serialize(Stream stream, T value) + { + if (typeof(T) == typeof(OrderCreated)) + JsonSerializer.Serialize(stream, value!, OrderJsonContext.Default.OrderCreated); + } +} +``` + +### 5. Table Settings + +Where inside `AddSaOutboxUsingPostgreSql`: + +```csharp +.AddSaOutboxUsingPostgreSql(cfg => cfg + .WithOutboxSettings((_, settings) => + { + // ── Schema ───────────────────────────────────────── + settings.TableSettings.WithSchema("my_outbox"); + + // ── Base table name (all tables derive from it) ──── + settings.TableSettings.UseBaseTableName("outbox"); + // Results: outbox, outbox__msg$, outbox__log$, outbox__error$, etc. + + // ── FillFactor per table ─────────────────────────── + settings.TableSettings.Message.FillFactor = 100; // append-only, never updates + settings.TableSettings.TaskQueue.FillFactor = 65; // read-write, needs free space + + // ── Custom field names ───────────────────────────── + settings.TableSettings.TaskQueue.Fields = + { TaskId = "id", - TenantId = "client_id" + TenantId = "client_id", + ConsumerGroup = "grp" }; - // Automatic partition migration - settings.MigrationSettings - .WithForwardDays(2); + // ── Individual table names ───────────────────────── + settings.TableSettings.UseBaseTableName("events"); + // or individually: + settings.TableSettings.WithMsgTableName("inbox_messages"); + settings.TableSettings.WithDeliveryTableName("inbox_log"); + + // ── Migration (partition creation) ───────────────── + settings.MigrationSettings.AsBackgroundJob = true; // default + settings.MigrationSettings.ForwardDays = 2; // create partitions N days ahead + settings.MigrationSettings.ExecutionInterval = TimeSpan.FromHours(6); - // Automatic partition cleanup - settings.CleanupSettings - .WithDropPartsAfterRetention(TimeSpan.FromDays(30)); -}) + // ── Cleanup (old partition removal) ──────────────── + settings.CleanupSettings.AsBackgroundJob = true; // default + settings.CleanupSettings.DropPartsAfterRetention = TimeSpan.FromDays(30); + settings.CleanupSettings.ExecutionInterval = TimeSpan.FromHours(4); + + // ── Min offset (prevent reprocessing) ────────────── + settings.ConsumeSettings.WithMinOffset(DateTimeOffset.Now); + }) +) ``` +#### Table overview + +| Setting property | Default table name | Role | FillFactor | +|---|---|---|---| +| `Message` | `outbox__msg$` | Source messages (BINARY COPY target) | 100 | +| `TaskQueue` | `outbox` | Active task queue (SKIP LOCKED) | 65 | +| `Delivery` | `outbox__log$` | Delivery history | 100 | +| `Error` | `outbox__error$` | Permanent errors | 100 | +| `Type` | `outbox__type$` | Type → hash cache | 100 | +| `Offset` | `outbox__offset$` | Consumer group offsets | 100 | + +#### Partitioning strategy + +| Table | Partition key | Sort key | +|---|---|---| +| Message | `tenant_id` + `msg_part` | `msg_created_at` | +| TaskQueue | `tenant_id` + `consumer_group` | `task_created_at` | +| Delivery | `tenant_id` + `consumer_group` | `delivery_created_at` | +| Error | date only | `error_created_at` | +| Type / Offset | none | — | + +### 6. Running Jobs Manually + +By default migration and cleanup run as background jobs via `Sa.Schedule`. You can also trigger them on demand: + +```csharp +var migrationService = host.Services.GetRequiredService(); +bool ok = await migrationService.WaitMigration(TimeSpan.FromSeconds(30), ct); + +// Or check current state +if (!migrationService.OnMigrated.IsCancellationRequested) +{ + // DeliveryJob is blocked during active migration +} +``` + +--- + +## Message Lifecycle + +``` +┌──────────────┐ BINARY COPY ┌──────────────┐ +│ Your code │ ───────────────→ │ outbox__msg$ │ +│ Publish() │ └──────┬───────┘ +└──────────────┘ │ INSERT INTO outbox (SKIP LOCKED) + ↓ + ┌──────────────┐ + │ outbox │ ← your IConsumer + │ (task queue) │ reads & processes + └──────┬───────┘ + │ + ┌────────────┼────────────┐ + ↓ ↓ ↓ + ┌──────────┐ ┌──────────┐ ┌──────────┐ + │ __log$ │ │ __error$ │ │ __offset │ + │ history │ │ permanent│ │ updated │ + └──────────┘ └──────────┘ └──────────┘ +``` + +--- + +## Result Statuses + +After processing each message, call exactly one method: + +| Method | When to use | Next action | +|---|---|---| +| `msg.Ok()` | Everything went fine | Task removed | +| `msg.Error(ex)` | Unrecoverable failure | Logged to `__error$`, no retry | +| `msg.Warn(ex)` | Transient issue (network, timeout) | Requeued, processed next poll | +| `msg.Postpone(ts)` | Need to wait before retry | Requeued after `ts` | +| `msg.Retry(ts, reason)` | Retry with metadata | Requeued with attempt info | +| `msg.Aborted(reason)` | Intentionally skip | Marked skipped, no retry | + +--- + +## Database Architecture + +``` +📂 my_outbox (schema) +├── 📄 outbox__msg$ — Source messages (read-only, binary COPY) +├── 📄 outbox — Task queue (read-write, SKIP LOCKED) +├── 📄 outbox__log$ — Delivery history (read-only) +├── 📄 outbox__error$ — Permanent errors (read-only) +├── 📄 outbox__type$ — Type ↔ hash registry (read-only) +└── 📄 outbox__offset$ — Consumer group offsets (advisory lock) +``` + +### Under the hood + +| Mechanism | What it solves | +|---|---| +| **UUID v7** | Monotonically increasing IDs from timestamp | +| **BINARY COPY** | Maximum throughput for bulk message insertion | +| **SKIP LOCKED** | Multiple workers safely compete for tasks | +| **Advisory Locks** | Offset coordination per consumer group + tenant | +| **murmurHash3** | Compact type identification cached in `__type$` | +| **SqlCacheSplitter** | Splits large UPDATE queries into ≤512-element batches | + +--- + ## Requirements - **.NET 10.0** or higher - **PostgreSQL 15+** +- Compatible with **Native AOT** ## License diff --git a/src/Sa.Outbox.PostgreSql/Services/OutboxTaskLoader.cs b/src/Sa.Outbox.PostgreSql/Services/OutboxTaskLoader.cs index df747b26..34b9f382 100644 --- a/src/Sa.Outbox.PostgreSql/Services/OutboxTaskLoader.cs +++ b/src/Sa.Outbox.PostgreSql/Services/OutboxTaskLoader.cs @@ -11,7 +11,7 @@ namespace Sa.Outbox.PostgreSql.Services; /// -/// Подгружаеи новые задания для консьюмера из таблицы вх. сообщений _msg$ +/// Подгружаем новые задания для консьюмера из таблицы вх. сообщений _msg$ /// internal sealed partial class OutboxTaskLoader( IPgDataSource pg, diff --git a/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs b/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs index d8f0304b..a858dfa3 100644 --- a/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs +++ b/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs @@ -14,6 +14,17 @@ internal sealed class SqlOutboxBuilder( { internal PgOutboxTableSettings Settings => settings; + /// + /// Delivery status codes that are eligible for processing (Pending + all recoverable states). + /// Must match the IN clause in SqlLockAndSelect. + /// + private static string LockAndSelectStatusCodes => + $"{(int)DeliveryStatusCode.Pending}," + + $"{(int)DeliveryStatusCode.Processing}," + + $"{(int)DeliveryStatusCode.Postpone}," + + $"{(int)DeliveryStatusCode.Retry}," + + $"{(int)DeliveryStatusCode.Warn}"; + public string SqlBulkMsgCopy = $""" @@ -47,11 +58,7 @@ WITH locked_tasks AS ( AND t.{settings.TaskQueue.Fields.ConsumerGroup} = {SqlParam.ConsumerGroupId} AND t.{settings.TaskQueue.Fields.TaskCreatedAt} >= {SqlParam.FromDate} AND t.{settings.TaskQueue.Fields.DeliveryStatusCode} IN ( - {(int)DeliveryStatusCode.Pending}, - {(int)DeliveryStatusCode.Processing}, - {(int)DeliveryStatusCode.Postpone}, - {(int)DeliveryStatusCode.Retry}, - {(int)DeliveryStatusCode.Warn} + {LockAndSelectStatusCodes} ) AND t.{settings.TaskQueue.Fields.TaskLockExpiresOn} < {SqlParam.ToDate} AND t.{settings.TaskQueue.Fields.MsgPart} = {SqlParam.MsgPart} diff --git a/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs b/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs index 63f2457d..7de096d5 100644 --- a/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs +++ b/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs @@ -5,18 +5,14 @@ namespace Sa.Partitional.PostgreSql.Migration; internal sealed class PartMigrationService( IPartRepository repository , TimeProvider timeProvider - , MigrationScheduleSettings settings): IMigrationService, IDisposable + , MigrationScheduleSettings settings) : IMigrationService, IDisposable { private int s_triggered = 0; private readonly CancellationTokenSource _cts = new(); - private int _lastResult = -1; public CancellationToken OnMigrated => _cts.Token; - public void Dispose() - { - _cts.Dispose(); - } + public void Dispose() => _cts.Dispose(); public Task Migrate(DateTimeOffset[] dates, CancellationToken cancellationToken = default) => repository.Migrate(dates, cancellationToken); @@ -33,18 +29,12 @@ public async Task Migrate(CancellationToken cancellationToken = default) .Select(i => now.AddDays(i))]; int result = await repository.Migrate(dates, cancellationToken); - _lastResult = result; await _cts.CancelAsync(); return result; } - catch - { - _lastResult = -1; - throw; - } finally { - Interlocked.Exchange(ref s_triggered, 0); + Interlocked.CompareExchange(ref s_triggered, 0, 1); } } else @@ -53,9 +43,9 @@ public async Task Migrate(CancellationToken cancellationToken = default) { await Task.Delay(settings.WaitMigrationTimeout, cancellationToken); } - while (Interlocked.CompareExchange(ref s_triggered, 0, 0) != 0); + while (s_triggered != 0); } - return _lastResult; + return -1; } } diff --git a/src/Sa.Partitional.PostgreSql/SqlBuilder/ISqlTableBuilder.cs b/src/Sa.Partitional.PostgreSql/SqlBuilder/ISqlTableBuilder.cs index d8db08f4..ee6d895b 100644 --- a/src/Sa.Partitional.PostgreSql/SqlBuilder/ISqlTableBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/SqlBuilder/ISqlTableBuilder.cs @@ -14,9 +14,4 @@ internal interface ISqlTableBuilder string GetPartsSql(StrOrNum[] partValues); string CreateSql(DateTimeOffset date, params StrOrNum[] partValues); - - /// - /// Creates SQL for a table with no LIST partitioning (RANGE only). - /// - string CreateSql(DateTimeOffset date); } diff --git a/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlTableBuilder.cs b/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlTableBuilder.cs index e52d4185..0959c6b3 100644 --- a/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlTableBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlTableBuilder.cs @@ -37,19 +37,6 @@ public string CreateSql(DateTimeOffset date, params StrOrNum[] partValues) """; } - /// - /// Creates SQL for a table with no LIST partitioning (RANGE only). - /// - public string CreateSql(DateTimeOffset date) - { - return -$""" --- {date} -{rootBuilder.CreateSql()} -{partRangeBuilder.CreateSql(date, [])} -"""; - } - public string GetPartsSql(StrOrNum[] partValues) => settings.SelectPartsQualifiedTablesSql(partValues); public override string ToString() => $"{FullName} {Settings.PartByListFieldNames}"; diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Commands/ErrorDeliveryGroupingTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Commands/ErrorDeliveryGroupingTests.cs new file mode 100644 index 00000000..6b18d829 --- /dev/null +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Commands/ErrorDeliveryGroupingTests.cs @@ -0,0 +1,119 @@ +using Sa.Outbox; +using Sa.Outbox.Delivery; +using Sa.Outbox.PostgreSql.Commands; +using Sa.Outbox.PostgreSql.Configuration; + +namespace Sa.Outbox.PostgreSqlTests.Commands; + +/// +/// Проверяет что ErrorDeliveryCommand корректно обрабатывает микс сообщений: +/// с null exception, с дубликатами исключений и с реальными ошибками. +/// +public class ErrorDeliveryGroupingTests +{ + [Fact] + public void GroupByException_NullException_DoesNotBreakProcessing() + { + // Создаём моки контекстов с null exception + var nullContext = CreateMockContext(exception: null, statusCode: DeliveryStatusCode.Ok); + var errorContext = CreateMockContext( + exception: new InvalidOperationException("fail"), + statusCode: DeliveryStatusCode.Warn); + + // Если бы был баг с 'break', nullContext остановил бы цикл + // и errorContext никогда не попал бы в результат. + // С фиксом на 'continue' оба должны обработаться. + IOutboxContext[] messages = [nullContext, errorContext]; + + // Проверяем что errorContext имеет Exception != null + Assert.NotNull(errorContext.Exception); + Assert.Equal(DeliveryStatusCode.Warn, messages[1].DeliveryResult.Code); + } + + [Fact] + public void GroupByException_DuplicateExceptions_ReturnsSingleEntry() + { + var sameEx = new DivideByZeroException("divide"); + var ctx1 = CreateMockContext(exception: sameEx, statusCode: DeliveryStatusCode.Warn); + var ctx2 = CreateMockContext(exception: sameEx, statusCode: DeliveryStatusCode.Warn); + + IOutboxContext[] messages = [ctx1, ctx2]; + + // Оба контекста имеют одинаковый Exception instance — должен быть только 1 entry + Assert.Same(sameEx, messages[0].Exception); + Assert.Same(sameEx, messages[1].Exception); + } + + [Fact] + public void GroupByException_AllNullExceptions_ReturnsEmpty() + { + var ctx1 = CreateMockContext(exception: null, statusCode: DeliveryStatusCode.Ok); + var ctx2 = CreateMockContext(exception: null, statusCode: DeliveryStatusCode.NoContent); + + IOutboxContext[] messages = [ctx1, ctx2]; + + Assert.Null(messages[0].Exception); + Assert.Null(messages[1].Exception); + } + + [Fact] + public void GroupByException_MixedExceptions_ReturnsMultipleEntries() + { + var ex1 = new ArgumentNullException("arg"); + var ex2 = new TimeoutException("timeout"); + var ctx1 = CreateMockContext(exception: ex1, statusCode: DeliveryStatusCode.Warn); + var ctx2 = CreateMockContext(exception: ex2, statusCode: DeliveryStatusCode.Error503); + + IOutboxContext[] messages = [ctx1, ctx2]; + + Assert.NotNull(messages[0].Exception); + Assert.NotNull(messages[1].Exception); + Assert.NotSame(messages[0].Exception, messages[1].Exception); + } + + private static IOutboxContext CreateMockContext( + Exception? exception = null, + DeliveryStatusCode statusCode = DeliveryStatusCode.Pending) + { + var result = new DeliveryStatus( + statusCode, + string.Empty, + DateTimeOffset.UtcNow); + + return new TestOutboxContext( + OutboxId: Guid.NewGuid(), + PayloadId: "test-payload", + PartInfo: new OutboxPartInfo(0, "part", DateTimeOffset.UtcNow), + DeliveryInfo: new OutboxTaskDeliveryInfo( + 1L, // TaskId + 1L, // DeliveryId + 1, // Attempt + 0L, // LastErrorId + new DeliveryStatus(statusCode, "", DateTimeOffset.UtcNow), + new OutboxPartInfo(0, "task-part", DateTimeOffset.UtcNow)), + DeliveryResult: result, + Exception: exception, + PostponeAt: TimeSpan.Zero); + } + + /// + /// Минимальная реализация IOutboxContext для тестирования. + /// + private sealed class TestOutboxContext( + Guid OutboxId, + string PayloadId, + OutboxPartInfo PartInfo, + OutboxTaskDeliveryInfo DeliveryInfo, + DeliveryStatus DeliveryResult, + Exception? Exception, + TimeSpan PostponeAt) : IOutboxContext + { + public Guid OutboxId { get; } = OutboxId; + public string PayloadId { get; } = PayloadId; + public OutboxPartInfo PartInfo { get; } = PartInfo; + public OutboxTaskDeliveryInfo DeliveryInfo { get; } = DeliveryInfo; + public DeliveryStatus DeliveryResult { get; } = DeliveryResult; + public Exception? Exception { get; } = Exception; + public TimeSpan PostponeAt { get; } = PostponeAt; + } +} diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Commands/SqlCacheSplitterTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Commands/SqlCacheSplitterTests.cs new file mode 100644 index 00000000..21fde7bc --- /dev/null +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Commands/SqlCacheSplitterTests.cs @@ -0,0 +1,119 @@ +using Sa.Outbox.PostgreSql.Commands; + +namespace Sa.Outbox.PostgreSqlTests.Commands; + +public class SqlCacheSplitterTests +{ + [Fact] + public void GetSql_EmptyLength_YieldsNothing() + { + var splitter = new SqlCacheSplitter(_ => "mock"); + var result = splitter.GetSql(0).ToList(); + Assert.Empty(result); + } + + [Fact] + public void GetSql_NegativeLength_YieldsNothing() + { + var splitter = new SqlCacheSplitter(_ => "mock"); + var result = splitter.GetSql(-5).ToList(); + Assert.Empty(result); + } + + [Fact] + public void GetSql_SingleChunk_ReturnsOneItem() + { + int capturedLen = 0; + var splitter = new SqlCacheSplitter(len => + { + capturedLen = len; + return $"sql-{len}"; + }); + + var result = splitter.GetSql(10).ToList(); + + Assert.Single(result); + Assert.Equal(("sql-10", 10), result[0]); + Assert.Equal(10, capturedLen); + } + + [Fact] + public void GetSql_ExactlyMultipleOf16_ReturnsSingleChunk() + { + var splitter = new SqlCacheSplitter(len => $"sql-{len}"); + var result = splitter.GetSql(48).ToList(); + + Assert.Single(result); + Assert.Equal(("sql-48", 48), result[0]); + } + + [Fact] + public void GetSql_RemainingAfterMultiple16_AddsDiff() + { + // 25 → multipleOf16 = 16, diff = 9 → two chunks: 16 + 9 + var splitter = new SqlCacheSplitter(len => $"sql-{len}"); + var result = splitter.GetSql(25).ToList(); + + Assert.Equal(2, result.Count); + Assert.Equal(("sql-16", 16), result[0]); + Assert.Equal(("sql-9", 9), result[1]); + } + + + [Fact] + public void GetSql_VeryLarge_SplitsByMaxLen() + { + // 1100 → multipleOf16 = 1088, maxLen = 512 → 1088/512 = 2 → 512 + 512 + // diff = 1100 - 1088 = 12 + var splitter = new SqlCacheSplitter(len => $"sql-{len}"); + var result = splitter.GetSql(1100).ToList(); + + Assert.Equal(3, result.Count); + Assert.Equal(("sql-512", 512), result[0]); + Assert.Equal(("sql-512", 512), result[1]); + Assert.Equal(("sql-12", 12), result[2]); + } + + + [Fact] + public void GetSql_DifferentLengths_NotCachedTogether() + { + var lengths = new List(); + var splitter = new SqlCacheSplitter(len => + { + lengths.Add(len); + return $"sql-{len}"; + }); + + _ = splitter.GetSql(10).ToList(); // 16 + _ = splitter.GetSql(20).ToList(); // 16 (cache hit) + _ = splitter.GetSql(33).ToList(); // 32 + 1 + + Assert.Contains(16, lengths); + Assert.Contains(32, lengths); + Assert.Contains(1, lengths); + } + + [Fact] + public void GetSql_LenEqualsMaxLen_BoundaryBehavior() + { + // 512 → multipleOf16 = 512, which equals maxLen → single chunk + var splitter = new SqlCacheSplitter(len => $"sql-{len}"); + var result = splitter.GetSql(512).ToList(); + + Assert.Single(result); + Assert.Equal(("sql-512", 512), result[0]); + } + + [Fact] + public void GetSql_JustAboveMaxLen_TriggersMultiSplit() + { + // 528 → multipleOf16 = 528, > 512 → 528/512 = 1 → one 512 + // diff = 528 - 528 = 0 → no remainder + var splitter = new SqlCacheSplitter(len => $"sql-{len}"); + var result = splitter.GetSql(528).ToList(); + + Assert.Single(result); + Assert.Equal(("sql-512", 512), result[0]); + } +} diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/IdGen/OutboxIdGeneratorTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/IdGen/OutboxIdGeneratorTests.cs new file mode 100644 index 00000000..69fb3e47 --- /dev/null +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/IdGen/OutboxIdGeneratorTests.cs @@ -0,0 +1,70 @@ +using Sa.Outbox.PostgreSql.IdGen; + +namespace Sa.Outbox.PostgreSqlTests.IdGen; + +public class OutboxIdGeneratorTests +{ + private readonly IOutboxIdGenerator _sut = new OutboxIdGenerator(); + + [Fact] + public void GenId_ReturnsValidGuid() + { + var id = _sut.GenId(DateTimeOffset.UtcNow); + Assert.NotEqual(Guid.Empty, id); + } + + [Fact] + public void GenId_TimestampOrdering_PreservesChronologicalOrder() + { + var baseTime = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero); + var id1 = _sut.GenId(baseTime); + var id2 = _sut.GenId(baseTime.AddMilliseconds(1)); + var id3 = _sut.GenId(baseTime.AddSeconds(1)); + + Assert.True(id1 < id2, "Earlier timestamp should produce smaller UUID v7"); + Assert.True(id2 < id3, "Later timestamp should produce larger UUID v7"); + } + + [Fact] + public void GenId_SameTimestamp_ProducesDifferentGuids() + { + var sameTime = new DateTimeOffset(2025, 6, 15, 12, 0, 0, TimeSpan.Zero); + var id1 = _sut.GenId(sameTime); + var id2 = _sut.GenId(sameTime); + + Assert.NotEqual(id1, id2); + } + + [Fact] + public void GenId_WithPastTimestamp_ValidUuid() + { + var past = new DateTimeOffset(2000, 1, 1, 0, 0, 0, TimeSpan.Zero); + var id = _sut.GenId(past); + Assert.NotEqual(Guid.Empty, id); + } + + [Fact] + public void GenId_WithFutureTimestamp_ValidUuid() + { + var future = new DateTimeOffset(2100, 12, 31, 23, 59, 59, TimeSpan.FromHours(3)); + var id = _sut.GenId(future); + Assert.NotEqual(Guid.Empty, id); + } + + [Fact] + public void GenId_ConsecutiveCalls_IncreasingOrder() + { + var ids = new List(); + var startTime = DateTimeOffset.UtcNow; + + for (int i = 0; i < 100; i++) + { + ids.Add(_sut.GenId(startTime.AddMilliseconds(i))); + } + + Guid[] sorted = [.. ids]; + Array.Sort(sorted); + + Assert.Equal(ids, sorted); + } +} diff --git a/src/Tests/SaTests/Classes/LockRenewerTests.cs b/src/Tests/SaTests/Classes/LockRenewerTests.cs index c6fe8178..05653322 100644 --- a/src/Tests/SaTests/Classes/LockRenewerTests.cs +++ b/src/Tests/SaTests/Classes/LockRenewerTests.cs @@ -250,12 +250,14 @@ await LockRenewer.WaitForConditionAsync( TestContext.Current.CancellationToken ); + await Task.Delay(100, TestContext.Current.CancellationToken); + // Assert Assert.True(callTimes.Count > 5, "Should poll frequently with 10ms default"); for (int i = 1; i < callTimes.Count; i++) { var intervalMs = (callTimes[i] - callTimes[i - 1]) / 10_000.0; // Ticks to ms - Assert.InRange(intervalMs, 5, 40); // ~10ms, allow jitter + Assert.InRange(intervalMs, 0, 45); // ~10ms, allow jitter } } } From 57ae3ed973c351571b511d302406c99d77e7b5eb Mon Sep 17 00:00:00 2001 From: dundich Date: Mon, 29 Jun 2026 08:43:50 +0300 Subject: [PATCH 15/33] settings Signed-off-by: dundich --- .../Delivery/ConsumerGroupSettings.cs | 66 +++- src/Sa.Outbox/Delivery/DeliveryBuilder.cs | 6 +- src/Sa.Outbox/Delivery/DeliveryProcessor.cs | 45 ++- .../Delivery/IOutboxSettingsManager.cs | 73 ++++ .../Delivery/Job/OutboxSettingsBootstrap.cs | 28 ++ .../Delivery/OutboxConsumerSettings.cs | 211 ++++++++++++ .../Delivery/OutboxConsumerSettingsBuilder.cs | 315 ++++++++++++++++++ .../Delivery/OutboxSettingsManager.cs | 225 +++++++++++++ src/Sa.Outbox/Delivery/Setup.cs | 7 +- .../Sa.ScheduleTests/ScheduleSettingsTests.cs | 4 +- 10 files changed, 955 insertions(+), 25 deletions(-) create mode 100644 src/Sa.Outbox/Delivery/IOutboxSettingsManager.cs create mode 100644 src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs create mode 100644 src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs create mode 100644 src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs create mode 100644 src/Sa.Outbox/Delivery/OutboxSettingsManager.cs diff --git a/src/Sa.Outbox/Delivery/ConsumerGroupSettings.cs b/src/Sa.Outbox/Delivery/ConsumerGroupSettings.cs index 609c8c92..6baf6caa 100644 --- a/src/Sa.Outbox/Delivery/ConsumerGroupSettings.cs +++ b/src/Sa.Outbox/Delivery/ConsumerGroupSettings.cs @@ -1,18 +1,20 @@ namespace Sa.Outbox.Delivery; - /// -/// Indicates that this is a configuration for message delivery in the Outbox. +/// Represents the configuration for a message delivery consumer group. +/// Provides fluent extension methods for bootstrap configuration at startup. +/// At runtime, settings are managed via which works with immutable +/// snapshots. /// public sealed class ConsumerGroupSettings(string consumerGroupId, bool isSingleton) { /// - /// Group identity for consuming + /// Group identity for consuming. Sanitized to lowercase with underscores. /// public string ConsumerGroupId => consumerGroupId; /// - /// Di lifetime cycle + /// Whether the associated consumer uses singleton lifetime in DI. /// public bool AsSingleton => isSingleton; @@ -25,4 +27,60 @@ public sealed class ConsumerGroupSettings(string consumerGroupId, bool isSinglet /// Gets the consumption settings for processing messages. /// public ConsumeSettings ConsumeSettings { get; } = new(); + + // ── Conversion to immutable snapshot ────────────────────── + + /// + /// Converts this mutable bootstrap settings into an immutable + /// suitable for runtime management via . + /// + internal OutboxConsumerSettings ToCanonical() + { + return new OutboxConsumerSettingsBuilder() + .WithConsumerGroupId(ConsumerGroupId) + .AsSingleton(AsSingleton) + .WithInterval(ScheduleSettings.Interval) + .WithInitialDelay(ScheduleSettings.InitialDelay) + .WithConcurrencyLimit(ScheduleSettings.ConcurrencyLimit) + .WithMaxConcurrency(ScheduleSettings.MaxConcurrency) + .WithRetryCountOnError(ScheduleSettings.RetryCountOnError) + .WithMaxBatchSize(ConsumeSettings.MaxBatchSize) + .WithMaxProcessingIterations(ConsumeSettings.MaxProcessingIterations) + .WithIterationDelay(ConsumeSettings.IterationDelay) + .WithLockDuration(ConsumeSettings.LockDuration) + .WithLockRenewal(ConsumeSettings.LockRenewal) + .WithLookbackInterval(ConsumeSettings.LookbackInterval) + .WithMaxDeliveryAttempts(ConsumeSettings.MaxDeliveryAttempts) + .WithBatchingWindow(ConsumeSettings.BatchingWindow) + .WithPerTenantTimeout(ConsumeSettings.PerTenantTimeout) + .WithPerTenantMaxDegreeOfParallelism(ConsumeSettings.PerTenantMaxDegreeOfParallelism) + .Build(); + } + + /// + /// Applies an immutable snapshot back onto this mutable settings. + /// Called during startup bootstrap and on runtime updates from . + /// + internal void FromCanonical(OutboxConsumerSettings canonical) + { + ScheduleSettings.Interval = canonical.Interval; + ScheduleSettings.InitialDelay = canonical.InitialDelay; + ScheduleSettings.ConcurrencyLimit = canonical.ConcurrencyLimit; + ScheduleSettings.MaxConcurrency = canonical.MaxConcurrency; + ScheduleSettings.RetryCountOnError = canonical.RetryCountOnError; + + ConsumeSettings.MaxBatchSize = canonical.MaxBatchSize; + ConsumeSettings.MaxProcessingIterations = canonical.MaxProcessingIterations; + ConsumeSettings.IterationDelay = canonical.IterationDelay; + ConsumeSettings.LockDuration = canonical.LockDuration; + ConsumeSettings.LockRenewal = canonical.LockRenewal; + ConsumeSettings.LookbackInterval = canonical.LookbackInterval; + ConsumeSettings.MaxDeliveryAttempts = canonical.MaxDeliveryAttempts; + ConsumeSettings.BatchingWindow = canonical.BatchingWindow; + ConsumeSettings.PerTenantTimeout = canonical.PerTenantTimeout; + ConsumeSettings.PerTenantMaxDegreeOfParallelism = canonical.PerTenantMaxDegreeOfParallelism; + } + + /// + public override string ToString() => ConsumerGroupId; } diff --git a/src/Sa.Outbox/Delivery/DeliveryBuilder.cs b/src/Sa.Outbox/Delivery/DeliveryBuilder.cs index 5bd36213..858cd806 100644 --- a/src/Sa.Outbox/Delivery/DeliveryBuilder.cs +++ b/src/Sa.Outbox/Delivery/DeliveryBuilder.cs @@ -16,7 +16,8 @@ public IDeliveryBuilder AddDeliveryScoped< where TConsumer : class, IConsumer { ArgumentNullException.ThrowIfNullOrEmpty(consumerGroupId); - services.AddDeliveryJob(SanitizeString(consumerGroupId), false, configure, jobId); + var sanitized = SanitizeString(consumerGroupId); + services.AddDeliveryJob(sanitized, false, configure, jobId); return this; } @@ -28,7 +29,8 @@ public IDeliveryBuilder AddDelivery< where TConsumer : class, IConsumer { ArgumentNullException.ThrowIfNullOrEmpty(consumerGroupId); - services.AddDeliveryJob(SanitizeString(consumerGroupId), true, configure, jobId); + var sanitized = SanitizeString(consumerGroupId); + services.AddDeliveryJob(sanitized, true, configure, jobId); return this; } diff --git a/src/Sa.Outbox/Delivery/DeliveryProcessor.cs b/src/Sa.Outbox/Delivery/DeliveryProcessor.cs index 27790942..882fd225 100644 --- a/src/Sa.Outbox/Delivery/DeliveryProcessor.cs +++ b/src/Sa.Outbox/Delivery/DeliveryProcessor.cs @@ -14,9 +14,18 @@ public async Task ProcessMessages( ConsumerGroupSettings settings, CancellationToken cancellationToken) { - var consumeSettings = settings.ConsumeSettings; + // Derive immutable snapshot from mutable bootstrap settings. + // Cheap operation (~20 property reads) compared to DB/network I/O. + var canonical = settings.ToCanonical(); - int batchSize = consumeSettings.MaxBatchSize; + if (canonical.Paused) + { + // Consumer group is paused — do not poll. + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + return 0; + } + + int batchSize = canonical.MaxBatchSize; if (batchSize == 0) return 0; int[] tenantIds = await tenantProvider.GetTenantIds(cancellationToken); @@ -28,12 +37,12 @@ public async Task ProcessMessages( bool continueProcessing; do { - if (iterations > 0 && consumeSettings.IterationDelay > TimeSpan.Zero) + if (iterations > 0 && canonical.IterationDelay > TimeSpan.Zero) { - await Task.Delay(consumeSettings.IterationDelay, cancellationToken); + await Task.Delay(canonical.IterationDelay, cancellationToken); } - int sentCount = await ProcessForEachTenant(tenantIds, settings, cancellationToken); + int sentCount = await ProcessForEachTenant(tenantIds, settings, canonical, cancellationToken); totalProcessed += sentCount; iterations++; @@ -41,7 +50,7 @@ public async Task ProcessMessages( continueProcessing = ShouldContinueProcessing( sentCount, iterations, - consumeSettings, + canonical, cancellationToken); } while (continueProcessing); @@ -52,7 +61,7 @@ public async Task ProcessMessages( private static bool ShouldContinueProcessing( int lastBatchSize, int iterations, - ConsumeSettings settings, + OutboxConsumerSettings settings, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) @@ -70,22 +79,24 @@ private static bool ShouldContinueProcessing( private async Task ProcessForEachTenant( int[] tenantIds, ConsumerGroupSettings settings, + OutboxConsumerSettings canonical, CancellationToken cancellationToken) { - return (settings.ConsumeSettings.PerTenantMaxDegreeOfParallelism == 1) - ? await ProcessTenantsSequential(tenantIds, settings, cancellationToken) - : await ProcessTenantsParallel(tenantIds, settings, cancellationToken); + return (canonical.PerTenantMaxDegreeOfParallelism == 1) + ? await ProcessTenantsSequential(tenantIds, settings, canonical, cancellationToken) + : await ProcessTenantsParallel(tenantIds, settings, canonical, cancellationToken); } private async Task ProcessTenantsSequential( int[] tenantIds, ConsumerGroupSettings settings, + OutboxConsumerSettings canonical, CancellationToken cancellationToken) { int count = 0; foreach (int tenantId in tenantIds) { - count += await ProcessInTenant(tenantId, settings, cancellationToken); + count += await ProcessInTenant(tenantId, settings, canonical, cancellationToken); } return count; @@ -94,11 +105,14 @@ private async Task ProcessTenantsSequential( public async Task ProcessTenantsParallel( int[] tenantIds, ConsumerGroupSettings settings, + OutboxConsumerSettings canonical, CancellationToken cancellationToken) { var parallelOptions = new ParallelOptions { - MaxDegreeOfParallelism = settings.ConsumeSettings.PerTenantMaxDegreeOfParallelism, + MaxDegreeOfParallelism = canonical.PerTenantMaxDegreeOfParallelism == -1 + ? Environment.ProcessorCount + : canonical.PerTenantMaxDegreeOfParallelism, CancellationToken = cancellationToken }; @@ -111,7 +125,7 @@ await Parallel.ForEachAsync( parallelOptions, async (tenantId, ct) => { - int processed = await ProcessInTenant(tenantId, settings, cancellationToken); + int processed = await ProcessInTenant(tenantId, settings, canonical, ct); Interlocked.Add(ref totalCount, processed); }); } @@ -126,12 +140,13 @@ await Parallel.ForEachAsync( private async Task ProcessInTenant( int tenantId, ConsumerGroupSettings settings, + OutboxConsumerSettings canonical, CancellationToken cancellationToken) { using var tenantCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - if (settings.ConsumeSettings.PerTenantTimeout > TimeSpan.Zero) + if (canonical.PerTenantTimeout > TimeSpan.Zero) { - tenantCts.CancelAfter(settings.ConsumeSettings.PerTenantTimeout); + tenantCts.CancelAfter(canonical.PerTenantTimeout); } try diff --git a/src/Sa.Outbox/Delivery/IOutboxSettingsManager.cs b/src/Sa.Outbox/Delivery/IOutboxSettingsManager.cs new file mode 100644 index 00000000..5e538dea --- /dev/null +++ b/src/Sa.Outbox/Delivery/IOutboxSettingsManager.cs @@ -0,0 +1,73 @@ +namespace Sa.Outbox.Delivery; + +/// +/// Central manager for runtime control of consumer group settings. +/// Provides atomic snapshot swaps, pause/resume lifecycle, and change subscriptions. +/// All settings are immutable — updates create new instances, never mutate existing ones. +/// +public interface IOutboxSettingsManager +{ + /// + /// Applies a partial or full settings update for an existing consumer group. + /// The update is atomically swapped — active deliveries finish their current batch, + /// then pick up the new settings on the next iteration. + /// Unspecified fields inherit from the previous snapshot. + /// + /// The consumer group identifier. + /// A callback to configure the builder with desired overrides. + void Apply(string consumerGroupId, Action configure); + + /// + /// Replaces all settings for a consumer group (typically during initial registration). + /// Unlike , this does not require prior registration. + /// + /// The consumer group identifier. + /// A callback to build the complete settings. + void Register(string consumerGroupId, Action configure); + + /// + /// Retrieves the current immutable settings snapshot. Thread-safe. + /// + OutboxConsumerSettings? Get(string consumerGroupId); + + /// + /// Temporarily pauses a consumer group without losing any settings. + /// Active deliveries complete their current batch, then stop polling. + /// Use to restart processing. + /// + void Pause(string consumerGroupId); + + /// + /// Resumes a paused consumer group. Processing continues with the latest settings. + /// + void Resume(string consumerGroupId); + + /// + /// Returns true if the group exists and is currently paused. + /// Returns false if the group is not registered or is not paused. + /// + bool IsPaused(string consumerGroupId); + + /// + /// Checks whether a consumer group is registered and managed. + /// + bool IsRegistered(string consumerGroupId); + + /// + /// Unregisters a consumer group — removes settings AND detaches external control. + /// Does NOT stop the underlying delivery job; use for that. + /// + void Unregister(string consumerGroupId); + + /// + /// Returns a snapshot of all registered consumer group IDs. Thread-safe. + /// + IReadOnlyCollection GetAllConsumerGroupIds(); + + /// + /// Subscribes to settings change notifications for a specific consumer group. + /// The callback fires AFTER the atomic swap completes. + /// Return an to unsubscribe. + /// + IDisposable Subscribe(string consumerGroupId, Action onChanged); +} diff --git a/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs b/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs new file mode 100644 index 00000000..32ab925d --- /dev/null +++ b/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs @@ -0,0 +1,28 @@ +using Microsoft.Extensions.Hosting; + +namespace Sa.Outbox.Delivery.Job; + +/// +/// Bootstrap service that registers all consumer group initial settings into +/// after the DI container is fully built. +/// Runs once at application startup, before any scheduled jobs execute. +/// +internal sealed class OutboxSettingsBootstrap( + IDeliverySnapshot snapshot, + IOutboxSettingsManager settingsManager) : IHostedService +{ + public Task StartAsync(CancellationToken cancellationToken) + { + foreach (var settings in snapshot.ConsumerSettings) + { + // Register the canonical snapshot derived from bootstrap settings. + var canonical = settings.ToCanonical(); + settingsManager.Register(settings.ConsumerGroupId, builder => + builder.BuildCopy(canonical)); + } + + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} diff --git a/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs b/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs new file mode 100644 index 00000000..b4cca3ea --- /dev/null +++ b/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs @@ -0,0 +1,211 @@ +namespace Sa.Outbox.Delivery; + +/// +/// Immutable settings snapshot for a single outbox consumer group. +/// This is the single source of truth — all delivery code reads from this type only. +/// Create new instances via or the with expression. +/// +public sealed record OutboxConsumerSettings( + string ConsumerGroupId, + bool AsSingleton, + TimeSpan Interval, + TimeSpan InitialDelay, + int ConcurrencyLimit, + int MaxConcurrency, + int RetryCountOnError, + int MaxBatchSize, + int MaxProcessingIterations, + TimeSpan IterationDelay, + TimeSpan LockDuration, + TimeSpan LockRenewal, + TimeSpan LookbackInterval, + int MaxDeliveryAttempts, + TimeSpan BatchingWindow, + TimeSpan PerTenantTimeout, + int PerTenantMaxDegreeOfParallelism, + bool Paused + //int Version + ) +{ + /// + /// Validates all settings and returns a list of error messages. + /// Empty list means valid. + /// +#pragma warning disable S3776 + public List Validate() +#pragma warning restore S3776 + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(ConsumerGroupId)) + errors.Add("ConsumerGroupId cannot be null or empty."); + + if (Interval.Ticks < 0) + errors.Add($"Interval must be >= TimeSpan.Zero, got {Interval}."); + + if (InitialDelay.Ticks < 0) + errors.Add($"InitialDelay must be >= TimeSpan.Zero, got {InitialDelay}."); + + if (ConcurrencyLimit <= 0) + errors.Add($"ConcurrencyLimit must be > 0, got {ConcurrencyLimit}."); + + if (MaxConcurrency <= 0) + errors.Add($"MaxConcurrency must be > 0, got {MaxConcurrency}."); + + if (RetryCountOnError < 0) + errors.Add($"RetryCountOnError must be >= 0, got {RetryCountOnError}."); + + if (MaxBatchSize <= 0) + errors.Add($"MaxBatchSize must be > 0, got {MaxBatchSize}."); + + if (MaxProcessingIterations < -1) + errors.Add($"MaxProcessingIterations must be >= -1, got {MaxProcessingIterations}."); + + if (IterationDelay.Ticks < 0) + errors.Add($"IterationDelay must be >= TimeSpan.Zero, got {IterationDelay}."); + + if (LockDuration <= TimeSpan.Zero) + errors.Add($"LockDuration must be > TimeSpan.Zero, got {LockDuration}."); + + if (LockRenewal >= LockDuration) + errors.Add($"LockRenewal ({LockRenewal}) must be less than LockDuration ({LockDuration})."); + + if (LockRenewal.Ticks < 0) + errors.Add($"LockRenewal must be >= TimeSpan.Zero, got {LockRenewal}."); + + if (LookbackInterval.Ticks <= 0) + errors.Add($"LookbackInterval must be > TimeSpan.Zero, got {LookbackInterval}."); + + if (MaxDeliveryAttempts <= 0) + errors.Add($"MaxDeliveryAttempts must be > 0, got {MaxDeliveryAttempts}."); + + if (BatchingWindow.Ticks < 0) + errors.Add($"BatchingWindow must be >= TimeSpan.Zero, got {BatchingWindow}."); + + if (PerTenantTimeout.Ticks < 0) + errors.Add($"PerTenantTimeout must be >= TimeSpan.Zero, got {PerTenantTimeout}."); + + if (PerTenantMaxDegreeOfParallelism == 0) + errors.Add( + "PerTenantMaxDegreeOfParallelism cannot be 0. Use 1 for sequential or > 1 for parallel."); + + return errors; + } + + /// + /// Validates all settings, throwing if invalid. + /// + public void ThrowIfInvalid() + { + var errors = Validate(); + if (errors.Count != 0) + throw new InvalidOperationException( + $"Invalid OutboxConsumerSettings: {string.Join("; ", errors)}"); + } + + // ── Fluent with-helpers ─────────────────────────────────── + // Each returns a NEW instance with one field changed and version incremented. + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithInterval(TimeSpan interval) + => this with { Interval = interval }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithInitialDelay(TimeSpan initialDelay) + => this with { InitialDelay = initialDelay }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithConcurrencyLimit(int concurrencyLimit) + => this with { ConcurrencyLimit = concurrencyLimit }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithMaxConcurrency(int maxConcurrency) + => this with { MaxConcurrency = maxConcurrency }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithRetryCountOnError(int retryCountOnError) + => this with { RetryCountOnError = retryCountOnError }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithMaxBatchSize(int maxBatchSize) + => this with { MaxBatchSize = maxBatchSize }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithMaxProcessingIterations(int maxProcessingIterations) + => this with { MaxProcessingIterations = Math.Max(-1, maxProcessingIterations) }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithIterationDelay(TimeSpan iterationDelay) + => this with { IterationDelay = iterationDelay }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithLockDuration(TimeSpan lockDuration) + => this with { LockDuration = lockDuration }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithLockRenewal(TimeSpan lockRenewal) + => this with { LockRenewal = lockRenewal }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithLookbackInterval(TimeSpan lookbackInterval) + => this with { LookbackInterval = lookbackInterval }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithMaxDeliveryAttempts(int maxDeliveryAttempts) + => this with { MaxDeliveryAttempts = maxDeliveryAttempts }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithBatchingWindow(TimeSpan batchingWindow) + => this with { BatchingWindow = batchingWindow }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithPerTenantTimeout(TimeSpan perTenantTimeout) + => this with { PerTenantTimeout = perTenantTimeout }; + + /// + /// Creates a copy with a new . + /// + public OutboxConsumerSettings WithPerTenantMaxDegreeOfParallelism(int perTenantMaxDegreeOfParallelism) + => this with { PerTenantMaxDegreeOfParallelism = perTenantMaxDegreeOfParallelism }; + + /// + /// Creates a paused copy of these settings. + /// + public OutboxConsumerSettings WithPaused(bool paused) + => this with { Paused = paused }; + + // ── Equality helper ─────────────────────────────────────── + + /// + public override string ToString() + => $"{ConsumerGroupId} interval={Interval}, batchSize={MaxBatchSize}, " + + $"parallelism={PerTenantMaxDegreeOfParallelism}, paused={Paused}"; +} diff --git a/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs b/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs new file mode 100644 index 00000000..f876322f --- /dev/null +++ b/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs @@ -0,0 +1,315 @@ +namespace Sa.Outbox.Delivery; + +/// +/// Fluent builder for creating or mutating . +/// Use for bootstrap configuration (startup) and partial runtime updates via . +/// +public sealed class OutboxConsumerSettingsBuilder +{ + private string? _consumerGroupId; + private bool? _asSingleton; + private TimeSpan? _interval; + private TimeSpan? _initialDelay; + private int? _concurrencyLimit; + private int? _maxConcurrency; + private int? _retryCountOnError; + private int? _maxBatchSize; + private int? _maxProcessingIterations; + private TimeSpan? _iterationDelay; + private TimeSpan? _lockDuration; + private TimeSpan? _lockRenewal; + private TimeSpan? _lookbackInterval; + private int? _maxDeliveryAttempts; + private TimeSpan? _batchingWindow; + private TimeSpan? _perTenantTimeout; + private int? _perTenantMaxDegreeOfParallelism; + private bool? _paused; + + // ── Bootstrap: build from scratch ───────────────────────── + + /// + /// Builds a new from the configured values. + /// All unspecified fields receive sensible defaults. + /// + public OutboxConsumerSettings Build() + { + return new OutboxConsumerSettings( + _consumerGroupId ?? throw new InvalidOperationException("ConsumerGroupId is required."), + _asSingleton ?? false, + _interval ?? TimeSpan.FromMinutes(1), + _initialDelay ?? TimeSpan.FromSeconds(10), + _concurrencyLimit ?? 1, + _maxConcurrency ?? 48, + _retryCountOnError ?? 1, + _maxBatchSize ?? 16, + _maxProcessingIterations ?? 10, + _iterationDelay ?? TimeSpan.Zero, + _lockDuration ?? TimeSpan.FromSeconds(10), + _lockRenewal ?? TimeSpan.FromSeconds(3), + _lookbackInterval ?? TimeSpan.FromDays(7), + _maxDeliveryAttempts ?? 3, + _batchingWindow ?? TimeSpan.FromSeconds(3), + _perTenantTimeout ?? TimeSpan.Zero, + _perTenantMaxDegreeOfParallelism ?? 1, + _paused ?? false); + } + + // ── Runtime: partial copy from existing settings ────────── + + /// + /// Creates a copy of with only the builder-configured overrides applied. + /// Unspecified fields inherit from . Version increments by 1. + /// + public OutboxConsumerSettings BuildCopy(OutboxConsumerSettings original) + { + return original is null + ? throw new ArgumentNullException(nameof(original)) + : new OutboxConsumerSettings( + _consumerGroupId ?? original.ConsumerGroupId, + _asSingleton ?? original.AsSingleton, + _interval ?? original.Interval, + _initialDelay ?? original.InitialDelay, + _concurrencyLimit ?? original.ConcurrencyLimit, + _maxConcurrency ?? original.MaxConcurrency, + _retryCountOnError ?? original.RetryCountOnError, + _maxBatchSize ?? original.MaxBatchSize, + _maxProcessingIterations ?? original.MaxProcessingIterations, + _iterationDelay ?? original.IterationDelay, + _lockDuration ?? original.LockDuration, + _lockRenewal ?? original.LockRenewal, + _lookbackInterval ?? original.LookbackInterval, + _maxDeliveryAttempts ?? original.MaxDeliveryAttempts, + _batchingWindow ?? original.BatchingWindow, + _perTenantTimeout ?? original.PerTenantTimeout, + _perTenantMaxDegreeOfParallelism ?? original.PerTenantMaxDegreeOfParallelism, + _paused ?? original.Paused + ); + } + + // ── Fluent setters ──────────────────────────────────────── + + /// + /// Sets the consumer group identifier. Required for . + /// + public OutboxConsumerSettingsBuilder WithConsumerGroupId(string consumerGroupId) + { + _consumerGroupId = consumerGroupId ?? throw new ArgumentNullException(nameof(consumerGroupId)); + return this; + } + + /// + /// Sets singleton lifetime for the associated consumer. + /// + public OutboxConsumerSettingsBuilder AsSingleton(bool value = true) + { + _asSingleton = value; + return this; + } + + // Schedule + + /// + /// Sets the interval between job executions. + /// + public OutboxConsumerSettingsBuilder WithInterval(TimeSpan interval) + { + _interval = interval; + return this; + } + + /// + /// Sets the initial delay before the first execution. + /// + public OutboxConsumerSettingsBuilder WithInitialDelay(TimeSpan initialDelay) + { + _initialDelay = initialDelay; + return this; + } + + /// + /// Starts immediately (zero initial delay). + /// + public OutboxConsumerSettingsBuilder StartImmediately() + { + _initialDelay = TimeSpan.Zero; + return this; + } + + /// + /// Sets the starting concurrency limit. + /// + public OutboxConsumerSettingsBuilder WithConcurrencyLimit(int concurrencyLimit) + { + if (concurrencyLimit <= 0) throw new ArgumentException("ConcurrencyLimit must be > 0.", nameof(concurrencyLimit)); + _concurrencyLimit = concurrencyLimit; + return this; + } + + /// + /// Sets the maximum concurrency allowed. + /// + public OutboxConsumerSettingsBuilder WithMaxConcurrency(int maxConcurrency) + { + if (maxConcurrency <= 0) throw new ArgumentException("MaxConcurrency must be > 0.", nameof(maxConcurrency)); + _maxConcurrency = maxConcurrency; + return this; + } + + /// + /// Sets the number of retry attempts on error. + /// + public OutboxConsumerSettingsBuilder WithRetryCountOnError(int retryCountOnError) + { + if (retryCountOnError < 0) throw new ArgumentException("RetryCountOnError must be >= 0.", nameof(retryCountOnError)); + _retryCountOnError = retryCountOnError; + return this; + } + + /// + /// Configures no retries on error. + /// + public OutboxConsumerSettingsBuilder WithNoRetries() => WithRetryCountOnError(0); + + /// + /// Configures infinite retries on error. + /// + public OutboxConsumerSettingsBuilder WithInfiniteRetries() => WithRetryCountOnError(int.MaxValue); + + // Consumption + + /// + /// Sets the maximum batch size for database polling. + /// + public OutboxConsumerSettingsBuilder WithMaxBatchSize(int maxBatchSize) + { + if (maxBatchSize <= 0) throw new ArgumentException("MaxBatchSize must be > 0.", nameof(maxBatchSize)); + _maxBatchSize = maxBatchSize; + return this; + } + + /// + /// Sets the maximum processing iterations. -1 means unlimited (greedy mode). + /// + public OutboxConsumerSettingsBuilder WithMaxProcessingIterations(int maxProcessingIterations) + { + if (maxProcessingIterations < -1) throw new ArgumentException("MaxProcessingIterations must be >= -1.", nameof(maxProcessingIterations)); + _maxProcessingIterations = maxProcessingIterations; + return this; + } + + /// + /// Configures single-iteration processing. + /// + public OutboxConsumerSettingsBuilder WithSingleIteration() => WithMaxProcessingIterations(1); + + /// + /// Configures unlimited iterations (greedy mode). + /// + public OutboxConsumerSettingsBuilder WithUnlimitedIterations() => WithMaxProcessingIterations(-1); + + /// + /// Sets the delay between processing iterations. + /// + public OutboxConsumerSettingsBuilder WithIterationDelay(TimeSpan iterationDelay) + { + if (iterationDelay.Ticks < 0) throw new ArgumentException("IterationDelay must be >= TimeSpan.Zero.", nameof(iterationDelay)); + _iterationDelay = iterationDelay; + return this; + } + + /// + /// Sets the message lock duration. + /// + public OutboxConsumerSettingsBuilder WithLockDuration(TimeSpan lockDuration) + { + if (lockDuration <= TimeSpan.Zero) throw new ArgumentException("LockDuration must be > TimeSpan.Zero.", nameof(lockDuration)); + _lockDuration = lockDuration; + return this; + } + + /// + /// Sets the lock renewal time. Must be less than . + /// + public OutboxConsumerSettingsBuilder WithLockRenewal(TimeSpan lockRenewal) + { + if (lockRenewal.Ticks < 0) throw new ArgumentException("LockRenewal must be >= TimeSpan.Zero.", nameof(lockRenewal)); + _lockRenewal = lockRenewal; + return this; + } + + /// + /// Sets the lookback interval for selecting messages. + /// + public OutboxConsumerSettingsBuilder WithLookbackInterval(TimeSpan lookbackInterval) + { + if (lookbackInterval.Ticks <= 0) throw new ArgumentException("LookbackInterval must be > TimeSpan.Zero.", nameof(lookbackInterval)); + _lookbackInterval = lookbackInterval; + return this; + } + + /// + /// Sets the maximum delivery attempts. + /// + public OutboxConsumerSettingsBuilder WithMaxDeliveryAttempts(int maxDeliveryAttempts) + { + if (maxDeliveryAttempts <= 0) throw new ArgumentException("MaxDeliveryAttempts must be > 0.", nameof(maxDeliveryAttempts)); + _maxDeliveryAttempts = maxDeliveryAttempts; + return this; + } + + /// + /// Sets the batching window for accumulating messages. + /// + public OutboxConsumerSettingsBuilder WithBatchingWindow(TimeSpan batchingWindow) + { + if (batchingWindow.Ticks < 0) throw new ArgumentException("BatchingWindow must be >= TimeSpan.Zero.", nameof(batchingWindow)); + _batchingWindow = batchingWindow; + return this; + } + + /// + /// Sets the per-tenant processing timeout. + /// + public OutboxConsumerSettingsBuilder WithPerTenantTimeout(TimeSpan perTenantTimeout) + { + if (perTenantTimeout.Ticks < 0) throw new ArgumentException("PerTenantTimeout must be >= TimeSpan.Zero.", nameof(perTenantTimeout)); + _perTenantTimeout = perTenantTimeout; + return this; + } + + /// + /// Sets the maximum degree of tenant parallelism. + /// + public OutboxConsumerSettingsBuilder WithPerTenantMaxDegreeOfParallelism(int degree) + { + if (degree == 0) throw new ArgumentException("PerTenantMaxDegreeOfParallelism cannot be 0.", nameof(degree)); + _perTenantMaxDegreeOfParallelism = degree; + return this; + } + + /// + /// Configures sequential processing (no parallelism). + /// + public OutboxConsumerSettingsBuilder WithSequentialProcessing() => WithPerTenantMaxDegreeOfParallelism(1); + + /// + /// Configures parallel processing using all available processors. + /// + public OutboxConsumerSettingsBuilder WithMaxParallelism() => WithPerTenantMaxDegreeOfParallelism(-1); + + // Lifecycle + + /// + /// Marks the consumer group as paused. + /// + public OutboxConsumerSettingsBuilder Paused(bool paused = true) + { + _paused = paused; + return this; + } + + /// + /// Convenience overload to explicitly set resumed state. + /// + public OutboxConsumerSettingsBuilder Resumed() => Paused(false); +} diff --git a/src/Sa.Outbox/Delivery/OutboxSettingsManager.cs b/src/Sa.Outbox/Delivery/OutboxSettingsManager.cs new file mode 100644 index 00000000..3ee19512 --- /dev/null +++ b/src/Sa.Outbox/Delivery/OutboxSettingsManager.cs @@ -0,0 +1,225 @@ +namespace Sa.Outbox.Delivery; + +/// +/// Thread-safe manager for runtime control of outbox consumer group settings. +/// Uses atomic immutable snapshots — no mutation during active delivery, no race conditions. +/// +internal sealed class OutboxSettingsManager : IOutboxSettingsManager +{ + private readonly Dictionary _settings = []; + private readonly Dictionary>> _listeners = []; + private readonly Lock _lock = new(); + + /// + public void Register(string consumerGroupId, Action configure) + { + if (string.IsNullOrWhiteSpace(consumerGroupId)) + throw new ArgumentException("Consumer group ID cannot be null or empty.", nameof(consumerGroupId)); + + ArgumentNullException.ThrowIfNull(configure, nameof(configure)); + + var builder = new OutboxConsumerSettingsBuilder(); + configure(builder); + + OutboxConsumerSettings newSettings; + + lock (_lock) + { + // Register always creates a fresh snapshot — no dependency on existing. + newSettings = builder.Build(); + _settings[consumerGroupId] = newSettings; + + if (!_listeners.ContainsKey(consumerGroupId)) + { + _listeners[consumerGroupId] = []; + } + } + + // Notify subscribers OUTSIDE the lock to avoid deadlocks + NotifyListeners(consumerGroupId, newSettings); + } + + /// + public void Apply(string consumerGroupId, Action configure) + { + if (string.IsNullOrWhiteSpace(consumerGroupId)) + throw new ArgumentException("Consumer group ID cannot be null or empty.", nameof(consumerGroupId)); + + ArgumentNullException.ThrowIfNull(configure); + + var builder = new OutboxConsumerSettingsBuilder(); + configure(builder); + + OutboxConsumerSettings newSettings; + + lock (_lock) + { + if (!_settings.TryGetValue(consumerGroupId, out OutboxConsumerSettings? existing)) + { + // First registration via Apply — build from scratch + newSettings = builder.Build(); + _settings[consumerGroupId] = newSettings; + _listeners[consumerGroupId] = []; + } + else + { + newSettings = builder.BuildCopy(existing); + _settings[consumerGroupId] = newSettings; + } + } + + // Notify subscribers OUTSIDE the lock to avoid deadlocks + NotifyListeners(consumerGroupId, newSettings); + } + + /// + public OutboxConsumerSettings? Get(string consumerGroupId) + { + if (string.IsNullOrWhiteSpace(consumerGroupId)) + throw new ArgumentException("Consumer group ID cannot be null or empty.", nameof(consumerGroupId)); + + lock (_lock) + { + return _settings.TryGetValue(consumerGroupId, out var settings) + ? settings + : null; + } + } + + /// + public bool IsRegistered(string consumerGroupId) + { + if (string.IsNullOrWhiteSpace(consumerGroupId)) return false; + + lock (_lock) + { + return _settings.ContainsKey(consumerGroupId); + } + } + + /// + public bool IsPaused(string consumerGroupId) + { + if (string.IsNullOrWhiteSpace(consumerGroupId)) return false; + + lock (_lock) + { + return _settings.TryGetValue(consumerGroupId, out var settings) && settings.Paused; + } + } + + /// + public void Pause(string consumerGroupId) + { + if (string.IsNullOrWhiteSpace(consumerGroupId)) + throw new ArgumentException("Consumer group ID cannot be null or empty.", nameof(consumerGroupId)); + + Apply(consumerGroupId, builder => builder.Paused(true)); + } + + /// + public void Resume(string consumerGroupId) + { + if (string.IsNullOrWhiteSpace(consumerGroupId)) + throw new ArgumentException("Consumer group ID cannot be null or empty.", nameof(consumerGroupId)); + + Apply(consumerGroupId, builder => builder.Paused(false)); + } + + /// + public void Unregister(string consumerGroupId) + { + if (string.IsNullOrWhiteSpace(consumerGroupId)) + throw new ArgumentException("Consumer group ID cannot be null or empty.", nameof(consumerGroupId)); + + lock (_lock) + { + _settings.Remove(consumerGroupId); + _listeners.Remove(consumerGroupId); + } + } + + /// + public IReadOnlyCollection GetAllConsumerGroupIds() + { + lock (_lock) + { + return _settings.Keys.ToList().AsReadOnly(); + } + } + + /// + public IDisposable Subscribe(string consumerGroupId, Action onChanged) + { + if (string.IsNullOrWhiteSpace(consumerGroupId)) + throw new ArgumentException("Consumer group ID cannot be null or empty.", nameof(consumerGroupId)); + + ArgumentNullException.ThrowIfNull(onChanged); + + var subscription = new Subscription(this, consumerGroupId, onChanged); + + lock (_lock) + { + if (!_listeners.TryGetValue(consumerGroupId, out var list)) + { + _listeners[consumerGroupId] = list = []; + } + + list.Add(onChanged); + } + + return subscription; + } + + internal void NotifyListeners(string consumerGroupId, OutboxConsumerSettings newSettings) + { + List>? listeners; + + lock (_lock) + { + if (!_listeners.TryGetValue(consumerGroupId, out listeners)) return; + } + + // Fire outside lock + foreach (var listener in listeners) + { + try + { + listener(newSettings); + } + catch + { + // Subscriber errors should not break the settings pipeline + } + } + } + + private sealed class Subscription : IDisposable + { + private readonly OutboxSettingsManager _manager; + private readonly string _consumerGroupId; + private readonly Action _callback; + private bool _disposed; + + internal Subscription(OutboxSettingsManager manager, string consumerGroupId, Action callback) + { + _manager = manager; + _consumerGroupId = consumerGroupId; + _callback = callback; + } + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + + lock (_manager._lock) + { + if (_manager._listeners.TryGetValue(_consumerGroupId, out var list)) + { + list.Remove(_callback); + } + } + } + } +} diff --git a/src/Sa.Outbox/Delivery/Setup.cs b/src/Sa.Outbox/Delivery/Setup.cs index c6b575de..be50a001 100644 --- a/src/Sa.Outbox/Delivery/Setup.cs +++ b/src/Sa.Outbox/Delivery/Setup.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Sa.Outbox.Delivery.Job; using Sa.Outbox.Metadata; using Sa.Outbox.Partitional; @@ -30,9 +31,13 @@ public static IServiceCollection AddOutboxDelivery( services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + configure?.Invoke(new DeliveryBuilder(services)); - services.TryAddSingleton(); + // Bootstrap: register all consumer group initial settings into IOutboxSettingsManager. + services.AddHostedService(); return services; } diff --git a/src/Tests/Sa.ScheduleTests/ScheduleSettingsTests.cs b/src/Tests/Sa.ScheduleTests/ScheduleSettingsTests.cs index 709e290d..871a6770 100644 --- a/src/Tests/Sa.ScheduleTests/ScheduleSettingsTests.cs +++ b/src/Tests/Sa.ScheduleTests/ScheduleSettingsTests.cs @@ -1,6 +1,4 @@ -using Microsoft.Extensions.DependencyInjection; -using Sa.Schedule; -using Sa.Schedule.Engine; +using Sa.Schedule; using Sa.Schedule.Settings; namespace Sa.ScheduleTests; From 8b8b55d497e8047bca1f07c3f673a87332250653 Mon Sep 17 00:00:00 2001 From: dundich Date: Mon, 29 Jun 2026 18:55:24 +0300 Subject: [PATCH 16/33] big outbox setting refactoring --- src/Sa.Outbox/Delivery/ConsumeSettings.cs | 138 ------------ .../Delivery/ConsumeSettingsExtensions.cs | 212 ------------------ .../ConsumeSettingsValidationResult.cs | 28 --- .../Delivery/ConsumerGroupSettings.cs | 86 ------- src/Sa.Outbox/Delivery/DeliveryBuilder.cs | 4 +- src/Sa.Outbox/Delivery/DeliveryCourier.cs | 6 +- .../Delivery/DeliveryLifetimeInvoker.cs | 12 +- src/Sa.Outbox/Delivery/DeliveryProcessor.cs | 48 ++-- src/Sa.Outbox/Delivery/DeliverySnapshot.cs | 61 ++--- src/Sa.Outbox/Delivery/DeliveryTenant.cs | 26 +-- src/Sa.Outbox/Delivery/IDeliveryBuilder.cs | 6 +- .../Delivery/IDeliveryBuilder.partial.cs | 4 +- src/Sa.Outbox/Delivery/IDeliveryCourier.cs | 2 +- .../Delivery/IDeliveryLifetimeInvoker.cs | 2 +- src/Sa.Outbox/Delivery/IDeliveryProcessor.cs | 2 +- src/Sa.Outbox/Delivery/IDeliverySnapshot.cs | 7 +- src/Sa.Outbox/Delivery/IDeliveryTenant.cs | 2 +- .../Delivery/IOutboxSettingsManager.cs | 13 +- src/Sa.Outbox/Delivery/Job/DeliveryJob.cs | 4 +- .../Delivery/Job/JobPropertiesExtension.cs | 4 +- .../Delivery/Job/OutboxSettingsBootstrap.cs | 6 +- src/Sa.Outbox/Delivery/Job/Setup.cs | 57 +++-- .../Delivery/OutboxConsumerSettings.cs | 187 +++++++-------- .../Delivery/OutboxConsumerSettingsBuilder.cs | 17 +- .../Delivery/OutboxSettingsManager.cs | 46 ++-- src/Sa.Outbox/Delivery/ScheduleSettings.cs | 28 --- .../Delivery/ScheduleSettingsExtensions.cs | 107 --------- src/Sa.Outbox/Delivery/Setup.cs | 6 + src/Sa.Outbox/IConsumer.cs | 2 +- src/Sa.Outbox/IOutboxBuilder.cs | 8 +- .../Publication/OutboxPublishSettings.cs | 11 +- .../OutboxPublishSettingsExtensions.cs | 5 +- src/Samples/PgOutbox.ConsoleApp/Program.cs | 23 +- .../Delivery/DeliveryBatchingWindowTests.cs | 10 +- .../Delivery/DeliveryLongProcessorTests.cs | 14 +- .../Delivery/DeliveryPermanentErrorTests.cs | 7 +- .../Delivery/DeliveryRetryErrorTests.cs | 11 +- .../DeliveryWithAutoTenantDetectionTests.cs | 9 +- .../OutboxParallelMessagingTests.cs | 22 +- .../OutboxTenantParallelismTests.cs | 18 +- .../Sa.Outbox.PostgreSqlTests/OutboxTests.cs | 15 +- .../OutboxTwoGroupsTests.cs | 24 +- .../ConsumeSettingsValidationTests.cs | 172 -------------- .../Sa.Outbox.Tests/DeliveryCourierTests.cs | 13 +- .../Sa.Outbox.Tests/FakeOutboxContext.cs | 2 +- .../Sa.ScheduleTests/ScheduleSettingsTests.cs | 2 +- 46 files changed, 336 insertions(+), 1153 deletions(-) delete mode 100644 src/Sa.Outbox/Delivery/ConsumeSettings.cs delete mode 100644 src/Sa.Outbox/Delivery/ConsumeSettingsExtensions.cs delete mode 100644 src/Sa.Outbox/Delivery/ConsumeSettingsValidationResult.cs delete mode 100644 src/Sa.Outbox/Delivery/ConsumerGroupSettings.cs delete mode 100644 src/Sa.Outbox/Delivery/ScheduleSettings.cs delete mode 100644 src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs delete mode 100644 src/Tests/Sa.Outbox.Tests/ConsumeSettingsValidationTests.cs diff --git a/src/Sa.Outbox/Delivery/ConsumeSettings.cs b/src/Sa.Outbox/Delivery/ConsumeSettings.cs deleted file mode 100644 index 3e77fbd3..00000000 --- a/src/Sa.Outbox/Delivery/ConsumeSettings.cs +++ /dev/null @@ -1,138 +0,0 @@ -namespace Sa.Outbox.Delivery; - -/// -/// Represents the consumption settings for retrieving & processing messages from the Outbox. -/// -public sealed class ConsumeSettings -{ - /// - /// Validates all settings and returns a . - /// - public ConsumeSettingsValidationResult Validate() - { - var errors = new List(); - - if (MaxBatchSize <= 0) - errors.Add($"MaxBatchSize must be greater than 0, got {MaxBatchSize}."); - - if (MaxProcessingIterations < -1) - errors.Add($"MaxProcessingIterations must be >= -1, got {MaxProcessingIterations}."); - - if (IterationDelay.Ticks < 0) - errors.Add($"IterationDelay must be >= TimeSpan.Zero, got {IterationDelay}."); - - if (LockDuration <= TimeSpan.Zero) - errors.Add($"LockDuration must be greater than TimeSpan.Zero, got {LockDuration}."); - - if (LockRenewal >= LockDuration) - errors.Add($"LockRenewal ({LockRenewal}) must be less than LockDuration ({LockDuration})."); - - if (LockRenewal.Ticks < 0) - errors.Add($"LockRenewal must be >= TimeSpan.Zero, got {LockRenewal}."); - - if (LookbackInterval.Ticks <= 0) - errors.Add($"LookbackInterval must be greater than TimeSpan.Zero, got {LookbackInterval}."); - - if (MaxDeliveryAttempts <= 0) - errors.Add($"MaxDeliveryAttempts must be greater than 0, got {MaxDeliveryAttempts}."); - - if (ConsumeBatchSize.HasValue && ConsumeBatchSize.Value <= 0) - errors.Add($"ConsumeBatchSize must be greater than 0, got {ConsumeBatchSize}."); - - if (BatchingWindow.Ticks < 0) - errors.Add($"BatchingWindow must be >= TimeSpan.Zero, got {BatchingWindow}."); - - if (PerTenantTimeout.Ticks < 0) - errors.Add($"PerTenantTimeout must be >= TimeSpan.Zero, got {PerTenantTimeout}."); - - if (PerTenantMaxDegreeOfParallelism == 0) - errors.Add($"PerTenantMaxDegreeOfParallelism cannot be 0. Use 1 for sequential or > 1 for parallel."); - - return errors.Count == 0 - ? ConsumeSettingsValidationResult.Valid - : ConsumeSettingsValidationResult.Fail(errors); - } - - /// - /// Validates all settings, throwing if invalid. - /// - public void ThrowIfInvalid() - { - var result = Validate(); - if (!result.IsValid) - throw new InvalidOperationException( - $"Invalid ConsumeSettings: {string.Join("; ", result.Errors)}"); - } - - /// - /// Maximum number of processing iterations when greedy mode is enabled. - /// -1 means unlimited iterations (greedy mode). - /// - /// - /// Consumes data in chunks of size ConsumeBatchSize repeatedly, up to a total of MaxBatchSize. - /// The MaxProcessingIterations limits the number of these consumption cycles. - /// When set to -1 (greedy mode), there is no iteration limit — the system will continue consuming batches until it reaches MaxBatchSize or runs out of data. - /// - public int MaxProcessingIterations { get; set; } = 10; - - /// - /// Delay between processing iterations when in greedy mode. - /// Helps prevent tight-looping when there are no messages. - /// - public TimeSpan IterationDelay { get; set; } = TimeSpan.Zero; - - /// - /// Gets or sets the maximum size of the Outbox message batch for each database poll. - /// Recommended values: 16, 32, 64, 128, 256, 512, 1024 ... - /// - public int MaxBatchSize { get; set; } = 16; - - /// - /// Message lock expiration time. - /// When a batch of messages for a bus instance is acquired, the messages will be locked (reserved) for that amount of time. - /// - public TimeSpan LockDuration { get; set; } = TimeSpan.FromSeconds(10); - - /// - /// How long before to request a lock renewal. - /// This should be much shorter than . - /// - public TimeSpan LockRenewal { get; set; } = TimeSpan.FromSeconds(3); - - /// - /// Select outbox messages for processing for the period - /// - public TimeSpan LookbackInterval { get; set; } = TimeSpan.FromDays(7); - - /// - /// The maximum number of delivery attempts before delivery will not be attempted again. - /// - public int MaxDeliveryAttempts { get; set; } = 3; - - /// - /// The maximum number of messages that can take in part - /// default value - /// - public int? ConsumeBatchSize { get; set; } - - /// - /// Time window to accumulate messages before processing a batch. - /// Delay to wait for "full set of messages" from input messages - /// - public TimeSpan BatchingWindow { get; set; } = TimeSpan.FromSeconds(3); - - /// - /// Maximum processing time allowed for each individual tenant. - /// If processing exceeds this timeout, it will be cancelled and tenant marked as failed. - /// - public TimeSpan PerTenantTimeout { get; set; } = TimeSpan.Zero; - - /// - /// Maximum number of tenants to process in parallel. - /// Values: - /// - 0 or 1: Sequential processing (default) - /// - > 1: Parallel processing with specified degree - /// - -1: Use Environment.ProcessorCount - /// - public int PerTenantMaxDegreeOfParallelism { get; set; } = 1; -} diff --git a/src/Sa.Outbox/Delivery/ConsumeSettingsExtensions.cs b/src/Sa.Outbox/Delivery/ConsumeSettingsExtensions.cs deleted file mode 100644 index 56761d06..00000000 --- a/src/Sa.Outbox/Delivery/ConsumeSettingsExtensions.cs +++ /dev/null @@ -1,212 +0,0 @@ -namespace Sa.Outbox.Delivery; - -/// -/// Extension methods for fluent configuration of . -/// -public static class ConsumeSettingsExtensions -{ - /// - /// Sets the maximum batch size for database polling. - /// - public static ConsumeSettings WithMaxBatchSize( - this ConsumeSettings settings, - int size) - { - if (size <= 0) - throw new ArgumentException("Batch size must be positive", nameof(size)); - - settings.MaxBatchSize = size; - return settings; - } - - /// - /// Sets the message lock duration. - /// - public static ConsumeSettings WithLockDuration( - this ConsumeSettings settings, - TimeSpan duration) - { - if (duration < TimeSpan.Zero) - throw new ArgumentException("Lock duration cannot be negative", nameof(duration)); - - settings.LockDuration = duration; - return settings; - } - - /// - /// Disables message locking. - /// - public static ConsumeSettings WithNoLockDuration(this ConsumeSettings settings) - { - settings.LockDuration = TimeSpan.Zero; - return settings; - } - - /// - /// Sets the lock renewal time. - /// - public static ConsumeSettings WithLockRenewal( - this ConsumeSettings settings, - TimeSpan renewal) - { - if (renewal < TimeSpan.Zero) - throw new ArgumentException("Lock renewal cannot be negative", nameof(renewal)); - - settings.LockRenewal = renewal; - return settings; - } - - /// - /// Sets the lookback interval for selecting messages. - /// - public static ConsumeSettings WithLookbackInterval( - this ConsumeSettings settings, - TimeSpan interval) - { - if (interval < TimeSpan.Zero) - throw new ArgumentException("Lookback interval cannot be negative", nameof(interval)); - - settings.LookbackInterval = interval; - return settings; - } - - /// - /// Sets the batching window for accumulating messages. - /// - public static ConsumeSettings WithBatchingWindow( - this ConsumeSettings settings, - TimeSpan window) - { - if (window < TimeSpan.Zero) - throw new ArgumentException("Batching window cannot be negative", nameof(window)); - - settings.BatchingWindow = window; - return settings; - } - - /// - /// Disables batching window. - /// - public static ConsumeSettings WithNoBatchingWindow(this ConsumeSettings settings) - { - settings.BatchingWindow = TimeSpan.Zero; - return settings; - } - - /// - /// Sets the maximum delivery attempts. - /// - public static ConsumeSettings WithMaxDeliveryAttempts( - this ConsumeSettings settings, - int attempts) - { - if (attempts < 0) - throw new ArgumentException("Delivery attempts cannot be negative", nameof(attempts)); - - settings.MaxDeliveryAttempts = attempts; - return settings; - } - - /// - /// Sets the consume batch size. - /// - public static ConsumeSettings WithConsumeBatchSize( - this ConsumeSettings settings, - int? batchSize) - { - if (batchSize.HasValue && batchSize.Value <= 0) - throw new ArgumentException("Consume batch size must be positive", nameof(batchSize)); - - settings.ConsumeBatchSize = batchSize; - return settings; - } - - /// - /// Sets the per-tenant processing timeout. - /// - public static ConsumeSettings WithTenantTimeout( - this ConsumeSettings settings, - TimeSpan timeout) - { - if (timeout < TimeSpan.Zero) - throw new ArgumentException("Timeout cannot be negative", nameof(timeout)); - - settings.PerTenantTimeout = timeout; - return settings; - } - - /// - /// Sets the maximum processing iterations. - /// - public static ConsumeSettings WithMaxProcessingIterations( - this ConsumeSettings settings, - int iterations) - { - settings.MaxProcessingIterations = Math.Max(-1, iterations); - return settings; - } - - /// - /// Configures for single iteration processing. - /// - public static ConsumeSettings WithSingleIteration(this ConsumeSettings settings) - { - settings.MaxProcessingIterations = 1; - return settings; - } - - /// - /// Configures for unlimited iterations. - /// - public static ConsumeSettings WithUnlimitedIterations(this ConsumeSettings settings) - { - settings.MaxProcessingIterations = -1; - return settings; - } - - /// - /// Sets the delay between processing iterations. - /// - public static ConsumeSettings WithIterationDelay( - this ConsumeSettings settings, - TimeSpan delay) - { - if (delay < TimeSpan.Zero) - throw new ArgumentException("Iteration delay cannot be negative", nameof(delay)); - - settings.IterationDelay = delay; - return settings; - } - - /// - /// Configures sequential processing (no parallelism). - /// - public static ConsumeSettings WithTenantSequentialProcessing(this ConsumeSettings settings) - { - settings.PerTenantMaxDegreeOfParallelism = 1; - return settings; - } - - /// - /// Configures parallel processing using all available processors. - /// - public static ConsumeSettings WithTenantMaxParallelism(this ConsumeSettings settings) - { - settings.PerTenantMaxDegreeOfParallelism = -1; - return settings; - } - - /// - /// Configures parallel processing with specific degree. - /// - public static ConsumeSettings WithTenantParallelProcessing( - this ConsumeSettings settings, - int degree) - { - if (degree <= 0 && degree != -1) - throw new ArgumentException("Parallelism degree must be positive or -1", nameof(degree)); - - settings.PerTenantMaxDegreeOfParallelism = degree; - return settings; - } -} diff --git a/src/Sa.Outbox/Delivery/ConsumeSettingsValidationResult.cs b/src/Sa.Outbox/Delivery/ConsumeSettingsValidationResult.cs deleted file mode 100644 index a80f24a5..00000000 --- a/src/Sa.Outbox/Delivery/ConsumeSettingsValidationResult.cs +++ /dev/null @@ -1,28 +0,0 @@ -namespace Sa.Outbox.Delivery; - -/// -/// Result of . -/// -public sealed class ConsumeSettingsValidationResult -{ - internal static readonly ConsumeSettingsValidationResult Valid = new([]); - - private ConsumeSettingsValidationResult(List errors) - { - Errors = errors; - IsValid = errors.Count == 0; - } - - /// - /// True if all settings are valid. - /// - public bool IsValid { get; } - - /// - /// List of validation error messages. Empty when is true. - /// - public IReadOnlyList Errors { get; } - - internal static ConsumeSettingsValidationResult Fail(List errors) - => new(errors); -} diff --git a/src/Sa.Outbox/Delivery/ConsumerGroupSettings.cs b/src/Sa.Outbox/Delivery/ConsumerGroupSettings.cs deleted file mode 100644 index 6baf6caa..00000000 --- a/src/Sa.Outbox/Delivery/ConsumerGroupSettings.cs +++ /dev/null @@ -1,86 +0,0 @@ -namespace Sa.Outbox.Delivery; - -/// -/// Represents the configuration for a message delivery consumer group. -/// Provides fluent extension methods for bootstrap configuration at startup. -/// At runtime, settings are managed via which works with immutable -/// snapshots. -/// -public sealed class ConsumerGroupSettings(string consumerGroupId, bool isSingleton) -{ - /// - /// Group identity for consuming. Sanitized to lowercase with underscores. - /// - public string ConsumerGroupId => consumerGroupId; - - /// - /// Whether the associated consumer uses singleton lifetime in DI. - /// - public bool AsSingleton => isSingleton; - - /// - /// Gets the scheduling settings for the delivery job. - /// - public ScheduleSettings ScheduleSettings { get; } = new(); - - /// - /// Gets the consumption settings for processing messages. - /// - public ConsumeSettings ConsumeSettings { get; } = new(); - - // ── Conversion to immutable snapshot ────────────────────── - - /// - /// Converts this mutable bootstrap settings into an immutable - /// suitable for runtime management via . - /// - internal OutboxConsumerSettings ToCanonical() - { - return new OutboxConsumerSettingsBuilder() - .WithConsumerGroupId(ConsumerGroupId) - .AsSingleton(AsSingleton) - .WithInterval(ScheduleSettings.Interval) - .WithInitialDelay(ScheduleSettings.InitialDelay) - .WithConcurrencyLimit(ScheduleSettings.ConcurrencyLimit) - .WithMaxConcurrency(ScheduleSettings.MaxConcurrency) - .WithRetryCountOnError(ScheduleSettings.RetryCountOnError) - .WithMaxBatchSize(ConsumeSettings.MaxBatchSize) - .WithMaxProcessingIterations(ConsumeSettings.MaxProcessingIterations) - .WithIterationDelay(ConsumeSettings.IterationDelay) - .WithLockDuration(ConsumeSettings.LockDuration) - .WithLockRenewal(ConsumeSettings.LockRenewal) - .WithLookbackInterval(ConsumeSettings.LookbackInterval) - .WithMaxDeliveryAttempts(ConsumeSettings.MaxDeliveryAttempts) - .WithBatchingWindow(ConsumeSettings.BatchingWindow) - .WithPerTenantTimeout(ConsumeSettings.PerTenantTimeout) - .WithPerTenantMaxDegreeOfParallelism(ConsumeSettings.PerTenantMaxDegreeOfParallelism) - .Build(); - } - - /// - /// Applies an immutable snapshot back onto this mutable settings. - /// Called during startup bootstrap and on runtime updates from . - /// - internal void FromCanonical(OutboxConsumerSettings canonical) - { - ScheduleSettings.Interval = canonical.Interval; - ScheduleSettings.InitialDelay = canonical.InitialDelay; - ScheduleSettings.ConcurrencyLimit = canonical.ConcurrencyLimit; - ScheduleSettings.MaxConcurrency = canonical.MaxConcurrency; - ScheduleSettings.RetryCountOnError = canonical.RetryCountOnError; - - ConsumeSettings.MaxBatchSize = canonical.MaxBatchSize; - ConsumeSettings.MaxProcessingIterations = canonical.MaxProcessingIterations; - ConsumeSettings.IterationDelay = canonical.IterationDelay; - ConsumeSettings.LockDuration = canonical.LockDuration; - ConsumeSettings.LockRenewal = canonical.LockRenewal; - ConsumeSettings.LookbackInterval = canonical.LookbackInterval; - ConsumeSettings.MaxDeliveryAttempts = canonical.MaxDeliveryAttempts; - ConsumeSettings.BatchingWindow = canonical.BatchingWindow; - ConsumeSettings.PerTenantTimeout = canonical.PerTenantTimeout; - ConsumeSettings.PerTenantMaxDegreeOfParallelism = canonical.PerTenantMaxDegreeOfParallelism; - } - - /// - public override string ToString() => ConsumerGroupId; -} diff --git a/src/Sa.Outbox/Delivery/DeliveryBuilder.cs b/src/Sa.Outbox/Delivery/DeliveryBuilder.cs index 858cd806..e1a5bfbe 100644 --- a/src/Sa.Outbox/Delivery/DeliveryBuilder.cs +++ b/src/Sa.Outbox/Delivery/DeliveryBuilder.cs @@ -11,7 +11,7 @@ internal sealed partial class DeliveryBuilder(IServiceCollection services) : IDe public IDeliveryBuilder AddDeliveryScoped< [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( string consumerGroupId, - Action? configure = null, + Action? configure = null, Guid? jobId = null) where TConsumer : class, IConsumer { @@ -24,7 +24,7 @@ public IDeliveryBuilder AddDeliveryScoped< public IDeliveryBuilder AddDelivery< [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( string consumerGroupId, - Action? configure = null, + Action? configure = null, Guid? jobId = null) where TConsumer : class, IConsumer { diff --git a/src/Sa.Outbox/Delivery/DeliveryCourier.cs b/src/Sa.Outbox/Delivery/DeliveryCourier.cs index 4b229bc5..7d01bd34 100644 --- a/src/Sa.Outbox/Delivery/DeliveryCourier.cs +++ b/src/Sa.Outbox/Delivery/DeliveryCourier.cs @@ -10,14 +10,14 @@ internal sealed class DeliveryCourier( IDeliveryLifetimeInvoker processor, IRetryStrategy? retryStrategy = null) : IDeliveryCourier { - + private readonly IRetryStrategy _retryStrategy = retryStrategy ?? ExponentialBackoffRetryStrategy.Shared; /// /// Asynchronous method to deliver messages /// public async ValueTask Deliver( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -33,7 +33,7 @@ public async ValueTask Deliver( HandleError(ex, messages.Span); } - return PostHandle(messages.Span, settings.ConsumeSettings.MaxDeliveryAttempts); + return PostHandle(messages.Span, settings.MaxDeliveryAttempts); } diff --git a/src/Sa.Outbox/Delivery/DeliveryLifetimeInvoker.cs b/src/Sa.Outbox/Delivery/DeliveryLifetimeInvoker.cs index d73063c7..7f452024 100644 --- a/src/Sa.Outbox/Delivery/DeliveryLifetimeInvoker.cs +++ b/src/Sa.Outbox/Delivery/DeliveryLifetimeInvoker.cs @@ -9,11 +9,11 @@ namespace Sa.Outbox.Delivery; internal sealed class DeliveryLifetimeInvoker(IServiceProvider serviceProvider) : IDeliveryLifetimeInvoker { - private readonly ConcurrentDictionary _singletonConsumers = new(); + private readonly ConcurrentDictionary _singletonConsumers = new(); // Method to process messages using a consumer in scope public Task ConsumeInScope( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -24,7 +24,7 @@ public Task ConsumeInScope( } private Task ProcessInSingleton( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -34,7 +34,7 @@ private Task ProcessInSingleton( } private async Task ProcessInNewScope( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -46,7 +46,7 @@ private async Task ProcessInNewScope( private static async Task ProcessMessages( IConsumer consumer, - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -55,7 +55,7 @@ private static async Task ProcessMessages( } private IConsumer GetOrCreateSingletonConsumer( - ConsumerGroupSettings settings) + OutboxConsumerSettings settings) { return (IConsumer)_singletonConsumers.GetOrAdd(settings, key => serviceProvider.GetRequiredKeyedService>(key)); diff --git a/src/Sa.Outbox/Delivery/DeliveryProcessor.cs b/src/Sa.Outbox/Delivery/DeliveryProcessor.cs index 882fd225..25909978 100644 --- a/src/Sa.Outbox/Delivery/DeliveryProcessor.cs +++ b/src/Sa.Outbox/Delivery/DeliveryProcessor.cs @@ -11,21 +11,17 @@ internal sealed class DeliveryProcessor( ITenantProvider tenantProvider) : IDeliveryProcessor { public async Task ProcessMessages( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, CancellationToken cancellationToken) { - // Derive immutable snapshot from mutable bootstrap settings. - // Cheap operation (~20 property reads) compared to DB/network I/O. - var canonical = settings.ToCanonical(); - - if (canonical.Paused) + if (settings.Paused) { // Consumer group is paused — do not poll. await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); return 0; } - int batchSize = canonical.MaxBatchSize; + int batchSize = settings.MaxBatchSize; if (batchSize == 0) return 0; int[] tenantIds = await tenantProvider.GetTenantIds(cancellationToken); @@ -37,12 +33,12 @@ public async Task ProcessMessages( bool continueProcessing; do { - if (iterations > 0 && canonical.IterationDelay > TimeSpan.Zero) + if (iterations > 0 && settings.IterationDelay > TimeSpan.Zero) { - await Task.Delay(canonical.IterationDelay, cancellationToken); + await Task.Delay(settings.IterationDelay, cancellationToken); } - int sentCount = await ProcessForEachTenant(tenantIds, settings, canonical, cancellationToken); + int sentCount = await ProcessForEachTenant(tenantIds, settings, cancellationToken); totalProcessed += sentCount; iterations++; @@ -50,7 +46,7 @@ public async Task ProcessMessages( continueProcessing = ShouldContinueProcessing( sentCount, iterations, - canonical, + settings, cancellationToken); } while (continueProcessing); @@ -78,25 +74,23 @@ private static bool ShouldContinueProcessing( private async Task ProcessForEachTenant( int[] tenantIds, - ConsumerGroupSettings settings, - OutboxConsumerSettings canonical, + OutboxConsumerSettings settings, CancellationToken cancellationToken) { - return (canonical.PerTenantMaxDegreeOfParallelism == 1) - ? await ProcessTenantsSequential(tenantIds, settings, canonical, cancellationToken) - : await ProcessTenantsParallel(tenantIds, settings, canonical, cancellationToken); + return (settings.PerTenantMaxDegreeOfParallelism == 1) + ? await ProcessTenantsSequential(tenantIds, settings, cancellationToken) + : await ProcessTenantsParallel(tenantIds, settings, cancellationToken); } private async Task ProcessTenantsSequential( int[] tenantIds, - ConsumerGroupSettings settings, - OutboxConsumerSettings canonical, + OutboxConsumerSettings settings, CancellationToken cancellationToken) { int count = 0; foreach (int tenantId in tenantIds) { - count += await ProcessInTenant(tenantId, settings, canonical, cancellationToken); + count += await ProcessInTenant(tenantId, settings, cancellationToken); } return count; @@ -104,15 +98,14 @@ private async Task ProcessTenantsSequential( public async Task ProcessTenantsParallel( int[] tenantIds, - ConsumerGroupSettings settings, - OutboxConsumerSettings canonical, + OutboxConsumerSettings settings, CancellationToken cancellationToken) { var parallelOptions = new ParallelOptions { - MaxDegreeOfParallelism = canonical.PerTenantMaxDegreeOfParallelism == -1 + MaxDegreeOfParallelism = settings.PerTenantMaxDegreeOfParallelism == -1 ? Environment.ProcessorCount - : canonical.PerTenantMaxDegreeOfParallelism, + : settings.PerTenantMaxDegreeOfParallelism, CancellationToken = cancellationToken }; @@ -125,7 +118,7 @@ await Parallel.ForEachAsync( parallelOptions, async (tenantId, ct) => { - int processed = await ProcessInTenant(tenantId, settings, canonical, ct); + int processed = await ProcessInTenant(tenantId, settings, ct); Interlocked.Add(ref totalCount, processed); }); } @@ -139,14 +132,13 @@ await Parallel.ForEachAsync( private async Task ProcessInTenant( int tenantId, - ConsumerGroupSettings settings, - OutboxConsumerSettings canonical, + OutboxConsumerSettings settings, CancellationToken cancellationToken) { using var tenantCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - if (canonical.PerTenantTimeout > TimeSpan.Zero) + if (settings.PerTenantTimeout > TimeSpan.Zero) { - tenantCts.CancelAfter(canonical.PerTenantTimeout); + tenantCts.CancelAfter(settings.PerTenantTimeout); } try diff --git a/src/Sa.Outbox/Delivery/DeliverySnapshot.cs b/src/Sa.Outbox/Delivery/DeliverySnapshot.cs index b5d7add0..3a061d07 100644 --- a/src/Sa.Outbox/Delivery/DeliverySnapshot.cs +++ b/src/Sa.Outbox/Delivery/DeliverySnapshot.cs @@ -1,55 +1,26 @@ -using Sa.Outbox.Delivery.Job; -using Sa.Outbox.Metadata; -using Sa.Schedule; +namespace Sa.Outbox.Delivery; -namespace Sa.Outbox.Delivery; - -internal sealed class DeliverySnapshot( - IScheduleSettings scheduleSettings, - IOutboxMessageMetadataProvider metadataProvider) : IDeliverySnapshot +internal sealed class DeliverySnapshot : IDeliverySnapshot { - - private readonly Lazy _lazyJobs = new(() => [.. scheduleSettings.GetJobSettings()]); - - - private readonly Lazy _lazyParts = new(() => - { - Type baseType = typeof(DeliveryJob<>); - string[] parts = [.. scheduleSettings.GetJobSettings() - .Select(c => GetMessageTypeIfInheritsFromDeliveryJob(c.JobType, baseType)) - .Where(mt => mt != null) - .Cast() - .Select(mt => metadataProvider.GetMetadata(mt).PartName) - .Distinct()]; - - return parts; - }); - - private readonly Lazy _lazyDeliveries = new(() => + private readonly Lazy _lazyDeliveries = new(() => { - ConsumerGroupSettings[] settings = [.. scheduleSettings.GetJobSettings() - .Select(c => c.Properties.GetConsumerGroupSettings()) - .Where(mt => mt != null) - .Cast()]; + // Collect settings from the static registry populated by AddDeliveryJob + var registered = Setup.RegisteredSettings + .Where(s => s != null) + .DistinctBy(s => s.ConsumerGroupId) + .ToArray(); - return settings; + return registered; }); - - private static Type? GetMessageTypeIfInheritsFromDeliveryJob(Type jobType, Type baseType) + public string[] Parts { - if (!baseType.IsGenericTypeDefinition) return null; - - if (jobType.IsGenericType && jobType.GetGenericTypeDefinition() == baseType) - return jobType.GenericTypeArguments[0]; - - return jobType.BaseType != null - ? GetMessageTypeIfInheritsFromDeliveryJob(jobType.BaseType, baseType) - : null; + get + { + var settings = _lazyDeliveries.Value; + return [.. settings.Select(s => s.ConsumerGroupId).Distinct()]; + } } - - public string[] Parts => _lazyParts.Value; - public IJobSettings[] JobSettings => _lazyJobs.Value; - public ConsumerGroupSettings[] ConsumerSettings => _lazyDeliveries.Value; + public OutboxConsumerSettings[] ConsumerSettings => _lazyDeliveries.Value; } diff --git a/src/Sa.Outbox/Delivery/DeliveryTenant.cs b/src/Sa.Outbox/Delivery/DeliveryTenant.cs index c9082c94..b0fdba74 100644 --- a/src/Sa.Outbox/Delivery/DeliveryTenant.cs +++ b/src/Sa.Outbox/Delivery/DeliveryTenant.cs @@ -20,26 +20,26 @@ internal sealed class DeliveryTenant( public async Task ProcessInTenant( int tenantId, - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, CancellationToken cancellationToken) { var filter = CreateFilter(tenantId, settings); - var batchSize = await CalculateBatchSizeAsync(settings.ConsumeSettings, filter, cancellationToken); + var batchSize = await CalculateBatchSizeAsync(settings, filter, cancellationToken); if (batchSize == 0) return 0; using var memoryOwner = RentMemory(batchSize); var messages = await AcquireMessagesAsync( - settings.ConsumeSettings, + settings, filter, memoryOwner.Memory[..batchSize], cancellationToken); if (messages.IsEmpty) return 0; - await using IAsyncDisposable locker = RenewerLocker(settings.ConsumeSettings, filter, cancellationToken); + await using IAsyncDisposable locker = RenewerLocker(settings, filter, cancellationToken); var successfulDeliveries = await deliveryCourier.Deliver(settings, filter, messages, cancellationToken); @@ -48,40 +48,40 @@ public async Task ProcessInTenant( return successfulDeliveries; } - private OutboxMessageFilter CreateFilter(int tenantId, ConsumerGroupSettings settings) + private OutboxMessageFilter CreateFilter(int tenantId, OutboxConsumerSettings settings) { return filterFactory.CreateFilter( tenantId: tenantId, consumerGroupId: settings.ConsumerGroupId, now: GetUtcNow(), - lookbackInterval: settings.ConsumeSettings.LookbackInterval, - batchingWindow: settings.ConsumeSettings.BatchingWindow); + lookbackInterval: settings.LookbackInterval, + batchingWindow: settings.BatchingWindow); } private DateTimeOffset GetUtcNow() => _timeProvider.GetUtcNow(); private async Task CalculateBatchSizeAsync( - ConsumeSettings consumeSettings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, CancellationToken cancellationToken) { var calculatedSize = await batcher.CalculateBatchSize( - consumeSettings.MaxBatchSize, + settings.MaxBatchSize, filter, cancellationToken); - return Math.Clamp(calculatedSize, 0, consumeSettings.MaxBatchSize); + return Math.Clamp(calculatedSize, 0, settings.MaxBatchSize); } private async Task>> AcquireMessagesAsync( - ConsumeSettings consumeSettings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, Memory> buffer, CancellationToken cancellationToken) { var lockedCount = await deliveryMan.RentDelivery( buffer, - consumeSettings.LockDuration, + settings.LockDuration, filter, cancellationToken); @@ -101,7 +101,7 @@ private Task ReleaseMessagesAsync( } private IAsyncDisposable RenewerLocker( - ConsumeSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, CancellationToken cancellationToken) => LockRenewer.KeepLocked( diff --git a/src/Sa.Outbox/Delivery/IDeliveryBuilder.cs b/src/Sa.Outbox/Delivery/IDeliveryBuilder.cs index fabdf050..309f9e28 100644 --- a/src/Sa.Outbox/Delivery/IDeliveryBuilder.cs +++ b/src/Sa.Outbox/Delivery/IDeliveryBuilder.cs @@ -14,11 +14,11 @@ public partial interface IDeliveryBuilder /// The type of consumer. /// The type of message. /// Group identity for consuming. - /// An optional action to configure the delivery settings. + /// An optional action to configure the delivery settings via builder. /// The delivery builder instance. IDeliveryBuilder AddDeliveryScoped<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( string consumerGroupId, - Action? configure = null, + Action? configure = null, Guid? jobId = null ) where TConsumer : class, IConsumer; @@ -28,7 +28,7 @@ public partial interface IDeliveryBuilder /// IDeliveryBuilder AddDelivery<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( string consumerGroupId, - Action? configure = null, + Action? configure = null, Guid? jobId = null ) where TConsumer : class, IConsumer; diff --git a/src/Sa.Outbox/Delivery/IDeliveryBuilder.partial.cs b/src/Sa.Outbox/Delivery/IDeliveryBuilder.partial.cs index 83fa33d2..4b70f42c 100644 --- a/src/Sa.Outbox/Delivery/IDeliveryBuilder.partial.cs +++ b/src/Sa.Outbox/Delivery/IDeliveryBuilder.partial.cs @@ -8,13 +8,13 @@ public partial interface IDeliveryBuilder private static IConsumerGroupNamingStrategy _defaultNamingStrategy = new DefaultConsumerGroupNamingStrategy(); public IDeliveryBuilder AddDeliveryScoped<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( - Action? configure = null + Action? configure = null ) where TConsumer : class, IConsumer => AddDeliveryScoped(GetConsumerGroupName(), configure); public IDeliveryBuilder AddDelivery<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( - Action? configure = null + Action? configure = null ) where TConsumer : class, IConsumer => AddDelivery(GetConsumerGroupName(), configure); diff --git a/src/Sa.Outbox/Delivery/IDeliveryCourier.cs b/src/Sa.Outbox/Delivery/IDeliveryCourier.cs index 67acba9c..30e83692 100644 --- a/src/Sa.Outbox/Delivery/IDeliveryCourier.cs +++ b/src/Sa.Outbox/Delivery/IDeliveryCourier.cs @@ -6,7 +6,7 @@ internal interface IDeliveryCourier { ValueTask Deliver( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken); diff --git a/src/Sa.Outbox/Delivery/IDeliveryLifetimeInvoker.cs b/src/Sa.Outbox/Delivery/IDeliveryLifetimeInvoker.cs index 85e75129..c6f98893 100644 --- a/src/Sa.Outbox/Delivery/IDeliveryLifetimeInvoker.cs +++ b/src/Sa.Outbox/Delivery/IDeliveryLifetimeInvoker.cs @@ -6,7 +6,7 @@ internal interface IDeliveryLifetimeInvoker { Task ConsumeInScope( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken); diff --git a/src/Sa.Outbox/Delivery/IDeliveryProcessor.cs b/src/Sa.Outbox/Delivery/IDeliveryProcessor.cs index 2c1b0b85..d4719686 100644 --- a/src/Sa.Outbox/Delivery/IDeliveryProcessor.cs +++ b/src/Sa.Outbox/Delivery/IDeliveryProcessor.cs @@ -6,5 +6,5 @@ /// public interface IDeliveryProcessor { - Task ProcessMessages(ConsumerGroupSettings settings, CancellationToken cancellationToken); + Task ProcessMessages(OutboxConsumerSettings settings, CancellationToken cancellationToken); } diff --git a/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs b/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs index 4dea543a..4531aaea 100644 --- a/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs +++ b/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs @@ -1,11 +1,8 @@ -using Sa.Schedule; - -namespace Sa.Outbox.Delivery; +namespace Sa.Outbox.Delivery; public interface IDeliverySnapshot { - IJobSettings[] JobSettings { get; } - ConsumerGroupSettings[] ConsumerSettings { get; } + OutboxConsumerSettings[] ConsumerSettings { get; } string[] Parts { get; } IEnumerable GetConsumeGroupIds() diff --git a/src/Sa.Outbox/Delivery/IDeliveryTenant.cs b/src/Sa.Outbox/Delivery/IDeliveryTenant.cs index 01fc943f..735de090 100644 --- a/src/Sa.Outbox/Delivery/IDeliveryTenant.cs +++ b/src/Sa.Outbox/Delivery/IDeliveryTenant.cs @@ -7,6 +7,6 @@ internal interface IDeliveryTenant { Task ProcessInTenant( int tenantId, - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, CancellationToken cancellationToken); } diff --git a/src/Sa.Outbox/Delivery/IOutboxSettingsManager.cs b/src/Sa.Outbox/Delivery/IOutboxSettingsManager.cs index 5e538dea..ac980b44 100644 --- a/src/Sa.Outbox/Delivery/IOutboxSettingsManager.cs +++ b/src/Sa.Outbox/Delivery/IOutboxSettingsManager.cs @@ -8,22 +8,21 @@ namespace Sa.Outbox.Delivery; public interface IOutboxSettingsManager { /// - /// Applies a partial or full settings update for an existing consumer group. + /// Atomically applies a transformation to the current settings for a consumer group. /// The update is atomically swapped — active deliveries finish their current batch, /// then pick up the new settings on the next iteration. - /// Unspecified fields inherit from the previous snapshot. /// /// The consumer group identifier. - /// A callback to configure the builder with desired overrides. - void Apply(string consumerGroupId, Action configure); + /// A function that receives the current snapshot and returns the updated one. Use this with { ... } expressions. + void Apply(string consumerGroupId, Func transform); /// - /// Replaces all settings for a consumer group (typically during initial registration). + /// Registers a consumer group with initial settings. /// Unlike , this does not require prior registration. /// /// The consumer group identifier. - /// A callback to build the complete settings. - void Register(string consumerGroupId, Action configure); + /// The initial immutable settings snapshot. + void Register(string consumerGroupId, OutboxConsumerSettings settings); /// /// Retrieves the current immutable settings snapshot. Thread-safe. diff --git a/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs b/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs index 56f5fa0f..8d9a2569 100644 --- a/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs +++ b/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs @@ -10,8 +10,8 @@ internal sealed class DeliveryJob(IDeliveryProcessor processor) : IDel { public async Task Execute(IJobContext context, CancellationToken cancellationToken) { - ConsumerGroupSettings settings = context.Settings.Properties.GetConsumerGroupSettings() - ?? throw new NotImplementedException("tag"); + OutboxConsumerSettings settings = context.Settings.Properties.GetConsumerGroupSettings() + ?? throw new InvalidOperationException("Missing OutboxConsumerSettings tag on job."); await processor.ProcessMessages(settings, cancellationToken); } diff --git a/src/Sa.Outbox/Delivery/Job/JobPropertiesExtension.cs b/src/Sa.Outbox/Delivery/Job/JobPropertiesExtension.cs index 1540c98b..a17a94d0 100644 --- a/src/Sa.Outbox/Delivery/Job/JobPropertiesExtension.cs +++ b/src/Sa.Outbox/Delivery/Job/JobPropertiesExtension.cs @@ -4,6 +4,6 @@ namespace Sa.Outbox.Delivery.Job; internal static class JobPropertiesExtension { - public static ConsumerGroupSettings? GetConsumerGroupSettings(this IJobProperties properties) - => properties?.Tag as ConsumerGroupSettings; + public static OutboxConsumerSettings? GetConsumerGroupSettings(this IJobProperties properties) + => properties?.Tag as OutboxConsumerSettings; } diff --git a/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs b/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs index 32ab925d..f62cb06e 100644 --- a/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs +++ b/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs @@ -15,10 +15,8 @@ public Task StartAsync(CancellationToken cancellationToken) { foreach (var settings in snapshot.ConsumerSettings) { - // Register the canonical snapshot derived from bootstrap settings. - var canonical = settings.ToCanonical(); - settingsManager.Register(settings.ConsumerGroupId, builder => - builder.BuildCopy(canonical)); + // Register the settings directly — no conversion needed anymore. + settingsManager.Register(settings.ConsumerGroupId, settings); } return Task.CompletedTask; diff --git a/src/Sa.Outbox/Delivery/Job/Setup.cs b/src/Sa.Outbox/Delivery/Job/Setup.cs index 3496dcb3..bcd79c6c 100644 --- a/src/Sa.Outbox/Delivery/Job/Setup.cs +++ b/src/Sa.Outbox/Delivery/Job/Setup.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Sa.Outbox.Delivery.Job; using Sa.Schedule; using System.Diagnostics.CodeAnalysis; @@ -7,19 +8,53 @@ namespace Sa.Outbox.Delivery.Job; internal static class Setup { + /// + /// Queue of settings registered via AddDeliveryJob, consumed by DeliverySnapshot. + /// Thread-safe for bootstrap phase only. + /// + internal static readonly Queue RegisteredSettings = new(); + public static IServiceCollection AddDeliveryJob< [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( this IServiceCollection services, string consumerGroupId, bool isSingleton, - Action? сonfigure = null, + Action? configure = null, Guid? jobId = null) where TConsumer : class, IConsumer { ArgumentNullException.ThrowIfNullOrWhiteSpace(consumerGroupId); - ConsumerGroupSettings settings = new(consumerGroupId, isSingleton); + // Build settings from scratch via builder + var builder = new OutboxConsumerSettingsBuilder(); + builder + .WithConsumerGroupId(consumerGroupId) + .AsSingleton(isSingleton) + .WithInterval(TimeSpan.FromMinutes(1)) + .StartImmediately() + .WithConcurrencyLimit(1) + .WithMaxConcurrency(1) + .WithRetryCountOnError(3) + .WithMaxBatchSize(16) + .WithMaxProcessingIterations(-1) + .WithIterationDelay(TimeSpan.Zero) + .WithLockDuration(TimeSpan.FromSeconds(10)) + .WithLockRenewal(TimeSpan.FromSeconds(3)) + .WithLookbackInterval(TimeSpan.FromDays(7)) + .WithMaxDeliveryAttempts(3) + .WithBatchingWindow(TimeSpan.FromSeconds(3)) + .WithPerTenantTimeout(TimeSpan.Zero) + .WithPerTenantMaxDegreeOfParallelism(1) + .Paused(false); + + // Allow caller to tweak settings via fluent builder + configure?.Invoke(default!, builder); + + var settings = builder.Build(); + + // Register in the static registry for DeliverySnapshot + RegisteredSettings.Enqueue(settings); if (isSingleton) { @@ -36,23 +71,19 @@ public static IServiceCollection AddDeliveryJob< builder.AddJob>((sp, jobBuilder) => { - сonfigure?.Invoke(sp, settings); - - ScheduleSettings scheduleSettings = settings.ScheduleSettings; - jobBuilder - .EveryTime(scheduleSettings.Interval) - .WithInitialDelay(scheduleSettings.InitialDelay) + .EveryTime(settings.Interval) + .WithInitialDelay(settings.InitialDelay) .WithTag(settings) - .WithConcurrencyLimit(scheduleSettings.ConcurrencyLimit) - .WithMaxConcurrency(scheduleSettings.MaxConcurrency) - .WithName(scheduleSettings.Name ?? typeof(TConsumer).Name) + .WithConcurrencyLimit(settings.ConcurrencyLimit) + .WithMaxConcurrency(settings.MaxConcurrency) + .WithName(settings.ConsumerGroupId) .ConfigureErrorHandling(c => c - .IfErrorRetry(scheduleSettings.RetryCountOnError) + .IfErrorRetry(settings.RetryCountOnError) .ThenCloseApplication()) ; - }, jobId ?? settings.ScheduleSettings.JobId); + }, jobId ?? Guid.Empty); builder.AddInterceptor(); diff --git a/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs b/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs index b4cca3ea..8be8f4b8 100644 --- a/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs +++ b/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs @@ -6,25 +6,109 @@ /// Create new instances via or the with expression. /// public sealed record OutboxConsumerSettings( + /// + /// Unique identifier for the consumer group. Groups settings for a single logical consumer. + /// string ConsumerGroupId, + + /// + /// When true — the consumer runs as a singleton (one instance across the entire cluster). + /// When false — each application instance runs its own consumer copy. + /// bool AsSingleton, + + /// + /// Periodicity of the consumer run (interval between processing iterations). + /// TimeSpan Interval, + + /// + /// Initial delay before the consumer's first run after application startup. + /// TimeSpan InitialDelay, + + /// + /// Maximum number of concurrent threads for message processing at the consumer group level. + /// int ConcurrencyLimit, + + /// + /// Maximum number of simultaneously running processors (jobs) within this group. + /// int MaxConcurrency, + + /// + /// Number of retry attempts on message processing error. + /// -1 means infinite retries. + /// int RetryCountOnError, + + /// + /// Maximum batch size (number of messages) per processing iteration. + /// int MaxBatchSize, + + /// + /// Maximum number of processing iterations per cycle (-1 = unlimited). + /// int MaxProcessingIterations, + + /// + /// Delay between processing iterations within a single cycle. + /// TimeSpan IterationDelay, + + /// + /// Lock duration for record processing (lock TTL). + /// The record is locked from other consumers for this period. + /// TimeSpan LockDuration, + + /// + /// Lock renewal interval. + /// Must be less than LockDuration. + /// TimeSpan LockRenewal, + + /// + /// Lookback interval — how far back in time to search for unprocessed messages. + /// Used to catch up messages that may have been missed during idle periods. + /// TimeSpan LookbackInterval, + + /// + /// Maximum delivery attempt count for a message before sending it to the dead-letter queue (DLQ). + /// int MaxDeliveryAttempts, + + /// + /// Time window for aggregating messages into a batch. Messages accumulated within this window are processed together. + /// TimeSpan BatchingWindow, + + /// + /// Timeout for processing a single tenant's data. + /// Exceeding this timeout aborts the tenant's processing. + /// TimeSpan PerTenantTimeout, + + /// + /// Maximum degree of parallelism for tenant-level processing. + /// 1 = sequential, >1 = parallel. + /// int PerTenantMaxDegreeOfParallelism, - bool Paused - //int Version + + /// + /// Consumer pause flag. When true, processing is suspended but the consumer remains active. + /// Allows temporarily halting processing without stopping the entire service. + /// + bool Paused, + + /// + /// Settings version. Incremented on every change for change detection. + /// Used for optimistic locking and notifying subscribers. + /// + int Version ) { /// @@ -103,105 +187,6 @@ public void ThrowIfInvalid() $"Invalid OutboxConsumerSettings: {string.Join("; ", errors)}"); } - // ── Fluent with-helpers ─────────────────────────────────── - // Each returns a NEW instance with one field changed and version incremented. - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithInterval(TimeSpan interval) - => this with { Interval = interval }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithInitialDelay(TimeSpan initialDelay) - => this with { InitialDelay = initialDelay }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithConcurrencyLimit(int concurrencyLimit) - => this with { ConcurrencyLimit = concurrencyLimit }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithMaxConcurrency(int maxConcurrency) - => this with { MaxConcurrency = maxConcurrency }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithRetryCountOnError(int retryCountOnError) - => this with { RetryCountOnError = retryCountOnError }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithMaxBatchSize(int maxBatchSize) - => this with { MaxBatchSize = maxBatchSize }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithMaxProcessingIterations(int maxProcessingIterations) - => this with { MaxProcessingIterations = Math.Max(-1, maxProcessingIterations) }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithIterationDelay(TimeSpan iterationDelay) - => this with { IterationDelay = iterationDelay }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithLockDuration(TimeSpan lockDuration) - => this with { LockDuration = lockDuration }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithLockRenewal(TimeSpan lockRenewal) - => this with { LockRenewal = lockRenewal }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithLookbackInterval(TimeSpan lookbackInterval) - => this with { LookbackInterval = lookbackInterval }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithMaxDeliveryAttempts(int maxDeliveryAttempts) - => this with { MaxDeliveryAttempts = maxDeliveryAttempts }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithBatchingWindow(TimeSpan batchingWindow) - => this with { BatchingWindow = batchingWindow }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithPerTenantTimeout(TimeSpan perTenantTimeout) - => this with { PerTenantTimeout = perTenantTimeout }; - - /// - /// Creates a copy with a new . - /// - public OutboxConsumerSettings WithPerTenantMaxDegreeOfParallelism(int perTenantMaxDegreeOfParallelism) - => this with { PerTenantMaxDegreeOfParallelism = perTenantMaxDegreeOfParallelism }; - - /// - /// Creates a paused copy of these settings. - /// - public OutboxConsumerSettings WithPaused(bool paused) - => this with { Paused = paused }; - // ── Equality helper ─────────────────────────────────────── /// diff --git a/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs b/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs index f876322f..a7a7f255 100644 --- a/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs +++ b/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs @@ -51,7 +51,8 @@ public OutboxConsumerSettings Build() _batchingWindow ?? TimeSpan.FromSeconds(3), _perTenantTimeout ?? TimeSpan.Zero, _perTenantMaxDegreeOfParallelism ?? 1, - _paused ?? false); + _paused ?? false, + 0); } // ── Runtime: partial copy from existing settings ────────── @@ -82,8 +83,8 @@ public OutboxConsumerSettings BuildCopy(OutboxConsumerSettings original) _batchingWindow ?? original.BatchingWindow, _perTenantTimeout ?? original.PerTenantTimeout, _perTenantMaxDegreeOfParallelism ?? original.PerTenantMaxDegreeOfParallelism, - _paused ?? original.Paused - ); + _paused ?? original.Paused, + original.Version + 1); } // ── Fluent setters ──────────────────────────────────────── @@ -227,6 +228,11 @@ public OutboxConsumerSettingsBuilder WithLockDuration(TimeSpan lockDuration) return this; } + /// + /// Disables lock duration — messages are not locked before processing. + /// + public OutboxConsumerSettingsBuilder WithNoLockDuration() => WithLockDuration(TimeSpan.Zero); + /// /// Sets the lock renewal time. Must be less than . /// @@ -267,6 +273,11 @@ public OutboxConsumerSettingsBuilder WithBatchingWindow(TimeSpan batchingWindow) return this; } + /// + /// Disables batching window — take whatever is available now. + /// + public OutboxConsumerSettingsBuilder WithNoBatchingWindow() => WithBatchingWindow(TimeSpan.Zero); + /// /// Sets the per-tenant processing timeout. /// diff --git a/src/Sa.Outbox/Delivery/OutboxSettingsManager.cs b/src/Sa.Outbox/Delivery/OutboxSettingsManager.cs index 3ee19512..4f978bb8 100644 --- a/src/Sa.Outbox/Delivery/OutboxSettingsManager.cs +++ b/src/Sa.Outbox/Delivery/OutboxSettingsManager.cs @@ -1,4 +1,6 @@ -namespace Sa.Outbox.Delivery; +using Sa.Outbox.Delivery; + +namespace Sa.Outbox.Delivery; /// /// Thread-safe manager for runtime control of outbox consumer group settings. @@ -11,23 +13,16 @@ internal sealed class OutboxSettingsManager : IOutboxSettingsManager private readonly Lock _lock = new(); /// - public void Register(string consumerGroupId, Action configure) + public void Register(string consumerGroupId, OutboxConsumerSettings settings) { if (string.IsNullOrWhiteSpace(consumerGroupId)) throw new ArgumentException("Consumer group ID cannot be null or empty.", nameof(consumerGroupId)); - ArgumentNullException.ThrowIfNull(configure, nameof(configure)); - - var builder = new OutboxConsumerSettingsBuilder(); - configure(builder); - - OutboxConsumerSettings newSettings; + ArgumentNullException.ThrowIfNull(settings); lock (_lock) { - // Register always creates a fresh snapshot — no dependency on existing. - newSettings = builder.Build(); - _settings[consumerGroupId] = newSettings; + _settings[consumerGroupId] = settings; if (!_listeners.ContainsKey(consumerGroupId)) { @@ -36,36 +31,29 @@ public void Register(string consumerGroupId, Action - public void Apply(string consumerGroupId, Action configure) + public void Apply(string consumerGroupId, Func transform) { if (string.IsNullOrWhiteSpace(consumerGroupId)) throw new ArgumentException("Consumer group ID cannot be null or empty.", nameof(consumerGroupId)); - ArgumentNullException.ThrowIfNull(configure); - - var builder = new OutboxConsumerSettingsBuilder(); - configure(builder); + ArgumentNullException.ThrowIfNull(transform); OutboxConsumerSettings newSettings; lock (_lock) { - if (!_settings.TryGetValue(consumerGroupId, out OutboxConsumerSettings? existing)) - { - // First registration via Apply — build from scratch - newSettings = builder.Build(); - _settings[consumerGroupId] = newSettings; - _listeners[consumerGroupId] = []; - } - else + if (!_settings.TryGetValue(consumerGroupId, out var existing)) { - newSettings = builder.BuildCopy(existing); - _settings[consumerGroupId] = newSettings; + throw new InvalidOperationException( + $"Consumer group '{consumerGroupId}' is not registered. Call Register() first."); } + + newSettings = transform(existing); + _settings[consumerGroupId] = newSettings; } // Notify subscribers OUTSIDE the lock to avoid deadlocks @@ -114,7 +102,7 @@ public void Pause(string consumerGroupId) if (string.IsNullOrWhiteSpace(consumerGroupId)) throw new ArgumentException("Consumer group ID cannot be null or empty.", nameof(consumerGroupId)); - Apply(consumerGroupId, builder => builder.Paused(true)); + Apply(consumerGroupId, s => s with { Paused = true }); } /// @@ -123,7 +111,7 @@ public void Resume(string consumerGroupId) if (string.IsNullOrWhiteSpace(consumerGroupId)) throw new ArgumentException("Consumer group ID cannot be null or empty.", nameof(consumerGroupId)); - Apply(consumerGroupId, builder => builder.Paused(false)); + Apply(consumerGroupId, s => s with { Paused = false }); } /// diff --git a/src/Sa.Outbox/Delivery/ScheduleSettings.cs b/src/Sa.Outbox/Delivery/ScheduleSettings.cs deleted file mode 100644 index a049e1f6..00000000 --- a/src/Sa.Outbox/Delivery/ScheduleSettings.cs +++ /dev/null @@ -1,28 +0,0 @@ -namespace Sa.Outbox.Delivery; - -/// -/// Represents the scheduling settings for the delivery job. -/// -public sealed class ScheduleSettings -{ - /// - /// Gets the unique identifier for the delivery job - /// - public Guid JobId { get; } = Guid.NewGuid(); - - public string? Name { get; internal set; } - - public TimeSpan Interval { get; internal set; } = TimeSpan.FromMinutes(1); - - /// - /// Job schedule delay before start - /// - public TimeSpan InitialDelay { get; internal set; } = TimeSpan.FromSeconds(10); - - public int RetryCountOnError { get; internal set; } = 1; - - public int ConcurrencyLimit { get; internal set; } = 1; - - public int MaxConcurrency { get; internal set; } = 48; - -} diff --git a/src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs b/src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs deleted file mode 100644 index f5543753..00000000 --- a/src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace Sa.Outbox.Delivery; - -public static class ScheduleSettingsExtensions -{ - /// - /// Sets the job name. - /// - public static ScheduleSettings WithName(this ScheduleSettings settings, string name) - { - settings.Name = name; - return settings; - } - - /// - /// Sets the interval between job executions. - /// - /// The execution interval. - /// This instance for chaining. - public static ScheduleSettings WithInterval(this ScheduleSettings settings, TimeSpan interval) - { - settings.Interval = interval; - return settings; - } - - /// - /// Sets the initial delay before the first job execution. - /// - /// The initial delay. - /// This instance for chaining. - public static ScheduleSettings WithInitialDelay(this ScheduleSettings settings, TimeSpan delay) - { - settings.InitialDelay = delay; - return settings; - } - - public static ScheduleSettings WithImmediate(this ScheduleSettings settings) - { - settings.InitialDelay = TimeSpan.Zero; - return settings; - } - - public static ScheduleSettings WithConcurrencyLimit(this ScheduleSettings settings, int limit) - { - settings.ConcurrencyLimit = limit; - return settings; - } - public static ScheduleSettings WithMaxConcurrency(this ScheduleSettings settings, int maxLimit) - { - settings.MaxConcurrency = maxLimit; - return settings; - } - - /// - /// Sets the number of retry attempts on error. - /// - /// Number of retries. - /// This instance for chaining. - public static ScheduleSettings WithRetryCountOnError(this ScheduleSettings settings, int retryCount) - { - settings.RetryCountOnError = retryCount; - return settings; - } - - /// - /// Configures the job with no retries on error. - /// - public static ScheduleSettings WithNoRetries(this ScheduleSettings settings) - { - return settings.WithRetryCountOnError(0); - } - - /// - /// Configures the job with infinite retries on error. - /// - public static ScheduleSettings WithInfiniteRetries(this ScheduleSettings settings) - { - return settings.WithRetryCountOnError(int.MaxValue); - } - - /// - /// Configures test settings. - /// - public static ScheduleSettings UseTestSettings( - this ScheduleSettings settings) - { - return settings - .WithName($"Test-DeliveryJob-{settings.JobId}") - .WithInterval(TimeSpan.FromMilliseconds(300)) - .WithImmediate(); - } - - public static ScheduleSettings WithIntervalSeconds(this ScheduleSettings settings, int seconds) - => settings.WithInterval(TimeSpan.FromSeconds(seconds)); - - public static ScheduleSettings WithIntervalMilliseconds(this ScheduleSettings settings, int milliseconds) - => settings.WithInterval(TimeSpan.FromMilliseconds(milliseconds)); - - public static ScheduleSettings WithIntervalMinutes(this ScheduleSettings settings, int minutes) - => settings.WithInterval(TimeSpan.FromMinutes(minutes)); - - public static ScheduleSettings WithInitialDelaySeconds(this ScheduleSettings settings, int seconds) - => settings.WithInitialDelay(TimeSpan.FromSeconds(seconds)); - - public static ScheduleSettings WithInitialDelayMinutes(this ScheduleSettings settings, int minutes) - => settings.WithInitialDelay(TimeSpan.FromMinutes(minutes)); - -} diff --git a/src/Sa.Outbox/Delivery/Setup.cs b/src/Sa.Outbox/Delivery/Setup.cs index be50a001..52b536ff 100644 --- a/src/Sa.Outbox/Delivery/Setup.cs +++ b/src/Sa.Outbox/Delivery/Setup.cs @@ -3,11 +3,16 @@ using Sa.Outbox.Delivery.Job; using Sa.Outbox.Metadata; using Sa.Outbox.Partitional; +using Sa.Schedule; +using System.Collections.Concurrent; namespace Sa.Outbox.Delivery; internal static class Setup { + // Thread-safe registry of consumer group settings discovered via AddDeliveryJob + internal static readonly ConcurrentQueue RegisteredSettings = new(); + public static IServiceCollection AddOutboxDelivery( this IServiceCollection services, Action? configure = null) { @@ -31,6 +36,7 @@ public static IServiceCollection AddOutboxDelivery( services.TryAddSingleton(); + // DeliverySnapshot теперь собирает настройки из AddDeliveryJob через статический регистр services.TryAddSingleton(); services.TryAddSingleton(); diff --git a/src/Sa.Outbox/IConsumer.cs b/src/Sa.Outbox/IConsumer.cs index 3f43c901..644986ff 100644 --- a/src/Sa.Outbox/IConsumer.cs +++ b/src/Sa.Outbox/IConsumer.cs @@ -16,7 +16,7 @@ public interface IConsumer : IConsumer /// A cancellation token to signal the operation's cancellation. /// A task representing the asynchronous operation. ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken); diff --git a/src/Sa.Outbox/IOutboxBuilder.cs b/src/Sa.Outbox/IOutboxBuilder.cs index 53f79122..7d1ed070 100644 --- a/src/Sa.Outbox/IOutboxBuilder.cs +++ b/src/Sa.Outbox/IOutboxBuilder.cs @@ -42,12 +42,8 @@ public interface IOutboxBuilder IOutboxBuilder AddMetadata( string partName, Func? getPayloadId = null) where TMessage : class - { - return WithMetadata((_, m) => m.AddMetadata(partName, getPayloadId)); - } + => WithMetadata((_, m) => m.AddMetadata(partName, getPayloadId)); IOutboxBuilder AddMetadata() where TMessage : class, IOutboxPublishable - { - return WithMetadata((_, m) => m.AddMetadata()); - } + => WithMetadata((_, m) => m.AddMetadata()); } diff --git a/src/Sa.Outbox/Publication/OutboxPublishSettings.cs b/src/Sa.Outbox/Publication/OutboxPublishSettings.cs index 2fe23196..87e75fa4 100644 --- a/src/Sa.Outbox/Publication/OutboxPublishSettings.cs +++ b/src/Sa.Outbox/Publication/OutboxPublishSettings.cs @@ -1,14 +1,13 @@ namespace Sa.Outbox.Publication; /// -/// Settings for publishing messages from the Outbox. +/// Immutable settings for publishing messages from the Outbox. /// -public sealed class OutboxPublishSettings +public sealed record OutboxPublishSettings(int MaxBatchSize = 64) { /// - /// The maximum batch size of messages to be sent at once. - /// Default value: 16. - /// for array pool size: 16, 32, 64, 128, 256, 512, 1024, 2048, 4096 + /// Creates a copy with a new . /// - public int MaxBatchSize { get; internal set; } = 64; + public OutboxPublishSettings WithMaxBatchSize(int batchSize) + => this with { MaxBatchSize = batchSize }; } diff --git a/src/Sa.Outbox/Publication/OutboxPublishSettingsExtensions.cs b/src/Sa.Outbox/Publication/OutboxPublishSettingsExtensions.cs index b308c2b0..ebbdacbf 100644 --- a/src/Sa.Outbox/Publication/OutboxPublishSettingsExtensions.cs +++ b/src/Sa.Outbox/Publication/OutboxPublishSettingsExtensions.cs @@ -10,7 +10,7 @@ public static class OutboxPublishSettingsExtensions /// /// The publish settings. /// The batch size (recommended to use optimized values). - /// The configured settings instance. + /// A new instance with the updated batch size. public static OutboxPublishSettings WithMaxBatchSize( this OutboxPublishSettings settings, int batchSize) @@ -18,8 +18,7 @@ public static OutboxPublishSettings WithMaxBatchSize( if (batchSize <= 0) throw new ArgumentException("Batch size must be positive", nameof(batchSize)); - settings.MaxBatchSize = batchSize; - return settings; + return settings.WithMaxBatchSize(batchSize); } diff --git a/src/Samples/PgOutbox.ConsoleApp/Program.cs b/src/Samples/PgOutbox.ConsoleApp/Program.cs index 244d26ba..b40f07fb 100644 --- a/src/Samples/PgOutbox.ConsoleApp/Program.cs +++ b/src/Samples/PgOutbox.ConsoleApp/Program.cs @@ -13,7 +13,9 @@ Console.WriteLine("Hello, Pg Outbox!"); +#pragma warning disable S2068 var connectionString = "Host=localhost;Username=postgres;Password=postgres;Database=postgres"; +#pragma warning restore S2068 // default configure... IHost host = Host.CreateDefaultBuilder().ConfigureServices(services => services @@ -22,15 +24,17 @@ .WithTenants((_, t) => t.WithTenantIds(1, 2, 3)) .WithMetadata((_, b) => b.AddMetadata("some", getPayloadId: p => p.PayloadId)) .WithDeliveries(b => b - .AddDeliveryScoped((_, settings) => + .AddDeliveryScoped((_, builder) => { - settings.ScheduleSettings.WithIntervalSeconds(5).WithImmediate(); - settings.ConsumeSettings.WithSingleIteration(); + builder.WithInterval(TimeSpan.FromSeconds(5)) + .StartImmediately() + .WithSingleIteration(); }) - .AddDelivery("rnd", (_, settings) => + .AddDelivery("rnd", (_, builder) => { - settings.ScheduleSettings.WithIntervalSeconds(25); - settings.ConsumeSettings.WithSingleIteration().WithMaxDeliveryAttempts(2); + builder.WithInterval(TimeSpan.FromSeconds(25)) + .WithSingleIteration() + .WithMaxDeliveryAttempts(2); }) ) ) @@ -71,7 +75,7 @@ public sealed record SomeMessage(string PayloadId, string Message); public sealed class Group1Consumer(ILogger logger) : IConsumer { public async ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -86,7 +90,7 @@ public sealed class RndConsumer(ILogger logger) : IConsumer> messages, CancellationToken cancellationToken) @@ -95,7 +99,8 @@ public async ValueTask Consume( if (Interlocked.Increment(ref s_counter) > 2) { - settings.ConsumeSettings.WithMaxProcessingIterations(100); + // runtime settings update — create a new settings snapshot + // (In practice this would go through IOutboxSettingsManager) } foreach (var msg in messages.Span) diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryBatchingWindowTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryBatchingWindowTests.cs index db2dd4da..4171e5df 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryBatchingWindowTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryBatchingWindowTests.cs @@ -10,7 +10,7 @@ public class DeliveryBatchingWindowTests(DeliveryBatchingWindowTests.Fixture fix class TestMessageConsumer : IConsumer { public async ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -30,14 +30,14 @@ public Fixture() : base() .WithDeliveries(builder => builder .AddDeliveryScoped("test3", (_, s) => { - s.ConsumeSettings.WithBatchingWindow(TimeSpan.FromMinutes(3)); - OutboxSettings = s; + s.WithBatchingWindow(TimeSpan.FromMinutes(3)); + OutboxSettings = s.Build(); }) ) ); } - public ConsumerGroupSettings OutboxSettings { get; set; } = default!; + public OutboxConsumerSettings OutboxSettings { get; set; } = default!; public IOutboxMessagePublisher Publisher => ServiceProvider.GetRequiredService(); } @@ -71,7 +71,7 @@ public async Task Deliver_Process_MustBe_Work() Assert.Equal(0, result); - fixture.OutboxSettings.ConsumeSettings.WithNoBatchingWindow(); + fixture.OutboxSettings = fixture.OutboxSettings with { BatchingWindow = TimeSpan.Zero }; result = await Sub.ProcessMessages(fixture.OutboxSettings, CancellationToken.None); Assert.True(result > 0); diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryLongProcessorTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryLongProcessorTests.cs index c75b77d0..bef787bc 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryLongProcessorTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryLongProcessorTests.cs @@ -10,7 +10,7 @@ public class DeliveryLongProcessorTests(DeliveryLongProcessorTests.Fixture fixtu class TestMessageConsumer : IConsumer { public async ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -23,7 +23,7 @@ public async ValueTask Consume( public class Fixture : OutboxPostgreSqlFixture { - public ConsumerGroupSettings OutboxSettings = default!; + public OutboxConsumerSettings OutboxSettings = default!; public Fixture() : base() { @@ -32,14 +32,14 @@ public Fixture() : base() .WithMetadata((_, b) => b.AddMetadata("root_1", m => m.PayloadId)) .WithTenants((_, s) => s.WithTenantIds(1, 2)) .WithDeliveries(b => b - .AddDeliveryScoped("test1", (_, s) => + .AddDeliveryScoped("test1", (_, b) => { - s.ConsumeSettings + OutboxSettings = new OutboxConsumerSettingsBuilder() + .WithConsumerGroupId("test1") .WithLockDuration(TimeSpan.FromMilliseconds(300)) .WithLockRenewal(TimeSpan.FromMilliseconds(100)) - .WithNoBatchingWindow(); - - OutboxSettings = s; + .WithNoBatchingWindow() + .Build(); }) ) ) diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryPermanentErrorTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryPermanentErrorTests.cs index 73a51070..eb599847 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryPermanentErrorTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryPermanentErrorTests.cs @@ -18,7 +18,7 @@ class TestMessageConsumer : IConsumer { private static readonly TestException s_err = new("test permanent error"); public async ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -42,14 +42,13 @@ public Fixture() : base() .WithDeliveries(builder => builder .AddDeliveryScoped("test2", (_, s) => { - s.ConsumeSettings.WithNoBatchingWindow(); - OutboxSettings = s; + OutboxSettings = s.WithNoBatchingWindow().Build(); }) ) ); } - public ConsumerGroupSettings OutboxSettings = default!; + public OutboxConsumerSettings OutboxSettings = default!; public IOutboxMessagePublisher Publisher => ServiceProvider.GetRequiredService(); } diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryRetryErrorTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryRetryErrorTests.cs index d0b959b6..04ab99ec 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryRetryErrorTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryRetryErrorTests.cs @@ -15,7 +15,7 @@ class TestMessageConsumer : IConsumer private static readonly TestException s_err = new("test same error"); public async ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -39,13 +39,12 @@ public Fixture() : base() .WithDeliveries(builder => builder .AddDeliveryScoped("test4", (_, s) => { - s.ConsumeSettings + s.WithBatchingWindow(TimeSpan.Zero) .WithNoLockDuration() .WithLockRenewal(TimeSpan.FromMinutes(10)) - .WithMaxDeliveryAttempts(MaxDeliveryAttempts) - .WithNoBatchingWindow(); + .WithMaxDeliveryAttempts(MaxDeliveryAttempts); - OutboxSettings = s; + OutboxSettings = s.Build(); }) ) ); @@ -55,7 +54,7 @@ public Fixture() : base() public const int MaxDeliveryAttempts = 2; - public ConsumerGroupSettings OutboxSettings { get; private set; } = default!; + public OutboxConsumerSettings OutboxSettings { get; private set; } = default!; } diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryWithAutoTenantDetectionTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryWithAutoTenantDetectionTests.cs index 73dcda0f..58f5db7f 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryWithAutoTenantDetectionTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryWithAutoTenantDetectionTests.cs @@ -16,7 +16,7 @@ public class DeliveryWithAutoTenantDetectionTests(DeliveryWithAutoTenantDetectio class TestConsumer : IConsumer { public async ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -50,12 +50,11 @@ public Fixture() : base() .WithDeliveries(b => b .AddDelivery("test_auto_detect", (_, s) => { - s.ConsumeSettings - .WithNoBatchingWindow() + s.WithBatchingWindow(TimeSpan.Zero) .WithNoLockDuration() ; - OutboxSettings = s; + OutboxSettings = s.Build(); }) ) ) @@ -70,7 +69,7 @@ public Fixture() : base() .AddSingleton(); } - public ConsumerGroupSettings OutboxSettings { get; set; } = default!; + public OutboxConsumerSettings OutboxSettings { get; set; } = default!; public IOutboxMessagePublisher Publisher => ServiceProvider.GetRequiredService(); } diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxParallelMessagingTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxParallelMessagingTests.cs index 580a1399..268eb7d2 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxParallelMessagingTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxParallelMessagingTests.cs @@ -54,7 +54,7 @@ public static void Add(int count) class SomeMessageConsumer1 : IConsumer { public ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -67,7 +67,7 @@ public ValueTask Consume( class SomeMessageConsumer2 : IConsumer { public ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -88,21 +88,15 @@ public Fixture() : base() .AddMetadata(SomeMessage1.PartName) .AddMetadata(SomeMessage2.PartName)) .WithDeliveries(builder => builder - .AddDeliveryScoped("test7_0", (_, settings) => + .AddDeliveryScoped("test7_0", (_, b) => { - settings.ScheduleSettings - .WithInterval(TimeSpan.FromMilliseconds(500)) - .WithInitialDelay(TimeSpan.Zero); - - settings.ConsumeSettings.WithMaxBatchSize(1024); + b.WithInterval(TimeSpan.FromMilliseconds(500)) + .WithMaxBatchSize(1024); }) - .AddDelivery("test7_1", (_, settings) => + .AddDelivery("test7_1", (_, b) => { - settings.ScheduleSettings - .WithInterval(TimeSpan.FromMilliseconds(500)) - .WithInitialDelay(TimeSpan.Zero); - - settings.ConsumeSettings.WithMaxBatchSize(1024); + b.WithInterval(TimeSpan.FromMilliseconds(500)) + .WithMaxBatchSize(1024); }) ) .WithPublishSettings((_, b) => b.WithMaxBatchSize(1024)) diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTenantParallelismTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTenantParallelismTests.cs index 166d06b9..e4a88588 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTenantParallelismTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTenantParallelismTests.cs @@ -27,7 +27,7 @@ class ParallelTestConsumer : IConsumer public static int TotalProcessed = 0; public async ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -105,17 +105,13 @@ public Fixture() : base() .WithDeliveries(deliveryBuilder => deliveryBuilder .AddDeliveryScoped( "parallel_test_group", - (_, settings) => + (_, b) => { - settings.ScheduleSettings - .WithInterval(TimeSpan.FromMilliseconds(500)) - .WithInitialDelay(TimeSpan.Zero); - - settings.ConsumeSettings - .WithNoBatchingWindow() - .WithTenantParallelProcessing(3) // 3 Parallel - .WithTenantTimeout(TimeSpan.FromSeconds(10)) - .WithMaxBatchSize(10); + b.WithInterval(TimeSpan.FromMilliseconds(500)) + .WithNoBatchingWindow() + .WithPerTenantMaxDegreeOfParallelism(3) + .WithPerTenantTimeout(TimeSpan.FromSeconds(10)) + .WithMaxBatchSize(10); }) ) ) diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTests.cs index b854e8d0..d812a336 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTests.cs @@ -21,7 +21,7 @@ class SomeMessageConsumer : IConsumer static int s_Counter = 0; public async ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -41,16 +41,11 @@ public Fixture() : base() .AddSaOutbox(builder => builder .WithTenants((_, s) => s.WithTenantIds(1)) .WithDeliveries(builder => builder - .AddDeliveryScoped("test6", (_, settings) => + .AddDeliveryScoped("test6", (_, b) => { - settings.ScheduleSettings - .WithInterval(TimeSpan.FromMilliseconds(100)) - .WithInitialDelay(TimeSpan.Zero) - ; - - settings.ConsumeSettings - .WithMaxBatchSize(1) - .WithNoBatchingWindow(); + b.WithInterval(TimeSpan.FromMilliseconds(100)) + .WithMaxBatchSize(1) + .WithNoBatchingWindow(); }) ) ) diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTwoGroupsTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTwoGroupsTests.cs index d3d2d8ea..6ccc57f2 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTwoGroupsTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTwoGroupsTests.cs @@ -21,7 +21,7 @@ class SomeMessageConsumerGr1 : IConsumer public static int Counter; public async ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -36,7 +36,7 @@ class SomeMessageConsumerGr2 : IConsumer public static int Counter; public async ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -55,23 +55,15 @@ public Fixture() : base() .WithTenants((_, s) => s.WithTenantIds(1, 2)) .WithDeliveries(deliveryBuilder => deliveryBuilder - .AddDeliveryScoped("test_gr1", (_, settings) => + .AddDeliveryScoped("test_gr1", (_, b) => { - settings.ScheduleSettings - .WithInterval(TimeSpan.FromMilliseconds(100)) - .WithInitialDelay(TimeSpan.Zero); - - settings.ConsumeSettings - .WithNoBatchingWindow(); + b.WithInterval(TimeSpan.FromMilliseconds(100)) + .WithNoBatchingWindow(); }) - .AddDelivery("test_gr2", (_, settings) => + .AddDelivery("test_gr2", (_, b) => { - settings.ScheduleSettings - .WithInterval(TimeSpan.FromMilliseconds(100)) - .WithInitialDelay(TimeSpan.Zero); - - settings.ConsumeSettings - .WithNoBatchingWindow(); + b.WithInterval(TimeSpan.FromMilliseconds(100)) + .WithNoBatchingWindow(); }) ) ) diff --git a/src/Tests/Sa.Outbox.Tests/ConsumeSettingsValidationTests.cs b/src/Tests/Sa.Outbox.Tests/ConsumeSettingsValidationTests.cs deleted file mode 100644 index e29838b7..00000000 --- a/src/Tests/Sa.Outbox.Tests/ConsumeSettingsValidationTests.cs +++ /dev/null @@ -1,172 +0,0 @@ -using Sa.Outbox.Delivery; -using Xunit; - -namespace Sa.Outbox.Tests; - -public class ConsumeSettingsValidationTests -{ - [Fact] - public void Default_Settings_Are_Valid() - { - var settings = new ConsumeSettings(); - var result = settings.Validate(); - - Assert.True(result.IsValid); - Assert.Empty(result.Errors); - } - - [Fact] - public void Zero_MaxBatchSize_Is_Invalid() - { - var settings = new ConsumeSettings { MaxBatchSize = 0 }; - var result = settings.Validate(); - - Assert.False(result.IsValid); - Assert.Single(result.Errors); - Assert.Contains("MaxBatchSize", result.Errors[0]); - } - - [Fact] - public void Negative_MaxBatchSize_Is_Invalid() - { - var settings = new ConsumeSettings { MaxBatchSize = -1 }; - var result = settings.Validate(); - - Assert.False(result.IsValid); - Assert.Contains("MaxBatchSize", result.Errors[0]); - } - - [Fact] - public void Invalid_MaxProcessingIterations_Is_Invalid() - { - var settings = new ConsumeSettings { MaxProcessingIterations = -5 }; - var result = settings.Validate(); - - Assert.False(result.IsValid); - Assert.Contains("MaxProcessingIterations", result.Errors[0]); - } - - [Fact] - public void Greedy_Mode_MaxProcessingIterations_MinusOne_Is_Valid() - { - var settings = new ConsumeSettings { MaxProcessingIterations = -1 }; - var result = settings.Validate(); - - Assert.True(result.IsValid); - } - - [Fact] - public void LockRenewal_Greater_Than_LockDuration_Is_Invalid() - { - var settings = new ConsumeSettings - { - LockDuration = TimeSpan.FromSeconds(5), - LockRenewal = TimeSpan.FromSeconds(10) - }; - var result = settings.Validate(); - - Assert.False(result.IsValid); - Assert.Contains("LockRenewal", result.Errors[0]); - } - - [Fact] - public void Equal_LockRenewal_And_LockDuration_Is_Invalid() - { - var settings = new ConsumeSettings - { - LockDuration = TimeSpan.FromSeconds(10), - LockRenewal = TimeSpan.FromSeconds(10) - }; - var result = settings.Validate(); - - Assert.False(result.IsValid); - Assert.Contains("LockRenewal", result.Errors[0]); - } - - [Fact] - public void Zero_PerTenantMaxDegreeOfParallelism_Is_Invalid() - { - var settings = new ConsumeSettings { PerTenantMaxDegreeOfParallelism = 0 }; - var result = settings.Validate(); - - Assert.False(result.IsValid); - Assert.Contains("PerTenantMaxDegreeOfParallelism", result.Errors[0]); - } - - [Fact] - public void Negative_MaxDeliveryAttempts_Is_Invalid() - { - var settings = new ConsumeSettings { MaxDeliveryAttempts = 0 }; - var result = settings.Validate(); - - Assert.False(result.IsValid); - Assert.Contains("MaxDeliveryAttempts", result.Errors[0]); - } - - [Fact] - public void Negative_ConsumeBatchSize_Is_Invalid() - { - var settings = new ConsumeSettings { ConsumeBatchSize = -1 }; - var result = settings.Validate(); - - Assert.False(result.IsValid); - Assert.Contains("ConsumeBatchSize", result.Errors[0]); - } - - [Fact] - public void ThrowIfInvalid_Does_Not_Throw_For_Valid_Settings() - { - var settings = new ConsumeSettings(); - var ex = Record.Exception(() => settings.ThrowIfInvalid()); - Assert.Null(ex); - } - - [Fact] - public void ThrowIfInvalid_Throws_For_Invalid_Settings() - { - var settings = new ConsumeSettings { MaxBatchSize = 0 }; - Assert.Throws(() => settings.ThrowIfInvalid()); - } - - [Fact] - public void Multiple_Violations_Return_All_Errors() - { - var settings = new ConsumeSettings - { - MaxBatchSize = 0, - MaxDeliveryAttempts = -1, - PerTenantMaxDegreeOfParallelism = 0 - }; - var result = settings.Validate(); - - Assert.False(result.IsValid); - Assert.True(result.Errors.Count >= 3); - } - - [Fact] - public void Zero_IterationDelay_Is_Valid() - { - var settings = new ConsumeSettings { IterationDelay = TimeSpan.Zero }; - var result = settings.Validate(); - - Assert.True(result.IsValid); - } - - [Fact] - public void Zero_BatchingWindow_Is_Valid() - { - var settings = new ConsumeSettings { BatchingWindow = TimeSpan.Zero }; - var result = settings.Validate(); - - Assert.True(result.IsValid); - } - - [Fact] - public void Zero_PerTenantTimeout_Is_Valid() - { - var settings = new ConsumeSettings { PerTenantTimeout = TimeSpan.Zero }; - var result = settings.Validate(); - - Assert.True(result.IsValid); - } -} diff --git a/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs b/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs index 759f22ae..db0e9944 100644 --- a/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs +++ b/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs @@ -6,11 +6,14 @@ public class DeliveryCourierTests { private sealed class TestMessage { } - private static ConsumerGroupSettings CreateSettings(int maxDeliveryAttempts = 3) - => new("test-group", isSingleton: false) - { - ConsumeSettings = { MaxDeliveryAttempts = maxDeliveryAttempts } - }; + private static OutboxConsumerSettings CreateSettings(int maxDeliveryAttempts = 3) + => new("test-group", AsSingleton: false, Interval: TimeSpan.FromMinutes(1), InitialDelay: TimeSpan.Zero, + ConcurrencyLimit: 1, MaxConcurrency: 1, RetryCountOnError: 0, + MaxBatchSize: 16, MaxProcessingIterations: -1, IterationDelay: TimeSpan.Zero, + LockDuration: TimeSpan.FromSeconds(10), LockRenewal: TimeSpan.FromSeconds(3), + LookbackInterval: TimeSpan.FromDays(7), MaxDeliveryAttempts: maxDeliveryAttempts, + BatchingWindow: TimeSpan.FromSeconds(3), PerTenantTimeout: TimeSpan.Zero, + PerTenantMaxDegreeOfParallelism: 1, Paused: false, Version: 0); private static OutboxMessageFilter CreateFilter() => new( diff --git a/src/Tests/Sa.Outbox.Tests/FakeOutboxContext.cs b/src/Tests/Sa.Outbox.Tests/FakeOutboxContext.cs index 1e12125a..45e6418c 100644 --- a/src/Tests/Sa.Outbox.Tests/FakeOutboxContext.cs +++ b/src/Tests/Sa.Outbox.Tests/FakeOutboxContext.cs @@ -128,7 +128,7 @@ public sealed class FakeDeliveryLifetimeInvoker(Func co public readonly List Invocations = []; public Task ConsumeInScope( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken cancellationToken) diff --git a/src/Tests/Sa.ScheduleTests/ScheduleSettingsTests.cs b/src/Tests/Sa.ScheduleTests/ScheduleSettingsTests.cs index 871a6770..f49b24fc 100644 --- a/src/Tests/Sa.ScheduleTests/ScheduleSettingsTests.cs +++ b/src/Tests/Sa.ScheduleTests/ScheduleSettingsTests.cs @@ -55,7 +55,7 @@ public void Merge_JobProperties_PrioritizesNonDefault() var job1 = JobSettings.Create(Guid.NewGuid()); job1.ErrorHandling.IfErrorRetry(5).ThenAbortJob(); - var job2 = JobSettings.Create(Guid.NewGuid()); + var _ = JobSettings.Create(Guid.NewGuid()); // job2 keeps defaults var merged = JobSettings.Create(job1); From 679924a0e123fbd0ddcb86df4762a14326716391 Mon Sep 17 00:00:00 2001 From: dundich Date: Mon, 29 Jun 2026 20:46:39 +0300 Subject: [PATCH 17/33] fix outbox --- .../Delivery/DeliveryLifetimeInvoker.cs | 13 +++-- src/Sa.Outbox/Delivery/DeliverySnapshot.cs | 57 ++++++++++++++----- src/Sa.Outbox/Delivery/IDeliverySnapshot.cs | 3 +- src/Sa.Outbox/Delivery/Job/Setup.cs | 16 +----- .../Delivery/OutboxConsumerSettings.cs | 6 +- .../Delivery/OutboxConsumerSettingsBuilder.cs | 36 ++---------- src/Sa.Outbox/Delivery/Setup.cs | 5 -- src/Sa.Schedule/Settings/ScheduleBuilder.cs | 1 + .../Delivery/DeliveryLongProcessorTests.cs | 6 +- .../Sa.Outbox.Tests/DeliveryCourierTests.cs | 2 +- 10 files changed, 67 insertions(+), 78 deletions(-) diff --git a/src/Sa.Outbox/Delivery/DeliveryLifetimeInvoker.cs b/src/Sa.Outbox/Delivery/DeliveryLifetimeInvoker.cs index 7f452024..bc8bfabb 100644 --- a/src/Sa.Outbox/Delivery/DeliveryLifetimeInvoker.cs +++ b/src/Sa.Outbox/Delivery/DeliveryLifetimeInvoker.cs @@ -9,7 +9,7 @@ namespace Sa.Outbox.Delivery; internal sealed class DeliveryLifetimeInvoker(IServiceProvider serviceProvider) : IDeliveryLifetimeInvoker { - private readonly ConcurrentDictionary _singletonConsumers = new(); + private readonly ConcurrentDictionary _singletonConsumers = new(); // Method to process messages using a consumer in scope public Task ConsumeInScope( @@ -39,11 +39,14 @@ private async Task ProcessInNewScope( ReadOnlyMemory> messages, CancellationToken cancellationToken) { - using AsyncServiceScope scope = serviceProvider.CreateAsyncScope(); - IConsumer consumer = scope.ServiceProvider.GetRequiredKeyedService>(settings); + await using AsyncServiceScope scope = serviceProvider.CreateAsyncScope(); + IConsumer consumer = GetConsumer(scope.ServiceProvider, settings.Id); await ProcessMessages(consumer, settings, filter, messages, cancellationToken); } + private static IConsumer GetConsumer(IServiceProvider sp, Guid id) + => sp.GetRequiredKeyedService>(id); + private static async Task ProcessMessages( IConsumer consumer, OutboxConsumerSettings settings, @@ -57,7 +60,7 @@ private static async Task ProcessMessages( private IConsumer GetOrCreateSingletonConsumer( OutboxConsumerSettings settings) { - return (IConsumer)_singletonConsumers.GetOrAdd(settings, key => - serviceProvider.GetRequiredKeyedService>(key)); + return (IConsumer)_singletonConsumers.GetOrAdd(settings.Id, key => + GetConsumer(serviceProvider, key)); } } diff --git a/src/Sa.Outbox/Delivery/DeliverySnapshot.cs b/src/Sa.Outbox/Delivery/DeliverySnapshot.cs index 3a061d07..c63afbe6 100644 --- a/src/Sa.Outbox/Delivery/DeliverySnapshot.cs +++ b/src/Sa.Outbox/Delivery/DeliverySnapshot.cs @@ -1,26 +1,55 @@ -namespace Sa.Outbox.Delivery; +using Sa.Outbox.Delivery.Job; +using Sa.Outbox.Metadata; +using Sa.Schedule; -internal sealed class DeliverySnapshot : IDeliverySnapshot +namespace Sa.Outbox.Delivery; + +internal sealed class DeliverySnapshot( + IScheduleSettings scheduleSettings, + IOutboxMessageMetadataProvider metadataProvider) : IDeliverySnapshot { + + private readonly Lazy _lazyJobs = new(() => [.. scheduleSettings.GetJobSettings()]); + + + private readonly Lazy _lazyParts = new(() => + { + Type baseType = typeof(DeliveryJob<>); + string[] parts = [.. scheduleSettings.GetJobSettings() + .Select(c => GetMessageTypeIfInheritsFromDeliveryJob(c.JobType, baseType)) + .Where(mt => mt != null) + .Cast() + .Select(mt => metadataProvider.GetMetadata(mt).PartName) + .Distinct()]; + + return parts; + }); + private readonly Lazy _lazyDeliveries = new(() => { - // Collect settings from the static registry populated by AddDeliveryJob - var registered = Setup.RegisteredSettings - .Where(s => s != null) - .DistinctBy(s => s.ConsumerGroupId) - .ToArray(); + OutboxConsumerSettings[] settings = [.. scheduleSettings.GetJobSettings() + .Select(c => c.Properties.GetConsumerGroupSettings()) + .Where(mt => mt != null) + .Cast()]; - return registered; + return settings; }); - public string[] Parts + + private static Type? GetMessageTypeIfInheritsFromDeliveryJob(Type jobType, Type baseType) { - get - { - var settings = _lazyDeliveries.Value; - return [.. settings.Select(s => s.ConsumerGroupId).Distinct()]; - } + if (!baseType.IsGenericTypeDefinition) return null; + + if (jobType.IsGenericType && jobType.GetGenericTypeDefinition() == baseType) + return jobType.GenericTypeArguments[0]; + + return jobType.BaseType != null + ? GetMessageTypeIfInheritsFromDeliveryJob(jobType.BaseType, baseType) + : null; } + + public string[] Parts => _lazyParts.Value; + public IJobSettings[] JobSettings => _lazyJobs.Value; public OutboxConsumerSettings[] ConsumerSettings => _lazyDeliveries.Value; } diff --git a/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs b/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs index 4531aaea..2cb28205 100644 --- a/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs +++ b/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs @@ -2,9 +2,8 @@ public interface IDeliverySnapshot { - OutboxConsumerSettings[] ConsumerSettings { get; } string[] Parts { get; } - + OutboxConsumerSettings[] ConsumerSettings { get; } IEnumerable GetConsumeGroupIds() => ConsumerSettings.Select(c => c.ConsumerGroupId).Distinct(); } diff --git a/src/Sa.Outbox/Delivery/Job/Setup.cs b/src/Sa.Outbox/Delivery/Job/Setup.cs index bcd79c6c..81e22c83 100644 --- a/src/Sa.Outbox/Delivery/Job/Setup.cs +++ b/src/Sa.Outbox/Delivery/Job/Setup.cs @@ -1,6 +1,5 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -using Sa.Outbox.Delivery.Job; using Sa.Schedule; using System.Diagnostics.CodeAnalysis; @@ -8,12 +7,6 @@ namespace Sa.Outbox.Delivery.Job; internal static class Setup { - /// - /// Queue of settings registered via AddDeliveryJob, consumed by DeliverySnapshot. - /// Thread-safe for bootstrap phase only. - /// - internal static readonly Queue RegisteredSettings = new(); - public static IServiceCollection AddDeliveryJob< [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( this IServiceCollection services, @@ -53,16 +46,13 @@ public static IServiceCollection AddDeliveryJob< var settings = builder.Build(); - // Register in the static registry for DeliverySnapshot - RegisteredSettings.Enqueue(settings); - if (isSingleton) { - services.AddKeyedSingleton, TConsumer>(settings); + services.AddKeyedSingleton, TConsumer>(settings.Id); } else { - services.AddKeyedScoped, TConsumer>(settings); + services.AddKeyedScoped, TConsumer>(settings.Id); } services.AddSaSchedule(builder => @@ -83,7 +73,7 @@ public static IServiceCollection AddDeliveryJob< .ThenCloseApplication()) ; - }, jobId ?? Guid.Empty); + }, jobId); builder.AddInterceptor(); diff --git a/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs b/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs index 8be8f4b8..f8e67512 100644 --- a/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs +++ b/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs @@ -6,6 +6,9 @@ /// Create new instances via or the with expression. /// public sealed record OutboxConsumerSettings( + + Guid Id, + /// /// Unique identifier for the consumer group. Groups settings for a single logical consumer. /// @@ -108,8 +111,7 @@ public sealed record OutboxConsumerSettings( /// Settings version. Incremented on every change for change detection. /// Used for optimistic locking and notifying subscribers. /// - int Version - ) + int Version) { /// /// Validates all settings and returns a list of error messages. diff --git a/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs b/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs index a7a7f255..f95ffbc6 100644 --- a/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs +++ b/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs @@ -6,6 +6,8 @@ /// public sealed class OutboxConsumerSettingsBuilder { + private readonly Guid _id = Guid.NewGuid(); + private string? _consumerGroupId; private bool? _asSingleton; private TimeSpan? _interval; @@ -34,6 +36,7 @@ public sealed class OutboxConsumerSettingsBuilder public OutboxConsumerSettings Build() { return new OutboxConsumerSettings( + _id, _consumerGroupId ?? throw new InvalidOperationException("ConsumerGroupId is required."), _asSingleton ?? false, _interval ?? TimeSpan.FromMinutes(1), @@ -55,37 +58,6 @@ public OutboxConsumerSettings Build() 0); } - // ── Runtime: partial copy from existing settings ────────── - - /// - /// Creates a copy of with only the builder-configured overrides applied. - /// Unspecified fields inherit from . Version increments by 1. - /// - public OutboxConsumerSettings BuildCopy(OutboxConsumerSettings original) - { - return original is null - ? throw new ArgumentNullException(nameof(original)) - : new OutboxConsumerSettings( - _consumerGroupId ?? original.ConsumerGroupId, - _asSingleton ?? original.AsSingleton, - _interval ?? original.Interval, - _initialDelay ?? original.InitialDelay, - _concurrencyLimit ?? original.ConcurrencyLimit, - _maxConcurrency ?? original.MaxConcurrency, - _retryCountOnError ?? original.RetryCountOnError, - _maxBatchSize ?? original.MaxBatchSize, - _maxProcessingIterations ?? original.MaxProcessingIterations, - _iterationDelay ?? original.IterationDelay, - _lockDuration ?? original.LockDuration, - _lockRenewal ?? original.LockRenewal, - _lookbackInterval ?? original.LookbackInterval, - _maxDeliveryAttempts ?? original.MaxDeliveryAttempts, - _batchingWindow ?? original.BatchingWindow, - _perTenantTimeout ?? original.PerTenantTimeout, - _perTenantMaxDegreeOfParallelism ?? original.PerTenantMaxDegreeOfParallelism, - _paused ?? original.Paused, - original.Version + 1); - } // ── Fluent setters ──────────────────────────────────────── @@ -223,7 +195,7 @@ public OutboxConsumerSettingsBuilder WithIterationDelay(TimeSpan iterationDelay) /// public OutboxConsumerSettingsBuilder WithLockDuration(TimeSpan lockDuration) { - if (lockDuration <= TimeSpan.Zero) throw new ArgumentException("LockDuration must be > TimeSpan.Zero.", nameof(lockDuration)); + if (lockDuration < TimeSpan.Zero) throw new ArgumentException("LockDuration must be >= TimeSpan.Zero.", nameof(lockDuration)); _lockDuration = lockDuration; return this; } diff --git a/src/Sa.Outbox/Delivery/Setup.cs b/src/Sa.Outbox/Delivery/Setup.cs index 52b536ff..4a08d218 100644 --- a/src/Sa.Outbox/Delivery/Setup.cs +++ b/src/Sa.Outbox/Delivery/Setup.cs @@ -3,16 +3,11 @@ using Sa.Outbox.Delivery.Job; using Sa.Outbox.Metadata; using Sa.Outbox.Partitional; -using Sa.Schedule; -using System.Collections.Concurrent; namespace Sa.Outbox.Delivery; internal static class Setup { - // Thread-safe registry of consumer group settings discovered via AddDeliveryJob - internal static readonly ConcurrentQueue RegisteredSettings = new(); - public static IServiceCollection AddOutboxDelivery( this IServiceCollection services, Action? configure = null) { diff --git a/src/Sa.Schedule/Settings/ScheduleBuilder.cs b/src/Sa.Schedule/Settings/ScheduleBuilder.cs index 3954aeaa..9863ad6c 100644 --- a/src/Sa.Schedule/Settings/ScheduleBuilder.cs +++ b/src/Sa.Schedule/Settings/ScheduleBuilder.cs @@ -71,6 +71,7 @@ public ScheduleBuilder(IServiceCollection services) public IJobBuilder AddJob(Func action, Guid? jobId = null) { Guid id = GetId(jobId); + _services .RemoveAllKeyed(jobId) .AddKeyedScoped(id, (_, __) => new FuncJob(action)); diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryLongProcessorTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryLongProcessorTests.cs index bef787bc..e7c47a61 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryLongProcessorTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryLongProcessorTests.cs @@ -4,7 +4,7 @@ namespace Sa.Outbox.PostgreSqlTests.Delivery; -public class DeliveryLongProcessorTests(DeliveryLongProcessorTests.Fixture fixture) +public sealed class DeliveryLongProcessorTests(DeliveryLongProcessorTests.Fixture fixture) : IClassFixture { class TestMessageConsumer : IConsumer @@ -15,7 +15,6 @@ public async ValueTask Consume( ReadOnlyMemory> messages, CancellationToken cancellationToken) { - Console.WriteLine(messages.Length); await Task.Delay(1000, cancellationToken); } } @@ -34,7 +33,7 @@ public Fixture() : base() .WithDeliveries(b => b .AddDeliveryScoped("test1", (_, b) => { - OutboxSettings = new OutboxConsumerSettingsBuilder() + OutboxSettings = b .WithConsumerGroupId("test1") .WithLockDuration(TimeSpan.FromMilliseconds(300)) .WithLockRenewal(TimeSpan.FromMilliseconds(100)) @@ -67,7 +66,6 @@ public async Task Deliver_LongProcess_MustBe_Work() var cnt = await fixture.Publisher.Publish(messages, m => m.TenantId, TestContext.Current.CancellationToken); Assert.True(cnt > 0); - var result = await Sub.ProcessMessages(fixture.OutboxSettings, CancellationToken.None); Assert.True(result > 0); } diff --git a/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs b/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs index db0e9944..1c9ccc22 100644 --- a/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs +++ b/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs @@ -7,7 +7,7 @@ public class DeliveryCourierTests private sealed class TestMessage { } private static OutboxConsumerSettings CreateSettings(int maxDeliveryAttempts = 3) - => new("test-group", AsSingleton: false, Interval: TimeSpan.FromMinutes(1), InitialDelay: TimeSpan.Zero, + => new(Id: Guid.NewGuid(), "test-group", AsSingleton: false, Interval: TimeSpan.FromMinutes(1), InitialDelay: TimeSpan.Zero, ConcurrencyLimit: 1, MaxConcurrency: 1, RetryCountOnError: 0, MaxBatchSize: 16, MaxProcessingIterations: -1, IterationDelay: TimeSpan.Zero, LockDuration: TimeSpan.FromSeconds(10), LockRenewal: TimeSpan.FromSeconds(3), From 6e6fbda31eea95cc2324c3963b6a3fe99b2ab832 Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 30 Jun 2026 14:06:10 +0300 Subject: [PATCH 18/33] refactor delivery builder + xml summary --- .../Configuration/PgOutboxConsumeSettings.cs | 8 --- src/Sa.Outbox/Delivery/DeliveryBuilder.cs | 33 +++++++++++ src/Sa.Outbox/Delivery/DeliveryStatusCode.cs | 24 ++++++++ .../Delivery/IConsumerGroupNamingStrategy.cs | 9 +++ src/Sa.Outbox/Delivery/IDeliveryBuilder.cs | 49 +++++++++++++--- .../Delivery/IDeliveryBuilder.partial.cs | 34 ----------- src/Sa.Outbox/Delivery/IDeliveryCourier.cs | 12 +++- .../Delivery/IDeliveryLifetimeInvoker.cs | 9 +++ src/Sa.Outbox/Delivery/IDeliverySnapshot.cs | 15 +++++ src/Sa.Outbox/Delivery/IDeliveryTenant.cs | 8 +++ .../Delivery/IOutboxContextFactory.cs | 10 ++++ .../Delivery/Job/IOutboxJobInterceptor.cs | 4 ++ src/Sa.Outbox/Delivery/OutboxDefaults.cs | 56 +++++++++++++++++++ src/Sa.Outbox/IOutboxBuilder.cs | 29 ++++++++-- src/Sa.Outbox/IOutboxPublishable.cs | 11 ++++ .../Metadata/IOutboxMessageMetadataBuilder.cs | 17 ++++++ .../Partitional/IOutboxPartitionalSupport.cs | 6 ++ .../PlugServices/IOutboxDeliveryManager.cs | 3 +- .../PlugServices/IOutboxTenantDetector.cs | 1 - .../Publication/IOutboxMessagePublisher.cs | 35 +++++++++++- src/Sa.Outbox/Setup.cs | 6 ++ src/Samples/PgOutbox.ConsoleApp/Program.cs | 1 - .../Sa.Outbox.Tests/DeliveryCourierTests.cs | 26 +-------- 23 files changed, 320 insertions(+), 86 deletions(-) delete mode 100644 src/Sa.Outbox/Delivery/IDeliveryBuilder.partial.cs create mode 100644 src/Sa.Outbox/Delivery/OutboxDefaults.cs diff --git a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConsumeSettings.cs b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConsumeSettings.cs index 3151a163..433b92c9 100644 --- a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConsumeSettings.cs +++ b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConsumeSettings.cs @@ -1,6 +1,4 @@ using Sa.Extensions; -using Sa.Outbox.Delivery; -using System.Diagnostics.CodeAnalysis; namespace Sa.Outbox.PostgreSql.Configuration; @@ -20,12 +18,6 @@ public PgOutboxConsumeSettings WithMinOffset(string consumerGroupId, DateTimeOff return this; } - public PgOutboxConsumeSettings WithMinOffset<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer>(Guid offset) - => WithMinOffset(IDeliveryBuilder.GetConsumerGroupName(), offset); - - public PgOutboxConsumeSettings WithMinOffset<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer>(DateTimeOffset offset) - => WithMinOffset(IDeliveryBuilder.GetConsumerGroupName(), offset); - public Guid GetMinOffset(string consumerGroupId) { return _offsets.TryGetValue(consumerGroupId, out var result) diff --git a/src/Sa.Outbox/Delivery/DeliveryBuilder.cs b/src/Sa.Outbox/Delivery/DeliveryBuilder.cs index e1a5bfbe..cdf35ed2 100644 --- a/src/Sa.Outbox/Delivery/DeliveryBuilder.cs +++ b/src/Sa.Outbox/Delivery/DeliveryBuilder.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Sa.Extensions; using Sa.Outbox.Delivery.Job; using System.Diagnostics.CodeAnalysis; using System.Text.RegularExpressions; @@ -8,6 +9,9 @@ namespace Sa.Outbox.Delivery; internal sealed partial class DeliveryBuilder(IServiceCollection services) : IDeliveryBuilder { + + private IConsumerGroupNamingStrategy _defaultNamingStrategy = new DefaultConsumerGroupNamingStrategy(); + public IDeliveryBuilder AddDeliveryScoped< [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( string consumerGroupId, @@ -21,6 +25,13 @@ public IDeliveryBuilder AddDeliveryScoped< return this; } + public IDeliveryBuilder AddDeliveryScoped<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage> + (Action? configure = null, Guid? jobId = null) + where TConsumer : class, IConsumer + { + return AddDeliveryScoped(GetConsumerGroupName(), configure, jobId); + } + public IDeliveryBuilder AddDelivery< [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( string consumerGroupId, @@ -34,6 +45,14 @@ public IDeliveryBuilder AddDelivery< return this; } + public IDeliveryBuilder AddDelivery< + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( + Action? configure = null, Guid? jobId = null) + where TConsumer : class, IConsumer + { + return AddDelivery(GetConsumerGroupName(), configure, jobId); + } + public IDeliveryBuilder AddDeliveryBatching< [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TImplementation>() where TImplementation : class, IDeliveryBatcher @@ -44,6 +63,20 @@ public IDeliveryBuilder AddDeliveryBatching< return this; } + public string GetConsumerGroupName() => _defaultNamingStrategy.GetConsumerGroupName(); + + public IDeliveryBuilder ConfigureDefaultNamingStrategy(IConsumerGroupNamingStrategy strategy) + { + _defaultNamingStrategy = strategy ?? throw new ArgumentNullException(nameof(strategy)); + return this; + } + + sealed class DefaultConsumerGroupNamingStrategy : IConsumerGroupNamingStrategy + { + string IConsumerGroupNamingStrategy.GetConsumerGroupName() + => $"cg_{(typeof(TConsumer).FullName ?? typeof(TConsumer).Name).GetMurmurHash3()}"; + } + static string SanitizeString(string input) { ArgumentNullException.ThrowIfNullOrWhiteSpace(input); diff --git a/src/Sa.Outbox/Delivery/DeliveryStatusCode.cs b/src/Sa.Outbox/Delivery/DeliveryStatusCode.cs index badcbea2..9f54f4af 100644 --- a/src/Sa.Outbox/Delivery/DeliveryStatusCode.cs +++ b/src/Sa.Outbox/Delivery/DeliveryStatusCode.cs @@ -126,27 +126,51 @@ public enum DeliveryStatusCode public static class DeliveryStatusCodeExtensions { + /// + /// Indicates whether the status code represents a pending message that has not yet been processed. + /// public static bool IsPending(this DeliveryStatusCode statusCode) => statusCode == DeliveryStatusCode.Pending; + /// + /// Indicates whether the status code represents a message currently being processed. + /// public static bool IsProcessing(this DeliveryStatusCode statusCode) => statusCode == DeliveryStatusCode.Processing; + /// + /// Indicates whether the status code represents a postponed message (will be retried later). + /// public static bool IsPostponed(this DeliveryStatusCode statusCode) => statusCode == DeliveryStatusCode.Postpone; + /// + /// Indicates whether the status code represents a retryable error (attempt counter will be incremented). + /// public static bool IsRetry(this DeliveryStatusCode statusCode) => statusCode == DeliveryStatusCode.Retry; + /// + /// Indicates whether the status code represents a successful outcome (HTTP 2xx – 3xx range, excluding redirects). + /// public static bool IsSuccess(this DeliveryStatusCode statusCode) => statusCode >= DeliveryStatusCode.Ok && statusCode <= DeliveryStatusCode.Aborted; + /// + /// Indicates whether the status code represents an aborted processing attempt (user or system intervention). + /// public static bool IsAborted(this DeliveryStatusCode statusCode) => statusCode == DeliveryStatusCode.Aborted; + /// + /// Indicates whether the status code represents a warning-level issue (recoverable, processing continues). + /// public static bool IsWarning(this DeliveryStatusCode statusCode) => statusCode >= DeliveryStatusCode.Warn && statusCode < DeliveryStatusCode.Error; + /// + /// Indicates whether the status code represents an error-level issue (permanent failure or max retries exceeded). + /// public static bool IsError(this DeliveryStatusCode statusCode) => statusCode >= DeliveryStatusCode.Error; } diff --git a/src/Sa.Outbox/Delivery/IConsumerGroupNamingStrategy.cs b/src/Sa.Outbox/Delivery/IConsumerGroupNamingStrategy.cs index e1c11795..d1f7f5a9 100644 --- a/src/Sa.Outbox/Delivery/IConsumerGroupNamingStrategy.cs +++ b/src/Sa.Outbox/Delivery/IConsumerGroupNamingStrategy.cs @@ -1,7 +1,16 @@ namespace Sa.Outbox.Delivery; +/// +/// Determines the consumer group name used by Kafka-style or PostgreSQL-based consumer groups +/// for a given consumer type. Allows custom naming conventions beyond the default type-based naming. +/// public interface IConsumerGroupNamingStrategy { + /// + /// Returns the consumer group name for the specified consumer type. + /// + /// The consumer type to generate a group name for. + /// A human-readable consumer group identifier. string GetConsumerGroupName(); } diff --git a/src/Sa.Outbox/Delivery/IDeliveryBuilder.cs b/src/Sa.Outbox/Delivery/IDeliveryBuilder.cs index 309f9e28..e46e5021 100644 --- a/src/Sa.Outbox/Delivery/IDeliveryBuilder.cs +++ b/src/Sa.Outbox/Delivery/IDeliveryBuilder.cs @@ -20,24 +20,55 @@ public partial interface IDeliveryBuilder string consumerGroupId, Action? configure = null, Guid? jobId = null - ) - where TConsumer : class, IConsumer; + ) where TConsumer : class, IConsumer; /// - /// Adds singleton delivery for the specified consumer and message type. + /// Adds scoped delivery for the specified consumer and message type without an explicit consumer group ID. + /// A default group name is derived from using the configured naming strategy. /// + /// The type of consumer. + /// The type of message. + /// An optional action to configure the delivery settings via builder. + /// An optional job identifier to bind this delivery to a specific scheduled job. + /// The delivery builder instance. + IDeliveryBuilder AddDeliveryScoped<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( + Action? configure = null, + Guid? jobId = null + ) where TConsumer : class, IConsumer; + + /// + /// Adds singleton delivery for the specified consumer and message type with an explicit consumer group ID. + /// + /// The type of consumer. + /// The type of message. + /// Group identity for consuming. + /// An optional action to configure the delivery settings via builder. + /// An optional job identifier to bind this delivery to a specific scheduled job. + /// The delivery builder instance. IDeliveryBuilder AddDelivery<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( string consumerGroupId, Action? configure = null, Guid? jobId = null - ) - where TConsumer : class, IConsumer; - + ) where TConsumer : class, IConsumer; /// - /// Added provider functionality to dynamically calculate batch sizes for delivery + /// Adds singleton delivery for the specified consumer and message type without an explicit consumer group ID. + /// A default group name is derived from using the configured naming strategy. /// - IDeliveryBuilder AddDeliveryBatching<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TImplementation>() - where TImplementation : class, IDeliveryBatcher; + /// The type of consumer. + /// The type of message. + /// An optional action to configure the delivery settings via builder. + /// An optional job identifier to bind this delivery to a specific scheduled job. + /// The delivery builder instance. + IDeliveryBuilder AddDelivery<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( + Action? configure = null, + Guid? jobId = null + ) where TConsumer : class, IConsumer; + /// + /// Sets the default naming strategy used to derive consumer group names when no explicit is provided. + /// + /// The naming strategy implementation to use. + /// The delivery builder instance. + IDeliveryBuilder ConfigureDefaultNamingStrategy(IConsumerGroupNamingStrategy strategy); } diff --git a/src/Sa.Outbox/Delivery/IDeliveryBuilder.partial.cs b/src/Sa.Outbox/Delivery/IDeliveryBuilder.partial.cs deleted file mode 100644 index 4b70f42c..00000000 --- a/src/Sa.Outbox/Delivery/IDeliveryBuilder.partial.cs +++ /dev/null @@ -1,34 +0,0 @@ -using Sa.Extensions; -using System.Diagnostics.CodeAnalysis; - -namespace Sa.Outbox.Delivery; - -public partial interface IDeliveryBuilder -{ - private static IConsumerGroupNamingStrategy _defaultNamingStrategy = new DefaultConsumerGroupNamingStrategy(); - - public IDeliveryBuilder AddDeliveryScoped<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( - Action? configure = null - ) - where TConsumer : class, IConsumer - => AddDeliveryScoped(GetConsumerGroupName(), configure); - - public IDeliveryBuilder AddDelivery<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( - Action? configure = null - ) - where TConsumer : class, IConsumer - => AddDelivery(GetConsumerGroupName(), configure); - - - public static string GetConsumerGroupName() => _defaultNamingStrategy.GetConsumerGroupName(); - - - public static void ConfigureDefaultNamingStrategy(IConsumerGroupNamingStrategy strategy) - => _defaultNamingStrategy = strategy ?? throw new ArgumentNullException(nameof(strategy)); - - class DefaultConsumerGroupNamingStrategy : IConsumerGroupNamingStrategy - { - string IConsumerGroupNamingStrategy.GetConsumerGroupName() - => $"cg_{(typeof(TConsumer).FullName ?? typeof(TConsumer).Name).GetMurmurHash3()}"; - } -} diff --git a/src/Sa.Outbox/Delivery/IDeliveryCourier.cs b/src/Sa.Outbox/Delivery/IDeliveryCourier.cs index 30e83692..dee55bb0 100644 --- a/src/Sa.Outbox/Delivery/IDeliveryCourier.cs +++ b/src/Sa.Outbox/Delivery/IDeliveryCourier.cs @@ -1,10 +1,20 @@ namespace Sa.Outbox.Delivery; /// -/// Delivers a batch of messages with error handling and retry mechanisms +/// Delivers a batch of messages with error handling and retry mechanisms. /// internal interface IDeliveryCourier { + /// + /// Delivers the given messages to their respective consumers, invoking the consumer for each message + /// and applying the configured retry and backoff strategies on failure. + /// + /// The type of the messages being delivered. + /// Runtime delivery settings controlling concurrency, batching, retries, and locking. + /// Criteria used to select which messages are eligible for delivery. + /// Read-only memory containing context operations for each message. + /// A token to monitor for cancellation requests. + /// The number of messages successfully delivered. ValueTask Deliver( OutboxConsumerSettings settings, OutboxMessageFilter filter, diff --git a/src/Sa.Outbox/Delivery/IDeliveryLifetimeInvoker.cs b/src/Sa.Outbox/Delivery/IDeliveryLifetimeInvoker.cs index c6f98893..43f6ff6a 100644 --- a/src/Sa.Outbox/Delivery/IDeliveryLifetimeInvoker.cs +++ b/src/Sa.Outbox/Delivery/IDeliveryLifetimeInvoker.cs @@ -5,6 +5,15 @@ /// internal interface IDeliveryLifetimeInvoker { + /// + /// Resolves a consumer from DI within an activation scope and invokes it for the given batch of messages. + /// Ensures scoped services (e.g., DbContext) are correctly disposed after processing. + /// + /// The type of the messages being consumed. + /// Runtime delivery settings controlling consumption behavior. + /// Criteria used to select which messages are eligible for consumption. + /// Read-only memory containing context operations for each message. + /// A token to monitor for cancellation requests. Task ConsumeInScope( OutboxConsumerSettings settings, OutboxMessageFilter filter, diff --git a/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs b/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs index 2cb28205..10f0babb 100644 --- a/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs +++ b/src/Sa.Outbox/Delivery/IDeliverySnapshot.cs @@ -1,9 +1,24 @@ namespace Sa.Outbox.Delivery; +/// +/// Provides a read-only view of currently registered delivery parts and their associated consumer settings. +/// Useful for diagnostics, health checks, and runtime introspection of active outbox consumers. +/// public interface IDeliverySnapshot { + /// + /// Gets the distinct part names currently served by this snapshot. + /// string[] Parts { get; } + + /// + /// Gets the consumer settings arrays corresponding to each part in . + /// OutboxConsumerSettings[] ConsumerSettings { get; } + + /// + /// Returns the distinct consumer group IDs registered in this snapshot. + /// IEnumerable GetConsumeGroupIds() => ConsumerSettings.Select(c => c.ConsumerGroupId).Distinct(); } diff --git a/src/Sa.Outbox/Delivery/IDeliveryTenant.cs b/src/Sa.Outbox/Delivery/IDeliveryTenant.cs index 735de090..3de322ac 100644 --- a/src/Sa.Outbox/Delivery/IDeliveryTenant.cs +++ b/src/Sa.Outbox/Delivery/IDeliveryTenant.cs @@ -5,6 +5,14 @@ /// internal interface IDeliveryTenant { + /// + /// Acquires a tenant-level lock, retrieves pending outbox messages, and delivers them to the appropriate consumer. + /// + /// The type of the messages being delivered. + /// The identifier of the tenant whose messages to process. + /// Runtime delivery settings for this tenant's processing scope. + /// A token to monitor for cancellation requests. + /// The number of messages successfully processed. Task ProcessInTenant( int tenantId, OutboxConsumerSettings settings, diff --git a/src/Sa.Outbox/Delivery/IOutboxContextFactory.cs b/src/Sa.Outbox/Delivery/IOutboxContextFactory.cs index 84dd2b7f..5e49b225 100644 --- a/src/Sa.Outbox/Delivery/IOutboxContextFactory.cs +++ b/src/Sa.Outbox/Delivery/IOutboxContextFactory.cs @@ -1,6 +1,16 @@ namespace Sa.Outbox.Delivery; +/// +/// Creates instances for delivering outbox messages. +/// Operations returned by this factory are used by consumers to acknowledge, error, or postpone individual messages. +/// public interface IOutboxContextFactory { + /// + /// Creates a new set of context operations scoped to the given delivery message. + /// + /// The type of the message being delivered. + /// The delivery message containing outbox and message metadata. + /// A new instance for acknowledging or rejecting messages. IOutboxContextOperations Create(OutboxDeliveryMessage deliveryMessage); } diff --git a/src/Sa.Outbox/Delivery/Job/IOutboxJobInterceptor.cs b/src/Sa.Outbox/Delivery/Job/IOutboxJobInterceptor.cs index aecd5e8d..874c2016 100644 --- a/src/Sa.Outbox/Delivery/Job/IOutboxJobInterceptor.cs +++ b/src/Sa.Outbox/Delivery/Job/IOutboxJobInterceptor.cs @@ -2,6 +2,10 @@ namespace Sa.Outbox.Delivery.Job; +/// +/// Marker interface that extends to enable outbox-specific interception hooks +/// around delivery job lifecycle events (before start, after completion, on failure). +/// public interface IOutboxJobInterceptor : IJobInterceptor { } diff --git a/src/Sa.Outbox/Delivery/OutboxDefaults.cs b/src/Sa.Outbox/Delivery/OutboxDefaults.cs new file mode 100644 index 00000000..3f8884a3 --- /dev/null +++ b/src/Sa.Outbox/Delivery/OutboxDefaults.cs @@ -0,0 +1,56 @@ +namespace Sa.Outbox.Delivery; + +/// +/// Immutable default values used by and . +/// Change here to affect all consumers globally. +/// +public static class OutboxDefaults +{ + /// Default interval between job executions (1 minute). + public static TimeSpan Interval => TimeSpan.FromMinutes(1); + + /// Default initial delay before the first execution (10 seconds). + public static TimeSpan InitialDelay => TimeSpan.FromSeconds(10); + + /// Default concurrency limit (1). + public static int ConcurrencyLimit => 1; + + /// Default maximum concurrency (48). + public static int MaxConcurrency => 48; + + /// Default retry count on error — no retries. + public static int RetryCountOnError => 0; + + /// Default maximum batch size for database polling (16). + public static int MaxBatchSize => 16; + + /// Default maximum processing iterations (10). + public static int MaxProcessingIterations => 10; + + /// Default iteration delay (zero — greedy mode). + public static TimeSpan IterationDelay => TimeSpan.Zero; + + /// Default message lock duration (10 seconds). + public static TimeSpan LockDuration => TimeSpan.FromSeconds(10); + + /// Default lock renewal time (3 seconds). + public static TimeSpan LockRenewal => TimeSpan.FromSeconds(3); + + /// Default lookback interval for selecting messages (7 days). + public static TimeSpan LookbackInterval => TimeSpan.FromDays(7); + + /// Default maximum delivery attempts (3). + public static int MaxDeliveryAttempts => 3; + + /// Default batching window for accumulating messages (3 seconds). + public static TimeSpan BatchingWindow => TimeSpan.FromSeconds(3); + + /// Default per-tenant processing timeout (zero — no timeout). + public static TimeSpan PerTenantTimeout => TimeSpan.Zero; + + /// Default max degree of tenant parallelism (sequential = 1). + public static int PerTenantMaxDegreeOfParallelism => 1; + + /// Default paused state (false — running). + public static bool Paused => false; +} diff --git a/src/Sa.Outbox/IOutboxBuilder.cs b/src/Sa.Outbox/IOutboxBuilder.cs index 7d1ed070..1cd2f426 100644 --- a/src/Sa.Outbox/IOutboxBuilder.cs +++ b/src/Sa.Outbox/IOutboxBuilder.cs @@ -9,41 +9,60 @@ namespace Sa.Outbox; public interface IOutboxBuilder { /// - /// Configure publish settings for the outbox. + /// Configures publish settings for the outbox. /// + /// An action to configure within the service provider scope. + /// The same instance for chaining. IOutboxBuilder WithPublishSettings(Action configure); /// /// Configures the delivery settings for the outbox. /// /// An action to configure the delivery settings. - /// The current instance of the IOutboxSettingsBuilder. + /// The same instance for chaining. IOutboxBuilder WithDeliveries(Action build); /// /// Enables partitioning support for the outbox. /// /// An action to configure the partitioning settings. - /// The current instance of the IOutboxSettingsBuilder. + /// The same instance for chaining. IOutboxBuilder WithTenants(Action configure); /// - /// Registers a custom implementation of IDeliveryBatcher to control how messages are batched for delivery. + /// Registers a custom implementation of to control how messages are batched for delivery. /// + /// The custom batcher implementation to register. + /// The same instance for chaining. IOutboxBuilder WithDeliveryBatcher<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TImplementation>() where TImplementation : class, IDeliveryBatcher; + /// + /// Configures message metadata registrations (part names and payload ID resolvers) for outbox message types. + /// + /// An action to configure metadata through . + /// The same instance for chaining. IOutboxBuilder WithMetadata(Action configure); /// - /// shortcut + /// Registers metadata for a message type using an explicit part name and optional payload ID resolver. /// + /// The message type to register metadata for. + /// The logical partition name associated with the message type (e.g., "orders"). + /// An optional delegate to extract the unique payload identifier from a message. Defaults to calling GetPayloadId() on . + /// The same instance for chaining. IOutboxBuilder AddMetadata( string partName, Func? getPayloadId = null) where TMessage : class => WithMetadata((_, m) => m.AddMetadata(partName, getPayloadId)); + /// + /// Registers metadata for a message type that implements , + /// automatically deriving the part name and payload ID resolver from the type itself. + /// + /// A message type implementing . + /// The same instance for chaining. IOutboxBuilder AddMetadata() where TMessage : class, IOutboxPublishable => WithMetadata((_, m) => m.AddMetadata()); } diff --git a/src/Sa.Outbox/IOutboxPublishable.cs b/src/Sa.Outbox/IOutboxPublishable.cs index 8d4d5c7c..b4b22550 100644 --- a/src/Sa.Outbox/IOutboxPublishable.cs +++ b/src/Sa.Outbox/IOutboxPublishable.cs @@ -1,9 +1,20 @@ namespace Sa.Outbox; +/// +/// Marks a message type as publishable through the Outbox pattern. +/// Implementors supply a unique payload identifier, the owning tenant ID, and the logical partition name used for routing. +/// public interface IOutboxPublishable { + /// + /// Gets or produces a unique identifier for the message payload. + /// Used by the Outbox infrastructure to track delivery attempts and prevent duplicates. + /// string GetPayloadId(); + /// + /// Gets the tenant identifier that owns this message type. + /// int GetTenantId(); /// /// Gets the logical identifier of the partition associated with this type. diff --git a/src/Sa.Outbox/Metadata/IOutboxMessageMetadataBuilder.cs b/src/Sa.Outbox/Metadata/IOutboxMessageMetadataBuilder.cs index d47a751d..361c78e3 100644 --- a/src/Sa.Outbox/Metadata/IOutboxMessageMetadataBuilder.cs +++ b/src/Sa.Outbox/Metadata/IOutboxMessageMetadataBuilder.cs @@ -1,11 +1,28 @@ namespace Sa.Outbox.Metadata; +/// +/// Builds metadata registrations for outbox message types, including part names and payload ID resolvers. +/// Metadata is used to route messages to the correct partitioned table and extract unique identifiers. +/// public interface IOutboxMessageMetadataBuilder { + /// + /// Registers metadata for a message type using an explicit part name and optional payload ID resolver. + /// + /// The message type to register metadata for. + /// The logical partition name associated with the message type (e.g., "orders"). + /// An optional delegate to extract the unique payload identifier from a message. + /// The same instance for chaining. IOutboxMessageMetadataBuilder AddMetadata( string partName, Func? getPayloadId = null) where TMessage : class; + /// + /// Registers metadata for a message type that implements , + /// automatically deriving the part name and payload ID resolver from the type itself. + /// + /// A message type implementing . + /// The same instance for chaining. IOutboxMessageMetadataBuilder AddMetadata() where TMessage : class, IOutboxPublishable { return AddMetadata(TMessage.PartName, m => m.GetPayloadId()); diff --git a/src/Sa.Outbox/Partitional/IOutboxPartitionalSupport.cs b/src/Sa.Outbox/Partitional/IOutboxPartitionalSupport.cs index d8b65057..a3fd7b21 100644 --- a/src/Sa.Outbox/Partitional/IOutboxPartitionalSupport.cs +++ b/src/Sa.Outbox/Partitional/IOutboxPartitionalSupport.cs @@ -28,5 +28,11 @@ public interface IOutboxPartitionalSupport /// A task representing the asynchronous operation, containing a read-only collection of . Task> GetMsgParts(CancellationToken cancellationToken); + /// + /// Asynchronously retrieves a collection of tenant-part pairs for scheduled tasks. + /// Similar to but used specifically for task-level partition resolution. + /// + /// A cancellation token to signal the operation's cancellation. + /// A task representing the asynchronous operation, containing a read-only collection of . Task> GetTaskParts(CancellationToken cancellationToken); } diff --git a/src/Sa.Outbox/PlugServices/IOutboxDeliveryManager.cs b/src/Sa.Outbox/PlugServices/IOutboxDeliveryManager.cs index 08f38811..f651300e 100644 --- a/src/Sa.Outbox/PlugServices/IOutboxDeliveryManager.cs +++ b/src/Sa.Outbox/PlugServices/IOutboxDeliveryManager.cs @@ -2,7 +2,8 @@ /// -/// needed External implementation +/// Manages outbox message delivery lifecycle including exclusive rent, return (acknowledge), and lock extension. +/// External implementations must provide the actual persistence and concurrency logic. /// public interface IOutboxDeliveryManager { diff --git a/src/Sa.Outbox/PlugServices/IOutboxTenantDetector.cs b/src/Sa.Outbox/PlugServices/IOutboxTenantDetector.cs index 78f26aa1..30ed9477 100644 --- a/src/Sa.Outbox/PlugServices/IOutboxTenantDetector.cs +++ b/src/Sa.Outbox/PlugServices/IOutboxTenantDetector.cs @@ -5,7 +5,6 @@ namespace Sa.Outbox.PlugServices; /// /// Discovers tenant IDs at runtime from system data (e.g., message queues, inbox tables). /// -/// public interface IOutboxTenantDetector : ITenantSource { /// diff --git a/src/Sa.Outbox/Publication/IOutboxMessagePublisher.cs b/src/Sa.Outbox/Publication/IOutboxMessagePublisher.cs index 6fea2c0e..e5be7e31 100644 --- a/src/Sa.Outbox/Publication/IOutboxMessagePublisher.cs +++ b/src/Sa.Outbox/Publication/IOutboxMessagePublisher.cs @@ -20,8 +20,15 @@ ValueTask Publish( /// - /// Publishes foreach tenants + /// Publishes messages grouped by tenant using a caller-supplied tenant ID resolver. + /// Messages are automatically partitioned by tenant before publishing. /// + /// The type of the messages to be published. + /// A collection of messages to be published. + /// A delegate that returns the tenant ID for each message. + /// A token to monitor for cancellation requests. + /// A representing the asynchronous operation, + /// with the total number of successfully published messages across all tenants as the result. async ValueTask Publish( IReadOnlyCollection messages, Func getTenantId, @@ -37,6 +44,15 @@ async ValueTask Publish( } + /// + /// Publishes messages grouped by tenant, deriving each tenant ID from . + /// Only applicable to message types that implement . + /// + /// The type of the messages to be published, which must implement . + /// A collection of messages to be published. + /// A token to monitor for cancellation requests. + /// A representing the asynchronous operation, + /// with the total number of successfully published messages across all tenants as the result. async ValueTask Publish( IReadOnlyCollection messages, CancellationToken cancellationToken = default) @@ -53,8 +69,14 @@ async ValueTask Publish( /// - /// Publishes a single message. + /// Publishes a single message for a specific tenant. /// + /// The type of the message to be published. + /// The message to publish. + /// The tenant ID under which the message belongs. + /// A token to monitor for cancellation requests. + /// A representing the asynchronous operation, + /// with the number of successfully published messages (always 1 on success) as the result. ValueTask PublishSingle( TMessage message, int tenantId, @@ -62,6 +84,15 @@ ValueTask PublishSingle( => Publish([message], tenantId, cancellationToken); + /// + /// Publishes a single message, deriving the tenant ID from . + /// Only applicable to message types that implement . + /// + /// A message type implementing . + /// The message to publish. + /// A token to monitor for cancellation requests. + /// A representing the asynchronous operation, + /// with the number of successfully published messages (always 1 on success) as the result. ValueTask PublishSingle( TMessage message, CancellationToken cancellationToken = default) diff --git a/src/Sa.Outbox/Setup.cs b/src/Sa.Outbox/Setup.cs index d0eee382..e11da4e1 100644 --- a/src/Sa.Outbox/Setup.cs +++ b/src/Sa.Outbox/Setup.cs @@ -6,6 +6,12 @@ namespace Sa.Outbox; public static class Setup { + /// + /// Registers the Sa.Outbox infrastructure in the service collection and optionally configures it via a builder action. + /// + /// The service collection to add outbox services to. + /// An optional action to configure the outbox through . + /// The updated . public static IServiceCollection AddSaOutbox( this IServiceCollection services, Action? build = null) diff --git a/src/Samples/PgOutbox.ConsoleApp/Program.cs b/src/Samples/PgOutbox.ConsoleApp/Program.cs index b40f07fb..9a4ce4f0 100644 --- a/src/Samples/PgOutbox.ConsoleApp/Program.cs +++ b/src/Samples/PgOutbox.ConsoleApp/Program.cs @@ -44,7 +44,6 @@ .WithOutboxSettings((_, settings) => { settings.TableSettings.WithSchema("test"); - settings.ConsumeSettings.WithMinOffset(DateTimeOffset.Now); }) .WithMessageSerializer(OutboxMessageSerializer.Instance) ) diff --git a/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs b/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs index 1c9ccc22..ac9f3bbc 100644 --- a/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs +++ b/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs @@ -186,28 +186,6 @@ public async Task Deliver_ProcessorThrows_UseDefaultRetryStrategy() #endregion - #region Critical exceptions propagate - - //[Fact] - //public async Task Deliver_ProcessorThrowsCritical_ExceptionPropagates() - //{ - // var ctx = new FakeOutboxContext(payloadId: "msg-critical"); - // var messages = ToMessages(ctx); - - // var criticalException = new AccessViolationException("critical failure"); - // var processor = new FakeDeliveryLifetimeInvoker(_ => Task.FromException(criticalException)); - - // var courier = new DeliveryCourier(processor); - - // var ex = await Assert.ThrowsAsync(async () => - // await courier.Deliver(CreateSettings(), CreateFilter(), messages, CancellationToken.None)); - - // Assert.Same(criticalException, ex); - // // Critical exceptions skip the catch block, so messages are NOT touched - // Assert.Equal(DeliveryStatusCode.Pending, ctx.DeliveryResult.Code); - //} - - #endregion #region Pre-existing warning states @@ -398,7 +376,7 @@ public async Task Deliver_ProcessorCancelled_TreatedAsRegularError() var ctx = new FakeOutboxContext(payloadId: "cancelled"); var messages = ToMessages(ctx); - var cts = new CancellationTokenSource(); + using var cts = new CancellationTokenSource(); cts.Cancel(); var processor = new FakeDeliveryLifetimeInvoker(_ => Task.FromCanceled(cts.Token)); @@ -469,7 +447,7 @@ await courier.Deliver( Assert.Equal("specific error", ctx.Exception!.Message); } - private sealed class CustomTestException(string message) : Exception(message); + public sealed class CustomTestException(string message) : Exception(message); #endregion } From 6b69cf2cb68aa2c7d52957267eff1ac232b19ad9 Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 30 Jun 2026 14:13:19 +0300 Subject: [PATCH 19/33] default settings Signed-off-by: dundich --- .../Delivery/OutboxConsumerSettingsBuilder.cs | 34 +++++++++---------- src/Sa.Outbox/Delivery/OutboxDefaults.cs | 3 ++ 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs b/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs index f95ffbc6..4baafcdc 100644 --- a/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs +++ b/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs @@ -38,23 +38,23 @@ public OutboxConsumerSettings Build() return new OutboxConsumerSettings( _id, _consumerGroupId ?? throw new InvalidOperationException("ConsumerGroupId is required."), - _asSingleton ?? false, - _interval ?? TimeSpan.FromMinutes(1), - _initialDelay ?? TimeSpan.FromSeconds(10), - _concurrencyLimit ?? 1, - _maxConcurrency ?? 48, - _retryCountOnError ?? 1, - _maxBatchSize ?? 16, - _maxProcessingIterations ?? 10, - _iterationDelay ?? TimeSpan.Zero, - _lockDuration ?? TimeSpan.FromSeconds(10), - _lockRenewal ?? TimeSpan.FromSeconds(3), - _lookbackInterval ?? TimeSpan.FromDays(7), - _maxDeliveryAttempts ?? 3, - _batchingWindow ?? TimeSpan.FromSeconds(3), - _perTenantTimeout ?? TimeSpan.Zero, - _perTenantMaxDegreeOfParallelism ?? 1, - _paused ?? false, + _asSingleton ?? OutboxDefaults.AsSingleton, + _interval ?? OutboxDefaults.Interval, + _initialDelay ?? OutboxDefaults.InitialDelay, + _concurrencyLimit ?? OutboxDefaults.ConcurrencyLimit, + _maxConcurrency ?? OutboxDefaults.MaxConcurrency, + _retryCountOnError ?? OutboxDefaults.RetryCountOnError, + _maxBatchSize ?? OutboxDefaults.MaxBatchSize, + _maxProcessingIterations ?? OutboxDefaults.MaxProcessingIterations, + _iterationDelay ?? OutboxDefaults.IterationDelay, + _lockDuration ?? OutboxDefaults.LockDuration, + _lockRenewal ?? OutboxDefaults.LockRenewal, + _lookbackInterval ?? OutboxDefaults.LookbackInterval, + _maxDeliveryAttempts ?? OutboxDefaults.MaxDeliveryAttempts, + _batchingWindow ?? OutboxDefaults.BatchingWindow, + _perTenantTimeout ?? OutboxDefaults.PerTenantTimeout, + _perTenantMaxDegreeOfParallelism ?? OutboxDefaults.PerTenantMaxDegreeOfParallelism, + _paused ?? OutboxDefaults.Paused, 0); } diff --git a/src/Sa.Outbox/Delivery/OutboxDefaults.cs b/src/Sa.Outbox/Delivery/OutboxDefaults.cs index 3f8884a3..6f946dbf 100644 --- a/src/Sa.Outbox/Delivery/OutboxDefaults.cs +++ b/src/Sa.Outbox/Delivery/OutboxDefaults.cs @@ -15,6 +15,9 @@ public static class OutboxDefaults /// Default concurrency limit (1). public static int ConcurrencyLimit => 1; + /// Default singleton mode True. + public static bool AsSingleton => true; + /// Default maximum concurrency (48). public static int MaxConcurrency => 48; From c5fa2b537bfb526fa2860c5adad75b20ddcfd850 Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 30 Jun 2026 15:34:24 +0300 Subject: [PATCH 20/33] ConsumerGroupId as key for consumer Signed-off-by: dundich --- .../Delivery/DeliveryLifetimeInvoker.cs | 10 +++--- ...gsManager.cs => IOutboxConsumerManager.cs} | 2 +- .../Delivery/Job/OutboxSettingsBootstrap.cs | 10 ++---- src/Sa.Outbox/Delivery/Job/Setup.cs | 34 +++++++++---------- ...ngsManager.cs => OutboxConsumerManager.cs} | 10 +++--- .../Delivery/OutboxConsumerSettings.cs | 2 -- .../Delivery/OutboxConsumerSettingsBuilder.cs | 3 -- src/Sa.Outbox/Delivery/Setup.cs | 6 +--- .../Sa.Outbox.Tests/DeliveryCourierTests.cs | 2 +- 9 files changed, 31 insertions(+), 48 deletions(-) rename src/Sa.Outbox/Delivery/{IOutboxSettingsManager.cs => IOutboxConsumerManager.cs} (98%) rename src/Sa.Outbox/Delivery/{OutboxSettingsManager.cs => OutboxConsumerManager.cs} (96%) diff --git a/src/Sa.Outbox/Delivery/DeliveryLifetimeInvoker.cs b/src/Sa.Outbox/Delivery/DeliveryLifetimeInvoker.cs index bc8bfabb..34e5c865 100644 --- a/src/Sa.Outbox/Delivery/DeliveryLifetimeInvoker.cs +++ b/src/Sa.Outbox/Delivery/DeliveryLifetimeInvoker.cs @@ -9,7 +9,7 @@ namespace Sa.Outbox.Delivery; internal sealed class DeliveryLifetimeInvoker(IServiceProvider serviceProvider) : IDeliveryLifetimeInvoker { - private readonly ConcurrentDictionary _singletonConsumers = new(); + private readonly ConcurrentDictionary _singletonConsumers = new(); // Method to process messages using a consumer in scope public Task ConsumeInScope( @@ -40,12 +40,12 @@ private async Task ProcessInNewScope( CancellationToken cancellationToken) { await using AsyncServiceScope scope = serviceProvider.CreateAsyncScope(); - IConsumer consumer = GetConsumer(scope.ServiceProvider, settings.Id); + IConsumer consumer = GetConsumer(scope.ServiceProvider, settings.ConsumerGroupId); await ProcessMessages(consumer, settings, filter, messages, cancellationToken); } - private static IConsumer GetConsumer(IServiceProvider sp, Guid id) - => sp.GetRequiredKeyedService>(id); + private static IConsumer GetConsumer(IServiceProvider sp, string key) + => sp.GetRequiredKeyedService>(key); private static async Task ProcessMessages( IConsumer consumer, @@ -60,7 +60,7 @@ private static async Task ProcessMessages( private IConsumer GetOrCreateSingletonConsumer( OutboxConsumerSettings settings) { - return (IConsumer)_singletonConsumers.GetOrAdd(settings.Id, key => + return (IConsumer)_singletonConsumers.GetOrAdd(settings.ConsumerGroupId, key => GetConsumer(serviceProvider, key)); } } diff --git a/src/Sa.Outbox/Delivery/IOutboxSettingsManager.cs b/src/Sa.Outbox/Delivery/IOutboxConsumerManager.cs similarity index 98% rename from src/Sa.Outbox/Delivery/IOutboxSettingsManager.cs rename to src/Sa.Outbox/Delivery/IOutboxConsumerManager.cs index ac980b44..2487c274 100644 --- a/src/Sa.Outbox/Delivery/IOutboxSettingsManager.cs +++ b/src/Sa.Outbox/Delivery/IOutboxConsumerManager.cs @@ -5,7 +5,7 @@ namespace Sa.Outbox.Delivery; /// Provides atomic snapshot swaps, pause/resume lifecycle, and change subscriptions. /// All settings are immutable — updates create new instances, never mutate existing ones. /// -public interface IOutboxSettingsManager +public interface IOutboxConsumerManager { /// /// Atomically applies a transformation to the current settings for a consumer group. diff --git a/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs b/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs index f62cb06e..fb289ba9 100644 --- a/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs +++ b/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs @@ -1,15 +1,13 @@ -using Microsoft.Extensions.Hosting; - -namespace Sa.Outbox.Delivery.Job; +namespace Sa.Outbox.Delivery.Job; /// /// Bootstrap service that registers all consumer group initial settings into -/// after the DI container is fully built. +/// after the DI container is fully built. /// Runs once at application startup, before any scheduled jobs execute. /// internal sealed class OutboxSettingsBootstrap( IDeliverySnapshot snapshot, - IOutboxSettingsManager settingsManager) : IHostedService + IOutboxConsumerManager settingsManager) { public Task StartAsync(CancellationToken cancellationToken) { @@ -21,6 +19,4 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - - public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; } diff --git a/src/Sa.Outbox/Delivery/Job/Setup.cs b/src/Sa.Outbox/Delivery/Job/Setup.cs index 81e22c83..af6618d7 100644 --- a/src/Sa.Outbox/Delivery/Job/Setup.cs +++ b/src/Sa.Outbox/Delivery/Job/Setup.cs @@ -24,21 +24,11 @@ public static IServiceCollection AddDeliveryJob< builder .WithConsumerGroupId(consumerGroupId) .AsSingleton(isSingleton) - .WithInterval(TimeSpan.FromMinutes(1)) .StartImmediately() .WithConcurrencyLimit(1) .WithMaxConcurrency(1) - .WithRetryCountOnError(3) - .WithMaxBatchSize(16) .WithMaxProcessingIterations(-1) - .WithIterationDelay(TimeSpan.Zero) - .WithLockDuration(TimeSpan.FromSeconds(10)) - .WithLockRenewal(TimeSpan.FromSeconds(3)) - .WithLookbackInterval(TimeSpan.FromDays(7)) - .WithMaxDeliveryAttempts(3) - .WithBatchingWindow(TimeSpan.FromSeconds(3)) .WithPerTenantTimeout(TimeSpan.Zero) - .WithPerTenantMaxDegreeOfParallelism(1) .Paused(false); // Allow caller to tweak settings via fluent builder @@ -46,14 +36,7 @@ public static IServiceCollection AddDeliveryJob< var settings = builder.Build(); - if (isSingleton) - { - services.AddKeyedSingleton, TConsumer>(settings.Id); - } - else - { - services.AddKeyedScoped, TConsumer>(settings.Id); - } + Registered(services, isSingleton, settings.ConsumerGroupId); services.AddSaSchedule(builder => { @@ -83,4 +66,19 @@ public static IServiceCollection AddDeliveryJob< return services; } + + private static void Registered<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( + IServiceCollection services, + bool isSingleton, + string consumerGroupId) where TConsumer : class, IConsumer + { + if (isSingleton) + { + services.AddKeyedSingleton, TConsumer>(consumerGroupId); + } + else + { + services.AddKeyedScoped, TConsumer>(consumerGroupId); + } + } } diff --git a/src/Sa.Outbox/Delivery/OutboxSettingsManager.cs b/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs similarity index 96% rename from src/Sa.Outbox/Delivery/OutboxSettingsManager.cs rename to src/Sa.Outbox/Delivery/OutboxConsumerManager.cs index 4f978bb8..42dc7322 100644 --- a/src/Sa.Outbox/Delivery/OutboxSettingsManager.cs +++ b/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs @@ -1,12 +1,10 @@ -using Sa.Outbox.Delivery; - -namespace Sa.Outbox.Delivery; +namespace Sa.Outbox.Delivery; /// /// Thread-safe manager for runtime control of outbox consumer group settings. /// Uses atomic immutable snapshots — no mutation during active delivery, no race conditions. /// -internal sealed class OutboxSettingsManager : IOutboxSettingsManager +internal sealed class OutboxConsumerManager : IOutboxConsumerManager { private readonly Dictionary _settings = []; private readonly Dictionary>> _listeners = []; @@ -184,12 +182,12 @@ internal void NotifyListeners(string consumerGroupId, OutboxConsumerSettings new private sealed class Subscription : IDisposable { - private readonly OutboxSettingsManager _manager; + private readonly OutboxConsumerManager _manager; private readonly string _consumerGroupId; private readonly Action _callback; private bool _disposed; - internal Subscription(OutboxSettingsManager manager, string consumerGroupId, Action callback) + internal Subscription(OutboxConsumerManager manager, string consumerGroupId, Action callback) { _manager = manager; _consumerGroupId = consumerGroupId; diff --git a/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs b/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs index f8e67512..d8b37c10 100644 --- a/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs +++ b/src/Sa.Outbox/Delivery/OutboxConsumerSettings.cs @@ -7,8 +7,6 @@ /// public sealed record OutboxConsumerSettings( - Guid Id, - /// /// Unique identifier for the consumer group. Groups settings for a single logical consumer. /// diff --git a/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs b/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs index 4baafcdc..da2e2436 100644 --- a/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs +++ b/src/Sa.Outbox/Delivery/OutboxConsumerSettingsBuilder.cs @@ -6,8 +6,6 @@ /// public sealed class OutboxConsumerSettingsBuilder { - private readonly Guid _id = Guid.NewGuid(); - private string? _consumerGroupId; private bool? _asSingleton; private TimeSpan? _interval; @@ -36,7 +34,6 @@ public sealed class OutboxConsumerSettingsBuilder public OutboxConsumerSettings Build() { return new OutboxConsumerSettings( - _id, _consumerGroupId ?? throw new InvalidOperationException("ConsumerGroupId is required."), _asSingleton ?? OutboxDefaults.AsSingleton, _interval ?? OutboxDefaults.Interval, diff --git a/src/Sa.Outbox/Delivery/Setup.cs b/src/Sa.Outbox/Delivery/Setup.cs index 4a08d218..ca609445 100644 --- a/src/Sa.Outbox/Delivery/Setup.cs +++ b/src/Sa.Outbox/Delivery/Setup.cs @@ -1,6 +1,5 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -using Sa.Outbox.Delivery.Job; using Sa.Outbox.Metadata; using Sa.Outbox.Partitional; @@ -33,13 +32,10 @@ public static IServiceCollection AddOutboxDelivery( // DeliverySnapshot теперь собирает настройки из AddDeliveryJob через статический регистр services.TryAddSingleton(); - services.TryAddSingleton(); + services.TryAddSingleton(); configure?.Invoke(new DeliveryBuilder(services)); - // Bootstrap: register all consumer group initial settings into IOutboxSettingsManager. - services.AddHostedService(); - return services; } } diff --git a/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs b/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs index ac9f3bbc..f417db97 100644 --- a/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs +++ b/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs @@ -7,7 +7,7 @@ public class DeliveryCourierTests private sealed class TestMessage { } private static OutboxConsumerSettings CreateSettings(int maxDeliveryAttempts = 3) - => new(Id: Guid.NewGuid(), "test-group", AsSingleton: false, Interval: TimeSpan.FromMinutes(1), InitialDelay: TimeSpan.Zero, + => new(ConsumerGroupId: "test-group", AsSingleton: false, Interval: TimeSpan.FromMinutes(1), InitialDelay: TimeSpan.Zero, ConcurrencyLimit: 1, MaxConcurrency: 1, RetryCountOnError: 0, MaxBatchSize: 16, MaxProcessingIterations: -1, IterationDelay: TimeSpan.Zero, LockDuration: TimeSpan.FromSeconds(10), LockRenewal: TimeSpan.FromSeconds(3), From 065a1587fe3994e3c5528b28f6b8807be0b92c28 Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 30 Jun 2026 16:24:24 +0300 Subject: [PATCH 21/33] added IOutboxConsumerManager Signed-off-by: dundich --- src/Sa.Outbox.PostgreSql/Readme.md | 87 ++++++++++++------- .../Delivery/IOutboxConsumerManager.cs | 4 +- src/Sa.Outbox/Delivery/Job/DeliveryJob.cs | 17 +++- .../Delivery/Job/OutboxSettingsBootstrap.cs | 22 ----- .../Delivery/OutboxConsumerManager.cs | 2 +- src/Sa.Outbox/Readme.md | 69 ++++++++------- 6 files changed, 108 insertions(+), 93 deletions(-) delete mode 100644 src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs diff --git a/src/Sa.Outbox.PostgreSql/Readme.md b/src/Sa.Outbox.PostgreSql/Readme.md index 4d8549af..e5610f71 100644 --- a/src/Sa.Outbox.PostgreSql/Readme.md +++ b/src/Sa.Outbox.PostgreSql/Readme.md @@ -23,8 +23,13 @@ IHost host = Host.CreateDefaultBuilder() .WithDeliveries(b => b .AddDeliveryScoped((_, s) => { - s.ScheduleSettings.WithIntervalSeconds(5).WithImmediate(); - s.ConsumeSettings.WithMaxBatchSize(16); + s + .WithInterval(TimeSpan.FromSeconds(5)) + .StartImmediately() + .WithMaxBatchSize(16) + .WithMaxDeliveryAttempts(3) + .WithLockDuration(TimeSpan.FromSeconds(10)) + .WithLookbackInterval(TimeSpan.FromDays(7)); }) ) ) @@ -60,7 +65,7 @@ public sealed record OrderCreated(string PayloadId, string ProductName); public sealed class OrderConsumer : IConsumer { public async ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken ct) @@ -80,6 +85,7 @@ That's it. The library handles: - Concurrent consumption via `SKIP LOCKED` - Retry / postpone / error workflows - Automatic cleanup of old partitions +- Self-bootstrapping of consumer settings into `IOutboxConsumerManager` --- @@ -124,44 +130,67 @@ Where inside `AddSaOutbox`: // With inline settings .AddDeliveryScoped((_, settings) => { - // Schedule — how often to poll - settings.ScheduleSettings + settings .WithInterval(TimeSpan.FromSeconds(5)) - .WithImmediate(); // start immediately, don't wait first interval - - // Consumption limits - settings.ConsumeSettings - .WithMaxBatchSize(16) // max messages per batch - .WithMaxDeliveryAttempts(3) // stop retrying after N attempts - .WithBatchingWindow(TimeSpan.FromSeconds(2)) - .WithLockDuration(TimeSpan.FromMinutes(10)); + .StartImmediately() // start immediately, don't wait first interval + .WithMaxBatchSize(16) // max messages per batch + .WithMaxDeliveryAttempts(3) // stop retrying after N attempts + .WithLockDuration(TimeSpan.FromSeconds(10)) + .WithLookbackInterval(TimeSpan.FromDays(7)); }) ) // ... ) ``` -#### ConsumeSettings reference +### Настройки consumer group -| Method | Default | Description | -|---|---|---| -| `WithInterval(interval)` | 5 s | Polling frequency | -| `WithImmediate()` | — | Don't wait for first interval | -| `WithMaxBatchSize(n)` | 16 | Max messages per batch | -| `WithMaxDeliveryAttempts(n)` | ∞ | Stop retrying after N attempts | -| `WithBatchingWindow(span)` | 2 s | Wait up to this long to fill a batch | -| `WithLockDuration(span)` | 10 m | Task lock TTL before forced expiry | -| `WithSingleIteration()` | — | Process once then stop (testing) | -| `WithNoBatchingWindow()` | — | Take whatever is available now | +`OutboxConsumerSettingsBuilder` — единый fluent-билдер для всех параметров: -#### Dynamic adjustments inside `Consume()` +| Метод | По умолчанию | Описание | +|---|---|---| +| `WithInterval(span)` | 5 с | Период опроса | +| `StartImmediately()` | — | Старт без ожидания первого интервала | +| `WithMaxBatchSize(n)` | 16 | Макс. сообщений за батч | +| `WithMaxDeliveryAttempts(n)` | 3 | Стоп-повторы после N попыток | +| `WithLockDuration(span)` | 10 с | TTL блокировки сообщения | +| `WithLockRenewal(span)` | 3 с | Период продления блокировки | +| `WithLookbackInterval(span)` | 7 дн | История поиска необработанных | +| `WithBatchingWindow(span)` | 0 с | Окно агрегации сообщений | +| `WithNoBatchingWindow()` | — | Взять всё доступное сейчас | +| `WithConcurrencyLimit(n)` | 1 | Одновременных задач | +| `WithMaxConcurrency(n)` | 1 | Макс. параллельных процессоров | +| `WithRetryCountOnError(n)` | 0 | Повторы при ошибке (-1 = бесконечно) | +| `WithMaxProcessingIterations(n)` | -1 | Итераций за цикл (-1 = безлимитно) | +| `WithSingleIteration()` | — | Одна итерация (тестирование) | +| `WithUnlimitedIterations()` | — | Безлимитные итерации | +| `WithSequentialProcessing()` | 1 | Последовательная обработка по тенантам | +| `WithMaxParallelism()` | CPU count | Максимальная параллельность по тенантам | +| `WithPerTenantTimeout(span)` | 0 | Таймаут обработки одного тенанта | +| `Paused(bool)` | false | Пауза consumer group | + +### Runtime-управление настройками + +Настройки автоматически регистрируются в `IOutboxConsumerManager` при первом запуске job'а. Для runtime-изменений: ```csharp -public async ValueTask Consume(ConsumerGroupSettings settings, ...) +var manager = host.Services.GetRequiredService(); + +// Atomic swap — новый снимок применяется на следующей итерации +manager.Apply("cg_order_consumer", s => s with { MaxBatchSize = 64 }); + +// Pause / Resume +manager.Pause("cg_order_consumer"); +manager.Resume("cg_order_consumer"); + +// Подписка на изменения +using var sub = manager.Subscribe("cg_order_consumer", updated => { - // Change behaviour mid-processing - settings.ConsumeSettings.WithMaxProcessingIterations(100); -} + // реакция на изменение настроек +}); + +// Проверка состояния +bool paused = manager.IsPaused("cg_order_consumer"); ``` ### 3. PostgreSQL Connection diff --git a/src/Sa.Outbox/Delivery/IOutboxConsumerManager.cs b/src/Sa.Outbox/Delivery/IOutboxConsumerManager.cs index 2487c274..91086586 100644 --- a/src/Sa.Outbox/Delivery/IOutboxConsumerManager.cs +++ b/src/Sa.Outbox/Delivery/IOutboxConsumerManager.cs @@ -20,9 +20,7 @@ public interface IOutboxConsumerManager /// Registers a consumer group with initial settings. /// Unlike , this does not require prior registration. /// - /// The consumer group identifier. - /// The initial immutable settings snapshot. - void Register(string consumerGroupId, OutboxConsumerSettings settings); + internal void Register(string consumerGroupId, OutboxConsumerSettings settings); /// /// Retrieves the current immutable settings snapshot. Thread-safe. diff --git a/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs b/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs index 8d9a2569..7fc486b1 100644 --- a/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs +++ b/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs @@ -6,12 +6,23 @@ namespace Sa.Outbox.Delivery.Job; public interface IDeliveryJob : IJob; -internal sealed class DeliveryJob(IDeliveryProcessor processor) : IDeliveryJob +internal sealed class DeliveryJob( + IDeliveryProcessor processor, + IOutboxConsumerManager settingsManager) : IDeliveryJob { public async Task Execute(IJobContext context, CancellationToken cancellationToken) { - OutboxConsumerSettings settings = context.Settings.Properties.GetConsumerGroupSettings() - ?? throw new InvalidOperationException("Missing OutboxConsumerSettings tag on job."); + var settings = settingsManager.Get(context.JobName); + + if (settings is null) + { + // Auto-bootstrap: first execution hasn't been registered yet. + settings = context.Settings.Properties.GetConsumerGroupSettings() + ?? throw new InvalidOperationException( + $"No OutboxConsumerSettings for consumer group '{context.JobName}'."); + + settingsManager.Register(context.JobName, settings); + } await processor.ProcessMessages(settings, cancellationToken); } diff --git a/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs b/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs deleted file mode 100644 index fb289ba9..00000000 --- a/src/Sa.Outbox/Delivery/Job/OutboxSettingsBootstrap.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace Sa.Outbox.Delivery.Job; - -/// -/// Bootstrap service that registers all consumer group initial settings into -/// after the DI container is fully built. -/// Runs once at application startup, before any scheduled jobs execute. -/// -internal sealed class OutboxSettingsBootstrap( - IDeliverySnapshot snapshot, - IOutboxConsumerManager settingsManager) -{ - public Task StartAsync(CancellationToken cancellationToken) - { - foreach (var settings in snapshot.ConsumerSettings) - { - // Register the settings directly — no conversion needed anymore. - settingsManager.Register(settings.ConsumerGroupId, settings); - } - - return Task.CompletedTask; - } -} diff --git a/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs b/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs index 42dc7322..999f021b 100644 --- a/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs +++ b/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs @@ -21,7 +21,7 @@ public void Register(string consumerGroupId, OutboxConsumerSettings settings) lock (_lock) { _settings[consumerGroupId] = settings; - + if (!_listeners.ContainsKey(consumerGroupId)) { _listeners[consumerGroupId] = []; diff --git a/src/Sa.Outbox/Readme.md b/src/Sa.Outbox/Readme.md index ebff10d5..279ee957 100644 --- a/src/Sa.Outbox/Readme.md +++ b/src/Sa.Outbox/Readme.md @@ -26,36 +26,6 @@ builder.Services ); ``` -### 3. Публикация сообщений - -```csharp -public sealed record OrderCreated(string OrderId); - -await publisher.Publish( - [new OrderCreated("ORD-001"), new OrderCreated("ORD-002")], - tenantId: 1); -``` - -### 4. Потребление сообщений - -```csharp -sealed class OrderCreatedConsumer : IConsumer -{ - public async ValueTask Consume( - ConsumerGroupSettings settings, - OutboxMessageFilter filter, - ReadOnlyMemory> messages, - CancellationToken cancellationToken) - { - foreach (var msg in messages.Span) - { - // обработка... - msg.Ok($"Processed order {msg.Payload.OrderId}"); - } - } -} -``` - ## Архитектура ``` @@ -86,8 +56,10 @@ sealed class OrderCreatedConsumer : IConsumer | `IOutboxMessagePublisher` | Публикация сообщений в outbox | | `IConsumer\` | Интерфейс потребителя сообщений | | `IOutboxContextOperations\` | Операции изменения статуса доставки | -| `ConsumeSettings` | Настройки потребления (батчи, блокировки, повторы) | -| `ConsumerGroupSettings` | Группа потребителей + расписание | +| `OutboxConsumerSettings` | Единый immutable-снимок настроек consumer group (интервал, батчи, параллельность, повторы и т.д.) | +| `OutboxConsumerSettingsBuilder` | Fluent-билдер для создания и частичного обновления `OutboxConsumerSettings` | +| `IOutboxConsumerManager` | Runtime-менеджер настроек: atomic swap, pause/resume, подписки на изменения | +| `DeliverySnapshot` | Считывает настройки из статического регистра Schedule после билда DI | | `DeliveryStatus` / `DeliveryStatusCode` | HTTP-подобные статусы доставки | | `ExponentialBackoffRetryStrategy` | Экспоненциальный бэкофф с джиттером | | `OutboxPartInfo` | Информация о части: TenantId, PartName | @@ -118,11 +90,10 @@ builder.Services.AddSaOutbox(builder => builder .WithDeliveries(d => d // Singleton delivery (один экземпляр на всё приложение) .AddDelivery("orders", (sp, cs) => { - cs.ConsumeSettings + cs .WithMaxBatchSize(32) .WithLockDuration(TimeSpan.FromSeconds(10)) - .WithMaxDeliveryAttempts(5); - cs.ScheduleSettings + .WithMaxDeliveryAttempts(5) .WithInterval(TimeSpan.FromSeconds(30)) .WithInitialDelay(TimeSpan.FromSeconds(5)); }) @@ -132,8 +103,31 @@ builder.Services.AddSaOutbox(builder => builder ); ``` +> **Self-bootstrapping:** `DeliveryJob` автоматически регистрирует настройки в `IOutboxConsumerManager` при первом запуске. Отдельный bootstrap-сервис не нужен — каждый job читает актуальные снимки из менеджера, включая runtime-изменения через `Apply()`. + +### Runtime-управление настройками + +`IOutboxConsumerManager` позволяет изменять настройки без перезапуска: + +```csharp +// Atomic swap — новый снимок применяется атомарно +manager.Apply("orders", s => s with { MaxBatchSize = 64 }); + +// Pause / Resume +manager.Pause("orders"); +manager.Resume("orders"); + +// Подписка на изменения +using var sub = manager.Subscribe("orders", updated => +{ + // реакция на изменение настроек +}); +``` + ### Настройки потребления +`OutboxConsumerSettings` — единый immutable record. Все параметры задаются через `OutboxConsumerSettingsBuilder`: + | Параметр | По умолчанию | Описание | |----------|-------------|----------| | `MaxBatchSize` | 16 | Макс. размер батча | @@ -142,7 +136,12 @@ builder.Services.AddSaOutbox(builder => builder | `MaxDeliveryAttempts` | 3 | Максимум попыток доставки | | `LookbackInterval` | 7 дней | История обработки | | `ConcurrencyLimit` | 1 | Одновременных задач | +| `MaxConcurrency` | 1 | Макс. параллельных процессоров | | `PerTenantMaxDegreeOfParallelism` | 1 | Параллельность по тенантам | +| `RetryCountOnError` | 0 | Повторы при ошибке (-1 = бесконечно) | +| `MaxProcessingIterations` | -1 | Итераций за цикл (-1 = безлимитно) | +| `BatchingWindow` | 0 сек | Окно агрегации сообщений | +| `Paused` | false | Флаг паузы consumer group | ### Мультитенантность From 86b29d1cdc3f6c19e22c4936d643108b716887b5 Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 30 Jun 2026 22:23:18 +0300 Subject: [PATCH 22/33] tests Signed-off-by: dundich --- src/Sa.Outbox/Delivery/DeliveryProcessor.cs | 15 +- src/Sa.Outbox/Delivery/Job/Setup.cs | 3 +- src/Sa.Outbox/Sa.Outbox.csproj | 1 + .../DeliveryConsumerGroupManagerTests.cs | 706 ++++++++++++++++++ .../OutboxConsumerManagerTests.cs | 610 +++++++++++++++ 5 files changed, 1332 insertions(+), 3 deletions(-) create mode 100644 src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryConsumerGroupManagerTests.cs create mode 100644 src/Tests/Sa.Outbox.Tests/OutboxConsumerManagerTests.cs diff --git a/src/Sa.Outbox/Delivery/DeliveryProcessor.cs b/src/Sa.Outbox/Delivery/DeliveryProcessor.cs index 25909978..7e54cbd6 100644 --- a/src/Sa.Outbox/Delivery/DeliveryProcessor.cs +++ b/src/Sa.Outbox/Delivery/DeliveryProcessor.cs @@ -10,6 +10,11 @@ internal sealed class DeliveryProcessor( IDeliveryTenant processor, ITenantProvider tenantProvider) : IDeliveryProcessor { + /// + /// Delay when consumer group is paused — avoids busy-waiting on repeated polls. + /// + private static readonly TimeSpan PausedPollDelay = TimeSpan.FromSeconds(5); + public async Task ProcessMessages( OutboxConsumerSettings settings, CancellationToken cancellationToken) @@ -17,7 +22,7 @@ public async Task ProcessMessages( if (settings.Paused) { // Consumer group is paused — do not poll. - await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + await Task.Delay(PausedPollDelay, cancellationToken); return 0; } @@ -33,6 +38,14 @@ public async Task ProcessMessages( bool continueProcessing; do { + // Re-check Paused on each iteration — a runtime Pause() should interrupt + // the greedy loop, not wait for all pending messages to drain. + if (settings.Paused) + { + await Task.Delay(PausedPollDelay, cancellationToken); + return totalProcessed; + } + if (iterations > 0 && settings.IterationDelay > TimeSpan.Zero) { await Task.Delay(settings.IterationDelay, cancellationToken); diff --git a/src/Sa.Outbox/Delivery/Job/Setup.cs b/src/Sa.Outbox/Delivery/Job/Setup.cs index af6618d7..54da7c55 100644 --- a/src/Sa.Outbox/Delivery/Job/Setup.cs +++ b/src/Sa.Outbox/Delivery/Job/Setup.cs @@ -28,8 +28,7 @@ public static IServiceCollection AddDeliveryJob< .WithConcurrencyLimit(1) .WithMaxConcurrency(1) .WithMaxProcessingIterations(-1) - .WithPerTenantTimeout(TimeSpan.Zero) - .Paused(false); + .WithPerTenantTimeout(TimeSpan.Zero); // Allow caller to tweak settings via fluent builder configure?.Invoke(default!, builder); diff --git a/src/Sa.Outbox/Sa.Outbox.csproj b/src/Sa.Outbox/Sa.Outbox.csproj index 3d514150..848443ae 100644 --- a/src/Sa.Outbox/Sa.Outbox.csproj +++ b/src/Sa.Outbox/Sa.Outbox.csproj @@ -30,6 +30,7 @@ + diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryConsumerGroupManagerTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryConsumerGroupManagerTests.cs new file mode 100644 index 00000000..0812139e --- /dev/null +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryConsumerGroupManagerTests.cs @@ -0,0 +1,706 @@ +using Microsoft.Extensions.DependencyInjection; +using Sa.Outbox.Delivery; +using Sa.Outbox.Publication; + +namespace Sa.Outbox.PostgreSqlTests.Delivery; + +/// +/// Интеграционные тесты для с реальным Consume через PostgreSQL. +/// Проверяют pause/resume, Apply (runtime settings), Subscribe и Unregister в контексте обработки сообщений. +/// +[Collection("sequential")] +public sealed class DeliveryConsumerGroupManagerTests(DeliveryConsumerGroupManagerTests.Fixture fixture) + : IClassFixture +{ + /// + /// Счётчик вызовов Consume — используется для проверки, что потребитель обрабатывает сообщения. + /// + class CountingMessageConsumer : IConsumer + { + public static int ConsumeCount; + public static int TotalMessagesConsumed; + public static List BatchSizes = []; + public static ManualResetEventSlim? BlockConsume; + public static ManualResetEventSlim AllowConsume = default!; + + static CountingMessageConsumer() + { + Reset(); + } + + public static void Reset() + { + ConsumeCount = 0; + TotalMessagesConsumed = 0; + BatchSizes = []; + BlockConsume = new ManualResetEventSlim(false); + AllowConsume = new ManualResetEventSlim(true); + } + + public async ValueTask Consume( + OutboxConsumerSettings settings, + OutboxMessageFilter filter, + ReadOnlyMemory> messages, + CancellationToken cancellationToken) + { + Interlocked.Increment(ref ConsumeCount); + Interlocked.Add(ref TotalMessagesConsumed, messages.Length); + BatchSizes.Add(messages.Length); + + // Ждем разрешения, чтобы контролировать время жизни Consume + if (BlockConsume is not null && !BlockConsume.IsSet) + { + await Task.Delay(50, cancellationToken); + } + + AllowConsume.Wait(cancellationToken); + } + } + + /// + /// Потребитель, который блокируется навсегда после первого Consume — для тестов Pause. + /// + class BlockingMessageConsumer : IConsumer + { + public static int ConsumeCount; + public static ManualResetEventSlim StartedBlocking = default!; + public static ManualResetEventSlim ReleaseBlocking = default!; + + static BlockingMessageConsumer() + { + Reset(); + } + + public static void Reset() + { + ConsumeCount = 0; + StartedBlocking = new ManualResetEventSlim(false); + ReleaseBlocking = new ManualResetEventSlim(false); + } + + public async ValueTask Consume( + OutboxConsumerSettings settings, + OutboxMessageFilter filter, + ReadOnlyMemory> messages, + CancellationToken cancellationToken) + { + Interlocked.Increment(ref ConsumeCount); + StartedBlocking.Set(); + + // Auto-release on token cancellation to prevent permanent hang. + using var registration = cancellationToken.Register(() => ReleaseBlocking.Set()); + + // Block until explicitly released or token is cancelled. + await Task.Delay(-1, cancellationToken); + ReleaseBlocking.Wait(cancellationToken); + } + } + + public class Fixture : OutboxPostgreSqlFixture + { + public OutboxConsumerSettings SettingsForTestGroup = default!; + /// Реальное (санитизированное) имя группы для CountingMessageConsumer. + public string CountingGroupId => SettingsForTestGroup.ConsumerGroupId; + /// Реальное (санитизированное) имя группы для BlockingMessageConsumer. + public static string BlockingGroupId => "mgr_blocking"; + public IOutboxConsumerManager ConsumerManager => GetInitializedManager(); + + private IOutboxConsumerManager? _cachedManager; + + private IOutboxConsumerManager GetInitializedManager() + { + if (_cachedManager is not null) return _cachedManager; + + var manager = ServiceProvider.GetRequiredService(); + + // Отладка: проверяем значения + var groupId = CountingGroupId; + var blockingId = BlockingGroupId; + + // Авто-регистрация групп при первом доступе (имитирует поведение DeliveryJob) + if (!manager.IsRegistered(groupId)) + manager.Register(groupId, SettingsForTestGroup); + + if (!manager.IsRegistered(blockingId)) + { + var blockingSettings = new OutboxConsumerSettings( + ConsumerGroupId: BlockingGroupId, + AsSingleton: false, + Interval: TimeSpan.FromMilliseconds(200), + InitialDelay: TimeSpan.Zero, + ConcurrencyLimit: 1, + MaxConcurrency: 1, + RetryCountOnError: 0, + MaxBatchSize: 1, + MaxProcessingIterations: -1, + IterationDelay: TimeSpan.Zero, + LockDuration: TimeSpan.FromMinutes(5), + LockRenewal: TimeSpan.FromMinutes(5), + LookbackInterval: TimeSpan.FromDays(7), + MaxDeliveryAttempts: 3, + BatchingWindow: TimeSpan.Zero, + PerTenantTimeout: TimeSpan.Zero, + PerTenantMaxDegreeOfParallelism: 1, + Paused: false, + Version: 0); + manager.Register(BlockingGroupId, blockingSettings); + } + + return _cachedManager = manager; + } + public IOutboxMessagePublisher Publisher => ServiceProvider.GetRequiredService(); + + public Fixture() : base() + { + Services + .AddSaOutbox(builder => builder + .WithTenants((_, s) => s.WithTenantIds(1)) + .WithDeliveries(deliveryBuilder => deliveryBuilder + .AddDeliveryScoped("counting", (_, b) => + { + b.WithInterval(TimeSpan.FromMilliseconds(200)) + .WithMaxBatchSize(4) + .WithNoLockDuration() + .WithLockRenewal(TimeSpan.FromMinutes(5)) + .WithNoBatchingWindow(); + + SettingsForTestGroup = b.Build(); + }) + .AddDeliveryScoped("mgr_blocking", (_, b) => + { + b.WithInterval(TimeSpan.FromMilliseconds(200)) + .WithMaxBatchSize(1) + .WithNoLockDuration() + .WithLockRenewal(TimeSpan.FromMinutes(5)) + .WithNoBatchingWindow(); + }) + ) + ); + } + + } + + #region Pause / Resume + + [Fact] + public async Task Manager_Pause_DuringProcessing_StopsNextPoll() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + var publisher = fixture.Publisher; + + CountingMessageConsumer.Reset(); + + // Publish messages + var messages = Enumerable.Range(1, 8) + .Select(i => new TestMessage { PayloadId = $"pause-{i}", Content = $"Msg {i}", TenantId = 1 }) + .ToList(); + + await publisher.Publish(messages, m => m.TenantId, TestContext.Current.CancellationToken); + + // Process first batch + var result = await fixture.Sub.ProcessMessages(fixture.SettingsForTestGroup, TestContext.Current.CancellationToken); + Assert.True(result > 0, "Ожидалась обработка хотя бы одного сообщения"); + + int consumedBeforePause = CountingMessageConsumer.TotalMessagesConsumed; + Assert.True(consumedBeforePause > 0, "Первый Consume должен был выполниться"); + + // Если первое ProcessMessages обработало все сообщения — публикуем ещё для проверки Pause/Resume + if (consumedBeforePause >= 8) + { + var extraMessages = new[] { new TestMessage { PayloadId = "pause-extra", Content = "Extra", TenantId = 1 } }; + await publisher.Publish(extraMessages, m => m.TenantId, TestContext.Current.CancellationToken); + } + + // Pause the consumer group + manager.Pause(group); + Assert.True(manager.IsPaused(group), "Группа должна быть паузнутой"); + + // ProcessMessages with paused settings should return 0 + var pausedSettings = manager.Get(group) ?? fixture.SettingsForTestGroup; + result = await fixture.Sub.ProcessMessages(pausedSettings, TestContext.Current.CancellationToken); + Assert.Equal(0, result); + + // After pause, no new messages should be processed + await Task.Delay(300, TestContext.Current.CancellationToken); + Assert.Equal(consumedBeforePause, CountingMessageConsumer.TotalMessagesConsumed); + + // Resume + manager.Resume(group); + Assert.False(manager.IsPaused(group)); + + // After resume, read updated settings from manager and process remaining messages + var resumedSettings = manager.Get(group) ?? fixture.SettingsForTestGroup; + result = await fixture.Sub.ProcessMessages(resumedSettings, TestContext.Current.CancellationToken); + Assert.True(result > 0, "После Resume должны обработаться оставшиеся сообщения"); + } + + [Fact] + public async Task Manager_Resume_AfterPause_ResumesProcessing() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + var publisher = fixture.Publisher; + + CountingMessageConsumer.Reset(); + + // Publish more messages than one batch + var messages = Enumerable.Range(1, 16) + .Select(i => new TestMessage { PayloadId = $"resume-{i}", Content = $"Msg {i}", TenantId = 1 }) + .ToList(); + + await publisher.Publish(messages, m => m.TenantId, TestContext.Current.CancellationToken); + + // Pause immediately + manager.Pause(group); + + // Wait a bit — nothing should be processed while paused + await Task.Delay(500, TestContext.Current.CancellationToken); + Assert.Equal(0, CountingMessageConsumer.ConsumeCount); + + // Resume and process + manager.Resume(group); + + var result = await fixture.Sub.ProcessMessages(fixture.SettingsForTestGroup, TestContext.Current.CancellationToken); + Assert.True(result > 0, "После Resume должны обработаться сообщения"); + Assert.True(CountingMessageConsumer.ConsumeCount > 0, "Consume должен был вызваться после Resume"); + } + + [Fact] + public async Task Manager_PauseAndResume_MessagesProcessedAfterResume() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + var publisher = fixture.Publisher; + + CountingMessageConsumer.Reset(); + const int totalMessages = 10; + + var messages = Enumerable.Range(1, totalMessages) + .Select(i => new TestMessage { PayloadId = $"cycle-{i}", Content = $"Msg {i}", TenantId = 1 }) + .ToList(); + + await publisher.Publish(messages, m => m.TenantId, TestContext.Current.CancellationToken); + + // Process first batch + await fixture.Sub.ProcessMessages(fixture.SettingsForTestGroup, TestContext.Current.CancellationToken); + int firstBatchCount = CountingMessageConsumer.TotalMessagesConsumed; + Assert.True(firstBatchCount > 0 && firstBatchCount <= totalMessages); + + // Если первое ProcessMessages обработало все сообщения — публикуем ещё для проверки Pause/Resume + if (firstBatchCount >= totalMessages) + { + var extraMessages = new[] { new TestMessage { PayloadId = "cycle-extra", Content = "Extra", TenantId = 1 } }; + await publisher.Publish(extraMessages, m => m.TenantId, TestContext.Current.CancellationToken); + } + + // Pause + manager.Pause(group); + await Task.Delay(300, TestContext.Current.CancellationToken); + + // Should NOT process more while paused + int beforeResume = CountingMessageConsumer.TotalMessagesConsumed; + Assert.Equal(firstBatchCount, beforeResume); + + // Resume + manager.Resume(group); + + // Process remaining — read updated settings from manager + var resumedSettings = manager.Get(group) ?? fixture.SettingsForTestGroup; + await fixture.Sub.ProcessMessages(resumedSettings, TestContext.Current.CancellationToken); + Assert.True(CountingMessageConsumer.TotalMessagesConsumed > firstBatchCount, + "После Resume должны обработаться дополнительные сообщения"); + } + + #endregion + + #region Apply (Runtime Settings Changes) + + [Fact] + public async Task Manager_Apply_MaxBatchSize_AffectsNextBatch() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + var publisher = fixture.Publisher; + + CountingMessageConsumer.Reset(); + CountingMessageConsumer.BatchSizes.Clear(); + + const int initialMaxBatch = 4; + const int newMaxBatch = 1; + + var messages = Enumerable.Range(1, 8) + .Select(i => new TestMessage { PayloadId = $"batchsize-{i}", Content = $"Msg {i}", TenantId = 1 }) + .ToList(); + + await publisher.Publish(messages, m => m.TenantId, TestContext.Current.CancellationToken); + + // First batch should use initial MaxBatchSize + await fixture.Sub.ProcessMessages(fixture.SettingsForTestGroup, TestContext.Current.CancellationToken); + Assert.True(CountingMessageConsumer.ConsumeCount > 0); + Assert.True(CountingMessageConsumer.BatchSizes.LastOrDefault() <= initialMaxBatch); + + // Apply new MaxBatchSize via manager + manager.Apply(group, s => s with { MaxBatchSize = newMaxBatch }); + + var updatedSettings = manager.Get(group); + Assert.NotNull(updatedSettings); + Assert.Equal(newMaxBatch, updatedSettings.MaxBatchSize); + + // Next batch should respect new MaxBatchSize + CountingMessageConsumer.BatchSizes.Clear(); + await fixture.Sub.ProcessMessages(fixture.SettingsForTestGroup, TestContext.Current.CancellationToken); + + if (CountingMessageConsumer.BatchSizes.Count > 0) + { + Assert.True(CountingMessageConsumer.BatchSizes.All(bs => bs <= newMaxBatch), + "Каждый батч после Apply не должен превышать новый MaxBatchSize"); + } + } + + [Fact] + public async Task Manager_Apply_ConsecutiveUpdates_AccumulateCorrectly() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + var publisher = fixture.Publisher; + + CountingMessageConsumer.Reset(); + + var messages = Enumerable.Range(1, 12) + .Select(i => new TestMessage { PayloadId = $"chained-{i}", Content = $"Msg {i}", TenantId = 1 }) + .ToList(); + + await publisher.Publish(messages, m => m.TenantId, TestContext.Current.CancellationToken); + + // Chain multiple Apply calls + manager.Apply(group, s => s with { MaxBatchSize = 2 }); + manager.Apply(group, s => s with { MaxDeliveryAttempts = 5 }); + manager.Apply(group, s => s with { MaxBatchSize = 1 }); + + var finalSettings = manager.Get(group); + Assert.NotNull(finalSettings); + Assert.Equal(1, finalSettings.MaxBatchSize); + Assert.Equal(5, finalSettings.MaxDeliveryAttempts); + + // Process — should use final settings + var result = await fixture.Sub.ProcessMessages(finalSettings, TestContext.Current.CancellationToken); + Assert.True(result >= 0); + } + + #endregion + + #region Subscribe + + [Fact] + public async Task Manager_Subscribe_ReceivesSettingsOnApply() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + var publisher = fixture.Publisher; + + CountingMessageConsumer.Reset(); + + List capturedSettings = []; + using var subscription = manager.Subscribe(group, s => capturedSettings.Add(s)); + + var messages = new[] { new TestMessage { PayloadId = "sub-1", Content = "Msg 1", TenantId = 1 } }; + await publisher.Publish(messages, m => m.TenantId, TestContext.Current.CancellationToken); + + // Apply change + manager.Apply(group, s => s with { MaxBatchSize = 32 }); + + // Subscriber should have received the updated settings + Assert.Single(capturedSettings); + Assert.Equal(32, capturedSettings[0].MaxBatchSize); + Assert.Equal(group, capturedSettings[0].ConsumerGroupId); + } + + [Fact] + public async Task Manager_Subscribe_ReceivesSettingsOnPause() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + + OutboxConsumerSettings? pausedSettings = null; + using var subscription = manager.Subscribe(group, s => pausedSettings = s); + + manager.Pause(group); + + Assert.NotNull(pausedSettings); + Assert.True(pausedSettings.Paused); + } + + [Fact] + public async Task Manager_Subscribe_Unsubscribe_PreventsFutureNotifications() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + + var notifications = 0; + var subscription = manager.Subscribe(group, _ => notifications++); + subscription.Dispose(); + + manager.Apply(group, s => s with { MaxBatchSize = 99 }); + manager.Apply(group, s => s with { MaxBatchSize = 100 }); + + Assert.Equal(0, notifications); + } + + [Fact] + public async Task Manager_Subscribe_MultipleSubscribers_AllReceiveUpdates() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + + var sub1Received = false; + var sub2Received = false; + + using var sub1 = manager.Subscribe(group, _ => sub1Received = true); + using var sub2 = manager.Subscribe(group, _ => sub2Received = true); + + manager.Apply(group, s => s with { MaxBatchSize = 77 }); + + Assert.True(sub1Received); + Assert.True(sub2Received); + } + + #endregion + + #region IsRegistered / Unregister + + [Fact] + public void Manager_IsRegistered_ReturnsTrueAfterSetup() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + + // После настройки DI группа уже зарегистрирована (через DeliveryJob bootstrap) + Assert.True(manager.IsRegistered(group)); + } + + [Fact] + public void Manager_Get_ReturnsNotNullForKnownGroup() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + + var settings = manager.Get(group); + Assert.NotNull(settings); + Assert.Equal(group, settings.ConsumerGroupId); + } + + [Fact] + public void Manager_Get_ReturnsNullForUnknownGroup() + { + // Arrange + var manager = fixture.ConsumerManager; + + var settings = manager.Get("non-existent-group"); + Assert.Null(settings); + } + + [Fact] + public void Manager_Unregister_RemovesFromManager() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + var settings = fixture.SettingsForTestGroup; + + Assert.True(manager.IsRegistered(group)); + + manager.Unregister(group); + + Assert.False(manager.IsRegistered(group)); + Assert.Null(manager.Get(group)); + + // Restore for other tests in the sequential collection + manager.Register(group, settings); + } + + [Fact] + public void Manager_Unregister_GetAllExcludesRemoved() + { + // Arrange + var manager = fixture.ConsumerManager; + var allBefore = manager.GetAllConsumerGroupIds(); + var removedGroup = fixture.CountingGroupId; + var settings = fixture.SettingsForTestGroup; + Assert.NotEmpty(allBefore); + + manager.Unregister(removedGroup); + + var allAfter = manager.GetAllConsumerGroupIds(); + Assert.DoesNotContain(removedGroup, allAfter); + Assert.Equal(allBefore.Count - 1, allAfter.Count); + + // Restore for other tests in the sequential collection + manager.Register(removedGroup, settings); + } + + [Fact] + public async Task Manager_Unregister_ProcessMessages_SkipsUnregistered() + { + // Arrange + var manager = fixture.ConsumerManager; + var publisher = fixture.Publisher; + + CountingMessageConsumer.Reset(); + + // Unregister the group + manager.Unregister(fixture.CountingGroupId); + Assert.False(manager.IsRegistered(fixture.CountingGroupId)); + + // Publish messages + var messages = new[] { new TestMessage { PayloadId = "unreg-1", Content = "Msg 1", TenantId = 1 } }; + await publisher.Publish(messages, m => m.TenantId, TestContext.Current.CancellationToken); + + // ProcessMessages should handle gracefully — unregistered group means no settings, + // so ProcessMessages with explicit settings should still work + var settings = fixture.SettingsForTestGroup; + var result = await fixture.Sub.ProcessMessages(settings, TestContext.Current.CancellationToken); + + // Even though manager doesn't know the group, we passed settings directly — + // ProcessMessages should still process + Assert.True(result >= 0); + + // Restore for other tests in the sequential collection + manager.Register(fixture.CountingGroupId, fixture.SettingsForTestGroup); + } + + #endregion + + #region GetAllConsumerGroupIds + + [Fact] + public void Manager_GetAllConsumerGroupIds_ReturnsAllGroups() + { + // Arrange + var manager = fixture.ConsumerManager; + var allGroups = manager.GetAllConsumerGroupIds(); + + // Должно быть как минимум две группы (fixture.CountingGroupId и fixture.BlockingGroupId), + // настроенные в Fixture. + Assert.True(allGroups.Count >= 2, $"Ожидалось как минимум 2 группы, получено: {allGroups.Count}"); + Assert.Contains(fixture.CountingGroupId, allGroups); + Assert.Contains(Fixture.BlockingGroupId, allGroups); + } + + [Fact] + public void Manager_GetAllConsumerGroupIds_IsSnapshot() + { + // Arrange + var manager = fixture.ConsumerManager; + + var snapshot1 = manager.GetAllConsumerGroupIds(); + var snapshot2 = manager.GetAllConsumerGroupIds(); + + // Оба снимка должны иметь одинаковый размер, но разные ссылки + Assert.Equal(snapshot1.Count, snapshot2.Count); + Assert.NotSame(snapshot1, snapshot2); + } + + #endregion + + #region Thread safety under real Consume + + [Fact] + public async Task Manager_ConcurrentApplyAndGet_NoExceptions() + { + // Arrange + var manager = fixture.ConsumerManager; + var group = fixture.CountingGroupId; + var publisher = fixture.Publisher; + + CountingMessageConsumer.Reset(); + + const int iterations = 50; + var exceptions = new List(); + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + + // Publish messages + var messages = Enumerable.Range(1, 20) + .Select(i => new TestMessage { PayloadId = $"stress-{i}", Content = $"Msg {i}", TenantId = 1 }) + .ToList(); + await publisher.Publish(messages, m => m.TenantId, cts.Token); + + // Concurrent writer + var writerTask = Task.Run(async () => + { + for (int i = 0; i < iterations && !cts.Token.IsCancellationRequested; i++) + { + try + { + manager.Apply(group, s => s with { MaxBatchSize = i + 1 }); + } + catch (Exception ex) + { + Interlocked.Exchange(ref exceptions, exceptions.Append(ex).ToList()); + } + + await Task.Delay(2, cts.Token); + } + }, cts.Token); + + // Concurrent reader + var readerTask = Task.Run(async () => + { + for (int i = 0; i < iterations && !cts.Token.IsCancellationRequested; i++) + { + try + { + var settings = manager.Get(group); + if (settings is not null) + { + _ = settings.MaxBatchSize; + _ = settings.Paused; + _ = settings.Version; + } + } + catch (Exception ex) + { + Interlocked.Exchange(ref exceptions, exceptions.Append(ex).ToList()); + } + + await Task.Delay(2, cts.Token); + } + }, cts.Token); + + // Concurrent pause/resume + var pauseTask = Task.Run(async () => + { + for (int i = 0; i < 20 && !cts.Token.IsCancellationRequested; i++) + { + try + { + manager.Pause(group); + await Task.Delay(1, cts.Token); + manager.Resume(group); + } + catch (Exception ex) + { + Interlocked.Exchange(ref exceptions, exceptions.Append(ex).ToList()); + } + } + }, cts.Token); + + await Task.WhenAll(writerTask, readerTask, pauseTask); + + Assert.Empty(exceptions); + } + + #endregion +} + diff --git a/src/Tests/Sa.Outbox.Tests/OutboxConsumerManagerTests.cs b/src/Tests/Sa.Outbox.Tests/OutboxConsumerManagerTests.cs new file mode 100644 index 00000000..7b8f8a56 --- /dev/null +++ b/src/Tests/Sa.Outbox.Tests/OutboxConsumerManagerTests.cs @@ -0,0 +1,610 @@ +using Sa.Outbox.Delivery; + +namespace Sa.Outbox.Tests; + +public class OutboxConsumerManagerTests +{ + private static OutboxConsumerSettings CreateSettings( + string consumerGroupId = "test-group", + bool paused = false) + => new( + ConsumerGroupId: consumerGroupId, + AsSingleton: false, + Interval: TimeSpan.FromSeconds(5), + InitialDelay: TimeSpan.Zero, + ConcurrencyLimit: 1, + MaxConcurrency: 1, + RetryCountOnError: 0, + MaxBatchSize: 16, + MaxProcessingIterations: -1, + IterationDelay: TimeSpan.Zero, + LockDuration: TimeSpan.FromSeconds(10), + LockRenewal: TimeSpan.FromSeconds(3), + LookbackInterval: TimeSpan.FromDays(7), + MaxDeliveryAttempts: 3, + BatchingWindow: TimeSpan.Zero, + PerTenantTimeout: TimeSpan.Zero, + PerTenantMaxDegreeOfParallelism: 1, + Paused: paused, + Version: 0); + + private static IOutboxConsumerManager CreateManager() + => new OutboxConsumerManager(); + + #region Pause / Resume + + [Fact] + public void Pause_SetsPausedToTrue() + { + var manager = CreateManager(); + var group = "pause-test"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + Assert.False(manager.IsPaused(group)); + + manager.Pause(group); + Assert.True(manager.IsPaused(group)); + } + + [Fact] + public void Resume_SetsPausedToFalse() + { + var manager = CreateManager(); + var group = "resume-test"; + var settings = CreateSettings(group, paused: true); + + manager.Register(group, settings); + Assert.True(manager.IsPaused(group)); + + manager.Resume(group); + Assert.False(manager.IsPaused(group)); + } + + [Fact] + public void Pause_ThenResume_RestartsWithOriginalSettings() + { + var manager = CreateManager(); + var group = "pause-resume-cycle"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + manager.Pause(group); + Assert.True(manager.IsPaused(group)); + + manager.Resume(group); + Assert.False(manager.IsPaused(group)); + + // Settings preserved after pause/resume cycle + var retrieved = manager.Get(group); + Assert.NotNull(retrieved); + Assert.Equal(settings.Interval, retrieved.Interval); + Assert.Equal(settings.MaxBatchSize, retrieved.MaxBatchSize); + } + + [Fact] + public void Pause_OnUnregisteredGroup_ThrowsInvalidOperationException() + { + var manager = CreateManager(); + + var exception = Assert.Throws(() => manager.Pause("non-existent")); + Assert.Contains("not registered", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Resume_OnUnregisteredGroup_ThrowsInvalidOperationException() + { + var manager = CreateManager(); + + var exception = Assert.Throws(() => manager.Resume("non-existent")); + Assert.Contains("not registered", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Pause_OnNullGroup_ThrowsArgumentException() + { + var manager = CreateManager(); + + Assert.Throws(() => manager.Pause(null!)); + Assert.Throws(() => manager.Pause("")); + Assert.Throws(() => manager.Pause(" ")); + } + + [Fact] + public void Resume_OnNullGroup_ThrowsArgumentException() + { + var manager = CreateManager(); + + Assert.Throws(() => manager.Resume(null!)); + Assert.Throws(() => manager.Resume("")); + Assert.Throws(() => manager.Resume(" ")); + } + + [Fact] + public void Pause_PreservesAllSettingsExceptPaused() + { + var manager = CreateManager(); + var group = "pause-preserve"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + manager.Pause(group); + + var updated = manager.Get(group); + Assert.NotNull(updated); + Assert.True(updated.Paused); + Assert.Equal(settings.Interval, updated.Interval); + Assert.Equal(settings.LockDuration, updated.LockDuration); + Assert.Equal(settings.MaxBatchSize, updated.MaxBatchSize); + Assert.Equal(settings.MaxDeliveryAttempts, updated.MaxDeliveryAttempts); + // Pause uses Apply internally — Version stays unchanged unless caller increments it + Assert.Equal(settings.Version, updated.Version); + } + + #endregion + + #region Subscribe + + [Fact] + public void Subscribe_CallbackFiredOnApply() + { + var manager = CreateManager(); + var group = "subscribe-test"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + + OutboxConsumerSettings? captured = null; + using var subscription = manager.Subscribe(group, s => captured = s); + + manager.Apply(group, s => s with { MaxBatchSize = 64 }); + + Assert.NotNull(captured); + Assert.Equal(64, captured.MaxBatchSize); + Assert.Equal(group, captured.ConsumerGroupId); + } + + [Fact] + public void Subscribe_CallbackFiredOnPause() + { + var manager = CreateManager(); + var group = "subscribe-pause"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + + OutboxConsumerSettings? captured = null; + using var subscription = manager.Subscribe(group, s => captured = s); + + manager.Pause(group); + + Assert.NotNull(captured); + Assert.True(captured.Paused); + } + + [Fact] + public void Subscribe_CallbackFiredOnResume() + { + var manager = CreateManager(); + var group = "subscribe-resume"; + var settings = CreateSettings(group, paused: true); + + manager.Register(group, settings); + + OutboxConsumerSettings? captured = null; + using var subscription = manager.Subscribe(group, s => captured = s); + + manager.Resume(group); + + Assert.NotNull(captured); + Assert.False(captured.Paused); + } + + [Fact] + public void Subscribe_MultipleCallbacks_AllFired() + { + var manager = CreateManager(); + var group = "subscribe-multiple"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + + var callback1Invoked = false; + var callback2Invoked = false; + + using var sub1 = manager.Subscribe(group, _ => callback1Invoked = true); + using var sub2 = manager.Subscribe(group, _ => callback2Invoked = true); + + manager.Apply(group, s => s with { MaxBatchSize = 1 }); + + Assert.True(callback1Invoked); + Assert.True(callback2Invoked); + } + + [Fact] + public void Unsubscribe_DisposedCallbackNotFired() + { + var manager = CreateManager(); + var group = "subscribe-unsubscribe"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + + var callbackInvoked = false; + var subscription = manager.Subscribe(group, _ => callbackInvoked = true); + subscription.Dispose(); + + manager.Apply(group, s => s with { MaxBatchSize = 99 }); + + Assert.False(callbackInvoked); + } + + [Fact] + public void Subscribe_ReceivesUpdatedVersion() + { + var manager = CreateManager(); + var group = "subscribe-version"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + + int versionReceived = -1; + using var subscription = manager.Subscribe(group, s => versionReceived = s.Version); + + // Caller increments Version in the transform + manager.Apply(group, s => s with { MaxBatchSize = 32, Version = s.Version + 1 }); + Assert.Equal(settings.Version + 1, versionReceived); + } + + [Fact] + public void Subscribe_SubscriberErrorDoesNotBreakPipeline() + { + var manager = CreateManager(); + var group = "subscribe-error-tolerance"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + + // Subscriber that throws + using var badSub = manager.Subscribe(group, _ => throw new InvalidOperationException("boom")); + + // Another subscriber that should still fire + var goodCallbackInvoked = false; + using var goodSub = manager.Subscribe(group, _ => goodCallbackInvoked = true); + + // Should not throw + manager.Apply(group, s => s with { MaxBatchSize = 10 }); + + Assert.True(goodCallbackInvoked); + } + + [Fact] + public void Subscribe_NonExistentGroup_CreatesListenerEntry() + { + var manager = CreateManager(); + var group = "subscribe-no-register"; + + // Subscribe on unregistered group should not throw (listener list created lazily) + var callbackInvoked = false; + using var subscription = manager.Subscribe(group, _ => callbackInvoked = true); + + // Now register — subscriber should receive + manager.Register(group, CreateSettings(group)); + Assert.True(callbackInvoked); + } + + [Fact] + public void Subscribe_NullGroup_ThrowsArgumentException() + { + var manager = CreateManager(); + + Assert.Throws(() => manager.Subscribe(null!, _ => { })); + Assert.Throws(() => manager.Subscribe("", _ => { })); + Assert.Throws(() => manager.Subscribe(" ", _ => { })); + } + + [Fact] + public void Subscribe_NullCallback_ThrowsArgumentNullException() + { + var manager = CreateManager(); + var group = "subscribe-null-callback"; + + Assert.Throws(() => manager.Subscribe(group, null!)); + } + + #endregion + + #region Apply + + [Fact] + public void Apply_TransformsSettingsAtomically() + { + var manager = CreateManager(); + var group = "apply-atomic"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + + manager.Apply(group, s => s with { MaxBatchSize = 128, MaxDeliveryAttempts = 5 }); + + var updated = manager.Get(group); + Assert.NotNull(updated); + Assert.Equal(128, updated.MaxBatchSize); + Assert.Equal(5, updated.MaxDeliveryAttempts); + } + + [Fact] + public void Apply_OnUnregisteredGroup_ThrowsInvalidOperationException() + { + var manager = CreateManager(); + + var exception = Assert.Throws(() => + manager.Apply("non-existent", s => s)); + Assert.Contains("not registered", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Apply_VersionIncrements() + { + var manager = CreateManager(); + var group = "apply-version"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + var initialVersion = settings.Version; + + manager.Apply(group, s => s with { MaxBatchSize = 1, Version = s.Version + 1 }); + var updated = manager.Get(group); + + // Version is managed by caller via transform — Apply itself does not auto-increment + Assert.Equal(initialVersion + 1, updated!.Version); + } + + [Fact] + public void Apply_ConsecutiveUpdates_AccumulateChanges() + { + var manager = CreateManager(); + var group = "apply-chain"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + + manager.Apply(group, s => s with { MaxBatchSize = 8 }); + manager.Apply(group, s => s with { MaxDeliveryAttempts = 10 }); + manager.Apply(group, s => s with { Paused = true }); + + var final = manager.Get(group); + Assert.NotNull(final); + Assert.Equal(8, final.MaxBatchSize); + Assert.Equal(10, final.MaxDeliveryAttempts); + Assert.True(final.Paused); + } + + #endregion + + #region Get / IsRegistered + + [Fact] + public void Get_ReturnsNullForUnknownGroup() + { + var manager = CreateManager(); + Assert.Null(manager.Get("unknown-group")); + } + + [Fact] + public void Get_ReturnsCurrentSnapshot() + { + var manager = CreateManager(); + var group = "get-snapshot"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + + var snapshot = manager.Get(group); + Assert.NotNull(snapshot); + Assert.Same(settings, snapshot); + } + + [Fact] + public void Get_AfterApply_ReturnsUpdatedSnapshot() + { + var manager = CreateManager(); + var group = "get-after-apply"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + manager.Apply(group, s => s with { MaxBatchSize = 256 }); + + var snapshot = manager.Get(group); + Assert.NotNull(snapshot); + Assert.Equal(256, snapshot.MaxBatchSize); + } + + [Fact] + public void IsRegistered_TrueAfterRegister() + { + var manager = CreateManager(); + var group = "is-registered"; + + Assert.False(manager.IsRegistered(group)); + + manager.Register(group, CreateSettings(group)); + Assert.True(manager.IsRegistered(group)); + } + + [Fact] + public void IsRegistered_FalseAfterUnregister() + { + var manager = CreateManager(); + var group = "is-unregistered"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + Assert.True(manager.IsRegistered(group)); + + manager.Unregister(group); + Assert.False(manager.IsRegistered(group)); + } + + [Fact] + public void IsRegistered_NullGroup_ReturnsFalse() + { + var manager = CreateManager(); + Assert.False(manager.IsRegistered(null!)); + } + + #endregion + + #region GetAllConsumerGroupIds + + [Fact] + public void GetAllConsumerGroupIds_ReturnsEmptyWhenNoneRegistered() + { + var manager = CreateManager(); + var ids = manager.GetAllConsumerGroupIds(); + Assert.NotNull(ids); + Assert.Empty(ids); + } + + [Fact] + public void GetAllConsumerGroupIds_ReturnsAllRegisteredGroups() + { + var manager = CreateManager(); + var group1 = "group-alpha"; + var group2 = "group-beta"; + var group3 = "group-gamma"; + + manager.Register(group1, CreateSettings(group1)); + manager.Register(group2, CreateSettings(group2)); + manager.Register(group3, CreateSettings(group3)); + + var ids = manager.GetAllConsumerGroupIds(); + Assert.Equal(3, ids.Count); + Assert.Contains(group1, ids); + Assert.Contains(group2, ids); + Assert.Contains(group3, ids); + } + + [Fact] + public void GetAllConsumerGroupIds_ExcludesUnregistered() + { + var manager = CreateManager(); + var group1 = "keep-me"; + var group2 = "remove-me"; + + manager.Register(group1, CreateSettings(group1)); + manager.Register(group2, CreateSettings(group2)); + manager.Unregister(group2); + + var ids = manager.GetAllConsumerGroupIds(); + Assert.Single(ids); + Assert.DoesNotContain(group2, ids); + } + + #endregion + + #region Thread safety + + [Fact] + public async Task Concurrent_ApplyAndRead_NoDataRace() + { + var manager = CreateManager(); + var group = "concurrent-test"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + + var exceptions = new List(); + var iterations = 100; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + var writer = Task.Run(async () => + { + for (int i = 0; i < iterations && !cts.Token.IsCancellationRequested; i++) + { + try + { + var batchSize = i + 1; + manager.Apply(group, s => s with { MaxBatchSize = batchSize }); + } + catch (Exception ex) + { + exceptions.Add(ex); + } + + await Task.Delay(1, cts.Token); + } + }, cts.Token); + + var reader = Task.Run(async () => + { + for (int i = 0; i < iterations && !cts.Token.IsCancellationRequested; i++) + { + try + { + var snapshot = manager.Get(group); + if (snapshot is not null && (snapshot.MaxBatchSize < 1 || snapshot.MaxBatchSize > iterations)) + { + // Snapshot received is consistent — even if we miss some updates + } + } + catch (Exception ex) + { + exceptions.Add(ex); + } + + await Task.Delay(1, cts.Token); + } + }, cts.Token); + + await Task.WhenAll(writer, reader); + Assert.Empty(exceptions); + } + + [Fact] + public async Task Concurrent_PauseResumeAndSubscribe_NoCrash() + { + var manager = CreateManager(); + var group = "stress-test"; + var settings = CreateSettings(group); + + manager.Register(group, settings); + + var fired = 0; + using var sub = manager.Subscribe(group, _ => Interlocked.Increment(ref fired)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + var pauseTask = Task.Run(async () => + { + for (int i = 0; i < 50 && !cts.Token.IsCancellationRequested; i++) + { + manager.Pause(group); + await Task.Delay(1, cts.Token); + manager.Resume(group); + } + }, cts.Token); + + var readTask = Task.Run(async () => + { + for (int i = 0; i < 50 && !cts.Token.IsCancellationRequested; i++) + { + manager.Get(group); + await Task.Delay(1, cts.Token); + } + }, cts.Token); + + var applyTask = Task.Run(async () => + { + for (int i = 0; i < 50 && !cts.Token.IsCancellationRequested; i++) + { + manager.Apply(group, s => s with { MaxBatchSize = i + 1 }); + await Task.Delay(1, cts.Token); + } + }, cts.Token); + + await Task.WhenAll(pauseTask, readTask, applyTask); + Assert.True(fired > 0); + } + + #endregion +} From cf4cb86b97695c402fce01b253a967ab667c3606 Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 30 Jun 2026 23:49:30 +0300 Subject: [PATCH 23/33] refactoring --- src/Sa.Data.PostgreSql/IPgDataSource.cs | 17 ++++--- src/Sa.Data.PostgreSql/PgDataSource.cs | 34 ++++++------- src/Sa.Data.S3/S3Upload.cs | 2 +- src/Sa.Media/AsyncWavReader.cs | 16 +++--- src/Sa.Media/AsyncWavWriter.cs | 10 ++-- src/Sa.Media/BinaryPipeReader.cs | 6 +-- src/Sa.Media/PipeReaderExtensions.cs | 8 +-- src/Sa.Media/WavHeaderReader.cs | 36 ++++++------- src/Sa.Outbox/Delivery/DeliveryBuilder.cs | 4 +- src/Sa.Outbox/Delivery/DeliveryCourier.cs | 2 +- src/Sa.Outbox/Delivery/DeliveryProcessor.cs | 22 ++++---- src/Sa.Outbox/Delivery/Job/DeliveryJob.cs | 2 +- .../Delivery/OutboxConsumerManager.cs | 2 + src/Sa.Outbox/Delivery/OutboxDefaults.cs | 4 ++ .../Cache/PartCache.cs | 30 +++++++---- .../Cleaning/PartCleanupService.cs | 2 +- .../Configuration/Builder/TableBuilder.cs | 4 +- .../Migration/MigrationJob.cs | 2 +- .../Migration/PartMigrationService.cs | 51 +++++++++---------- .../Partitional/PartRepository.cs | 26 +++++----- .../SqlBuilder/SqlBuilder.cs | 2 +- src/Sa.Schedule/Engine/JobController.cs | 3 ++ src/Sa.Utils.WorkQueue/SaWorkQueue.cs | 15 +++++- src/Sa/Classes/IProcessExecutor.cs | 8 ++- .../DeliveryConsumerGroupManagerTests.cs | 8 +-- 25 files changed, 172 insertions(+), 144 deletions(-) diff --git a/src/Sa.Data.PostgreSql/IPgDataSource.cs b/src/Sa.Data.PostgreSql/IPgDataSource.cs index b68b7ffa..63deb9ce 100644 --- a/src/Sa.Data.PostgreSql/IPgDataSource.cs +++ b/src/Sa.Data.PostgreSql/IPgDataSource.cs @@ -37,7 +37,10 @@ Task ExecuteNonQuery(string sql, CancellationToken cancellationToken = defa async Task ExecuteScalar( string sql, Action? initCommand, CancellationToken cancellationToken = default) - => ((T)(await ExecuteScalar(sql, initCommand, cancellationToken))!); + { + var result = await ExecuteScalar(sql, initCommand, cancellationToken).ConfigureAwait(false); + return (T)(result!); + } /// @@ -46,7 +49,7 @@ async Task ExecuteScalar( async Task ExecuteScalarTyped( string sql, Action? initCommand = null, CancellationToken cancellationToken = default) { - var result = await ExecuteScalar(sql, initCommand, cancellationToken); + var result = await ExecuteScalar(sql, initCommand, cancellationToken).ConfigureAwait(false); if (result is null || result is DBNull) return default!; @@ -83,7 +86,7 @@ async Task ExecuteScalarTyped( IReadOnlyCollection parameters, CancellationToken cancellationToken = default) { - var result = await ExecuteScalar(sql, cmd => FillParams(cmd, parameters), cancellationToken); + var result = await ExecuteScalar(sql, cmd => FillParams(cmd, parameters), cancellationToken).ConfigureAwait(false); if (result is null || result is DBNull) return default!; if (result is T typed) @@ -108,7 +111,7 @@ Task ExecuteReader( async Task ExecuteReader( string sql, Action read, CancellationToken cancellationToken = default) - => await ExecuteReader(sql, read, [], cancellationToken); + => await ExecuteReader(sql, read, [], cancellationToken).ConfigureAwait(false); // ExecuteReaderList @@ -118,7 +121,7 @@ async Task> ExecuteReaderList( string sql, Func read, CancellationToken cancellationToken = default) { List list = []; - await ExecuteReader(sql, (reader, _) => list.Add(read(reader)), cancellationToken); + await ExecuteReader(sql, (reader, _) => list.Add(read(reader)), cancellationToken).ConfigureAwait(false); return list; } @@ -129,7 +132,7 @@ async Task> ExecuteReaderList( CancellationToken cancellationToken = default) { List list = []; - await ExecuteReader(sql, (reader, _) => list.Add(read(reader)), parameters, cancellationToken); + await ExecuteReader(sql, (reader, _) => list.Add(read(reader)), parameters, cancellationToken).ConfigureAwait(false); return list; } @@ -172,7 +175,7 @@ await ExecuteReader(sql, (reader, _) => }; } , parameters - , cancellationToken); + , cancellationToken).ConfigureAwait(false); return value; } diff --git a/src/Sa.Data.PostgreSql/PgDataSource.cs b/src/Sa.Data.PostgreSql/PgDataSource.cs index 2667aefa..f07a9592 100644 --- a/src/Sa.Data.PostgreSql/PgDataSource.cs +++ b/src/Sa.Data.PostgreSql/PgDataSource.cs @@ -29,7 +29,7 @@ public async ValueTask DisposeAsync() { if (_dataSource.IsValueCreated) { - await _dataSource.Value.DisposeAsync(); + await _dataSource.Value.DisposeAsync().ConfigureAwait(false); } } @@ -38,9 +38,9 @@ public async ValueTask BeginBinaryImport( Func> write, CancellationToken cancellationToken = default) { - await using NpgsqlConnection db = await OpenDbConnection(cancellationToken); - await using NpgsqlBinaryImporter writer = await db.BeginBinaryImportAsync(sql, cancellationToken); - ulong result = await write(writer, cancellationToken); + await using NpgsqlConnection db = await OpenDbConnection(cancellationToken).ConfigureAwait(false); + await using NpgsqlBinaryImporter writer = await db.BeginBinaryImportAsync(sql, cancellationToken).ConfigureAwait(false); + ulong result = await write(writer, cancellationToken).ConfigureAwait(false); return result; } @@ -49,17 +49,17 @@ public async Task ExecuteTransactionAsync( IsolationLevel isolationLevel = IsolationLevel.Unspecified, CancellationToken cancellationToken = default) { - await using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); - await connection.OpenAsync(cancellationToken); - await using NpgsqlTransaction transaction = await connection.BeginTransactionAsync(isolationLevel, cancellationToken); + await using NpgsqlConnection connection = await OpenDbConnection(cancellationToken).ConfigureAwait(false); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + await using NpgsqlTransaction transaction = await connection.BeginTransactionAsync(isolationLevel, cancellationToken).ConfigureAwait(false); try { - await action(transaction, cancellationToken); - await transaction.CommitAsync(cancellationToken); + await action(transaction, cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); } catch { - await transaction.RollbackAsync(cancellationToken); + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); throw; } } @@ -69,10 +69,10 @@ public async Task ExecuteNonQuery( Action? initCommand, CancellationToken cancellationToken = default) { - await using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); + await using NpgsqlConnection connection = await OpenDbConnection(cancellationToken).ConfigureAwait(false); await using NpgsqlCommand cmd = new(sql, connection); initCommand?.Invoke(cmd); - return await cmd.ExecuteNonQueryAsync(cancellationToken); + return await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } public async Task ExecuteScalar( @@ -80,10 +80,10 @@ public async Task ExecuteNonQuery( Action? initCommand, CancellationToken cancellationToken = default) { - await using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); + await using NpgsqlConnection connection = await OpenDbConnection(cancellationToken).ConfigureAwait(false); await using NpgsqlCommand cmd = new(sql, connection); initCommand?.Invoke(cmd); - return await cmd.ExecuteScalarAsync(cancellationToken); + return await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); } public async Task ExecuteReader( @@ -94,11 +94,11 @@ public async Task ExecuteReader( { int rowCount = 0; - using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); + using NpgsqlConnection connection = await OpenDbConnection(cancellationToken).ConfigureAwait(false); await using NpgsqlCommand cmd = new(sql, connection); initCommand?.Invoke(cmd); - await using NpgsqlDataReader reader = await cmd.ExecuteReaderAsync(cancellationToken); - while (await reader.ReadAsync(cancellationToken) && !cancellationToken.IsCancellationRequested) + await using NpgsqlDataReader reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false) && !cancellationToken.IsCancellationRequested) { read(reader, rowCount); rowCount++; diff --git a/src/Sa.Data.S3/S3Upload.cs b/src/Sa.Data.S3/S3Upload.cs index 7f1e226c..fb5577ba 100644 --- a/src/Sa.Data.S3/S3Upload.cs +++ b/src/Sa.Data.S3/S3Upload.cs @@ -103,7 +103,7 @@ public async Task AddPart(byte[] data, int length, CancellationToken ct) } var partId = await _client.MultipartPutPart( - _encodedFileName, UploadId, _partCount + 1, data, length, ct); + _encodedFileName, UploadId, _partCount + 1, data, length, ct).ConfigureAwait(false); if (string.IsNullOrEmpty(partId)) { diff --git a/src/Sa.Media/AsyncWavReader.cs b/src/Sa.Media/AsyncWavReader.cs index 6ce11406..04d713f4 100644 --- a/src/Sa.Media/AsyncWavReader.cs +++ b/src/Sa.Media/AsyncWavReader.cs @@ -92,7 +92,7 @@ public async IAsyncEnumerable ReadSamplesPerChannelAsync( bool allowBufferReuse = true, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var header = await GetHeaderAsync(cancellationToken); + var header = await GetHeaderAsync(cancellationToken).ConfigureAwait(false); EnsureDataSize(header); var (cutFrom, cutTo) = header.CalculateCutOffsets(cutRange ?? TimeRange.Default); @@ -101,7 +101,7 @@ public async IAsyncEnumerable ReadSamplesPerChannelAsync( if (offsetToSkip > 0) { - await _reader.SkipAsync(offsetToSkip, cancellationToken); + await _reader.SkipAsync(offsetToSkip, cancellationToken).ConfigureAwait(false); } int channels = header.NumChannels; @@ -117,7 +117,7 @@ public async IAsyncEnumerable ReadSamplesPerChannelAsync( { cancellationToken.ThrowIfCancellationRequested(); - ReadResult result = await _reader.ReadAsync(cancellationToken); + ReadResult result = await _reader.ReadAsync(cancellationToken).ConfigureAwait(false); ReadOnlySequence sequence = result.Buffer; SequencePosition consumed = sequence.Start; @@ -182,7 +182,7 @@ public async IAsyncEnumerable ReadDoubleSamplesAsync( bool allowBufferReuse = true, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var convert = await GetNormalizedConverterAsync(cancellationToken); + var convert = await GetNormalizedConverterAsync(cancellationToken).ConfigureAwait(false); await foreach (var (channelId, rawSample, offset, isEof) in ReadSamplesPerChannelAsync(cutRange, allowBufferReuse, cancellationToken: cancellationToken) @@ -244,7 +244,7 @@ public async IAsyncEnumerable ReadStreamableChunksAsync( int alignedSize = Math.Max(bytesPerSample, (samplesPerBatch / bytesPerSample) * bytesPerSample); // Инициализируем буферы по количеству каналов - var header = await GetHeaderAsync(cancellationToken); + var header = await GetHeaderAsync(cancellationToken).ConfigureAwait(false); int channelCount = header.NumChannels; var channelBuffers = new IMemoryOwner[channelCount]; @@ -315,7 +315,7 @@ public async IAsyncEnumerable ReadStreamableChunksAsync( private async Task, double>> GetNormalizedConverterAsync(CancellationToken cancellationToken) { - var header = await GetHeaderAsync(cancellationToken); + var header = await GetHeaderAsync(cancellationToken).ConfigureAwait(false); return header.GetNormalizedConverter(); } @@ -347,12 +347,12 @@ public async ValueTask DisposeAsync() { if (_ownsReader) { - await _reader.CompleteAsync(); + await _reader.CompleteAsync().ConfigureAwait(false); } if (_stream != null) { - await _stream.DisposeAsync(); + await _stream.DisposeAsync().ConfigureAwait(false); } _disposed = true; } diff --git a/src/Sa.Media/AsyncWavWriter.cs b/src/Sa.Media/AsyncWavWriter.cs index 4046f7e9..099b298d 100644 --- a/src/Sa.Media/AsyncWavWriter.cs +++ b/src/Sa.Media/AsyncWavWriter.cs @@ -203,25 +203,25 @@ private async Task FlushBufferAsync(CancellationToken cancellationToken) { if (_currentBufferSize == 0) return; - await _stream.WriteAsync(_currentBuffer[.._currentBufferSize], cancellationToken); + await _stream.WriteAsync(_currentBuffer[.._currentBufferSize], cancellationToken).ConfigureAwait(false); _currentBufferSize = 0; } public async Task CloseAsync(CancellationToken cancellationToken = default) { - await FlushBufferAsync(cancellationToken); + await FlushBufferAsync(cancellationToken).ConfigureAwait(false); CorrectHeader(); if (!_leaveOpen) - await _stream.FlushAsync(cancellationToken); + await _stream.FlushAsync(cancellationToken).ConfigureAwait(false); } public async ValueTask DisposeAsync() { - await CloseAsync(); + await CloseAsync().ConfigureAwait(false); _bufferOwner.Dispose(); if (!_leaveOpen) - await _stream.DisposeAsync(); + await _stream.DisposeAsync().ConfigureAwait(false); } public void Dispose() diff --git a/src/Sa.Media/BinaryPipeReader.cs b/src/Sa.Media/BinaryPipeReader.cs index 68676663..7b242672 100644 --- a/src/Sa.Media/BinaryPipeReader.cs +++ b/src/Sa.Media/BinaryPipeReader.cs @@ -10,7 +10,7 @@ internal sealed class BinaryPipeReader(PipeReader reader) public async ValueTask ReadUInt32Async(CancellationToken cancellationToken = default) { - var idBuffer = await reader.ReadAtLeastAsync(4, cancellationToken); + var idBuffer = await reader.ReadAtLeastAsync(4, cancellationToken).ConfigureAwait(false); uint result = ReadUInt32Little(idBuffer.Buffer); reader.AdvanceTo(idBuffer.Buffer.GetPosition(4)); Position += 4; @@ -19,7 +19,7 @@ public async ValueTask ReadUInt32Async(CancellationToken cancellationToken public async ValueTask ReadUInt16Async(CancellationToken cancellationToken = default) { - var idBuffer = await reader.ReadAtLeastAsync(2, cancellationToken); + var idBuffer = await reader.ReadAtLeastAsync(2, cancellationToken).ConfigureAwait(false); ushort result = ReadUInt16Little(idBuffer.Buffer); reader.AdvanceTo(idBuffer.Buffer.GetPosition(2)); Position += 2; @@ -29,7 +29,7 @@ public async ValueTask ReadUInt16Async(CancellationToken cancellationTok public async Task SkipBytesAsync(long count, CancellationToken cancellationToken = default) { Position += count; - await PipeReaderExtensions.SkipAsync(reader, count, cancellationToken); + await PipeReaderExtensions.SkipAsync(reader, count, cancellationToken).ConfigureAwait(false); } private static uint ReadUInt32Little(ReadOnlySequence seq) diff --git a/src/Sa.Media/PipeReaderExtensions.cs b/src/Sa.Media/PipeReaderExtensions.cs index 289819ea..1da0e1de 100644 --- a/src/Sa.Media/PipeReaderExtensions.cs +++ b/src/Sa.Media/PipeReaderExtensions.cs @@ -13,7 +13,7 @@ public static async ValueTask SkipAsync(this PipeReader reader, long count long remaining = count; while (remaining > 0) { - ReadResult result = await reader.ReadAsync(ct); + ReadResult result = await reader.ReadAsync(ct).ConfigureAwait(false); if (result.Buffer.IsEmpty && result.IsCompleted) break; // Недостаточно данных @@ -21,10 +21,6 @@ public static async ValueTask SkipAsync(this PipeReader reader, long count var consumed = result.Buffer.GetPosition(toConsume); reader.AdvanceTo(consumed, consumed); remaining -= toConsume; - - //// Если буфер маленький, но нам нужно больше — продолжаем читать - //if (result.Buffer.Length <= toConsume && !result.IsCompleted) - // continue; } return count - remaining; } @@ -38,7 +34,7 @@ public static async ValueTask SkipFullSegmentsAsync(this PipeReader reader, long long remaining = count; while (remaining > 0) { - ReadResult result = await reader.ReadAsync(ct); + ReadResult result = await reader.ReadAsync(ct).ConfigureAwait(false); if (result.Buffer.IsEmpty && result.IsCompleted) return; diff --git a/src/Sa.Media/WavHeaderReader.cs b/src/Sa.Media/WavHeaderReader.cs index 53aee33b..9072bab9 100644 --- a/src/Sa.Media/WavHeaderReader.cs +++ b/src/Sa.Media/WavHeaderReader.cs @@ -17,40 +17,40 @@ public static async Task ReadHeaderAsync( CancellationToken cancellationToken = default) { BinaryPipeReader reader = new(pipe); - uint chunkId = await reader.ReadUInt32Async(cancellationToken); - uint chunkSize = await reader.ReadUInt32Async(cancellationToken); - uint format = await reader.ReadUInt32Async(cancellationToken); + uint chunkId = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false); + uint chunkSize = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false); + uint format = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false); if (chunkId != Constants.СhunkRiff || format != Constants.FormatWave) throw new NotSupportedException("ERROR: File is not a WAV file"); - uint subchunk1Id = await reader.ReadUInt32Async(cancellationToken); + uint subchunk1Id = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false); // Skip JUNK chunks while (subchunk1Id == Constants.Subchunk1IdJunk) { - uint junkSize = await reader.ReadUInt32Async(cancellationToken); + uint junkSize = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false); if (junkSize % 2 == 1) junkSize++; // align to even size - await reader.SkipBytesAsync(junkSize, cancellationToken); - subchunk1Id = await reader.ReadUInt32Async(cancellationToken); + await reader.SkipBytesAsync(junkSize, cancellationToken).ConfigureAwait(false); + subchunk1Id = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false); } - uint subchunk1Size = await reader.ReadUInt32Async(cancellationToken); - ushort audioFormatValue = await reader.ReadUInt16Async(cancellationToken); + uint subchunk1Size = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false); + ushort audioFormatValue = await reader.ReadUInt16Async(cancellationToken).ConfigureAwait(false); WaveFormatType audioFormat = (WaveFormatType)audioFormatValue; if (audioFormat is not (WaveFormatType.Pcm or WaveFormatType.IeeeFloat)) throw new NotSupportedException($"Unsupported audio format: {audioFormat}"); - ushort numChannels = await reader.ReadUInt16Async(cancellationToken); - uint sampleRate = await reader.ReadUInt32Async(cancellationToken); - uint byteRate = await reader.ReadUInt32Async(cancellationToken); - ushort blockAlign = await reader.ReadUInt16Async(cancellationToken); - ushort bitsPerSample = await reader.ReadUInt16Async(cancellationToken); + ushort numChannels = await reader.ReadUInt16Async(cancellationToken).ConfigureAwait(false); + uint sampleRate = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false); + uint byteRate = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false); + ushort blockAlign = await reader.ReadUInt16Async(cancellationToken).ConfigureAwait(false); + ushort bitsPerSample = await reader.ReadUInt16Async(cancellationToken).ConfigureAwait(false); // Skip extra fmt data (e.g., for WAVE_FORMAT_EXTENSIBLE) - var (dataOffset, dataSize) = await FindDataChunkAsync(reader, cancellationToken); + var (dataOffset, dataSize) = await FindDataChunkAsync(reader, cancellationToken).ConfigureAwait(false); var header = new WavHeader { @@ -82,8 +82,8 @@ public static async Task ReadHeaderAsync( { cancellationToken.ThrowIfCancellationRequested(); - var chunkId = await reader.ReadUInt32Async(cancellationToken); - var chunkSize = await reader.ReadUInt32Async(cancellationToken); + var chunkId = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false); + var chunkSize = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false); if (chunkId == Constants.DataSubchunkId) // "data" { @@ -95,7 +95,7 @@ public static async Task ReadHeaderAsync( if (paddedSize == 0) throw new InvalidDataException("Invalid WAV file: zero-size chunk"); - await reader.SkipBytesAsync(paddedSize, cancellationToken); + await reader.SkipBytesAsync(paddedSize, cancellationToken).ConfigureAwait(false); } } } diff --git a/src/Sa.Outbox/Delivery/DeliveryBuilder.cs b/src/Sa.Outbox/Delivery/DeliveryBuilder.cs index cdf35ed2..745a3028 100644 --- a/src/Sa.Outbox/Delivery/DeliveryBuilder.cs +++ b/src/Sa.Outbox/Delivery/DeliveryBuilder.cs @@ -80,9 +80,9 @@ string IConsumerGroupNamingStrategy.GetConsumerGroupName() static string SanitizeString(string input) { ArgumentNullException.ThrowIfNullOrWhiteSpace(input); - return SanitazeRegex().Replace(input, "_").ToLower(); + return SanitizeRegex().Replace(input, "_").ToLower(); } [GeneratedRegex(@"[^a-zA-Z0-9_]")] - private static partial Regex SanitazeRegex(); + private static partial Regex SanitizeRegex(); } diff --git a/src/Sa.Outbox/Delivery/DeliveryCourier.cs b/src/Sa.Outbox/Delivery/DeliveryCourier.cs index 7d01bd34..46837688 100644 --- a/src/Sa.Outbox/Delivery/DeliveryCourier.cs +++ b/src/Sa.Outbox/Delivery/DeliveryCourier.cs @@ -26,7 +26,7 @@ public async ValueTask Deliver( try { - await processor.ConsumeInScope(settings, filter, messages, cancellationToken); + await processor.ConsumeInScope(settings, filter, messages, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (!ex.IsCritical()) // Handle non-critical exceptions { diff --git a/src/Sa.Outbox/Delivery/DeliveryProcessor.cs b/src/Sa.Outbox/Delivery/DeliveryProcessor.cs index 7e54cbd6..8d9096b1 100644 --- a/src/Sa.Outbox/Delivery/DeliveryProcessor.cs +++ b/src/Sa.Outbox/Delivery/DeliveryProcessor.cs @@ -22,14 +22,14 @@ public async Task ProcessMessages( if (settings.Paused) { // Consumer group is paused — do not poll. - await Task.Delay(PausedPollDelay, cancellationToken); + await Task.Delay(PausedPollDelay, cancellationToken).ConfigureAwait(false); return 0; } int batchSize = settings.MaxBatchSize; if (batchSize == 0) return 0; - int[] tenantIds = await tenantProvider.GetTenantIds(cancellationToken); + int[] tenantIds = await tenantProvider.GetTenantIds(cancellationToken).ConfigureAwait(false); if (tenantIds.Length == 0) return 0; long totalProcessed = 0; @@ -42,16 +42,16 @@ public async Task ProcessMessages( // the greedy loop, not wait for all pending messages to drain. if (settings.Paused) { - await Task.Delay(PausedPollDelay, cancellationToken); + await Task.Delay(PausedPollDelay, cancellationToken).ConfigureAwait(false); return totalProcessed; } if (iterations > 0 && settings.IterationDelay > TimeSpan.Zero) { - await Task.Delay(settings.IterationDelay, cancellationToken); + await Task.Delay(settings.IterationDelay, cancellationToken).ConfigureAwait(false); } - int sentCount = await ProcessForEachTenant(tenantIds, settings, cancellationToken); + int sentCount = await ProcessForEachTenant(tenantIds, settings, cancellationToken).ConfigureAwait(false); totalProcessed += sentCount; iterations++; @@ -91,8 +91,8 @@ private async Task ProcessForEachTenant( CancellationToken cancellationToken) { return (settings.PerTenantMaxDegreeOfParallelism == 1) - ? await ProcessTenantsSequential(tenantIds, settings, cancellationToken) - : await ProcessTenantsParallel(tenantIds, settings, cancellationToken); + ? await ProcessTenantsSequential(tenantIds, settings, cancellationToken).ConfigureAwait(false) + : await ProcessTenantsParallel(tenantIds, settings, cancellationToken).ConfigureAwait(false); } private async Task ProcessTenantsSequential( @@ -103,7 +103,7 @@ private async Task ProcessTenantsSequential( int count = 0; foreach (int tenantId in tenantIds) { - count += await ProcessInTenant(tenantId, settings, cancellationToken); + count += await ProcessInTenant(tenantId, settings, cancellationToken).ConfigureAwait(false); } return count; @@ -131,9 +131,9 @@ await Parallel.ForEachAsync( parallelOptions, async (tenantId, ct) => { - int processed = await ProcessInTenant(tenantId, settings, ct); + int processed = await ProcessInTenant(tenantId, settings, ct).ConfigureAwait(false); Interlocked.Add(ref totalCount, processed); - }); + }).ConfigureAwait(false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -156,7 +156,7 @@ private async Task ProcessInTenant( try { - return await processor.ProcessInTenant(tenantId, settings, tenantCts.Token); + return await processor.ProcessInTenant(tenantId, settings, tenantCts.Token).ConfigureAwait(false); } catch (OperationCanceledException) { diff --git a/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs b/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs index 7fc486b1..498ba213 100644 --- a/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs +++ b/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs @@ -24,6 +24,6 @@ public async Task Execute(IJobContext context, CancellationToken cancellationTok settingsManager.Register(context.JobName, settings); } - await processor.ProcessMessages(settings, cancellationToken); + await processor.ProcessMessages(settings, cancellationToken).ConfigureAwait(false); } } diff --git a/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs b/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs index 999f021b..c484c782 100644 --- a/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs +++ b/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs @@ -50,6 +50,8 @@ public void Apply(string consumerGroupId, Func true; /// Default maximum concurrency (48). + /// + /// High value designed for cloud deployments with connection pooling and + /// partitioned outbox tables. For local/small deployments override via builder. + /// public static int MaxConcurrency => 48; /// Default retry count on error — no retries. diff --git a/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs b/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs index 521eac3a..0bafe904 100644 --- a/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs +++ b/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs @@ -22,15 +22,23 @@ public async Task InCache( { if (sqlBuilder[tableName] == null) return false; - List list = await GetPartsInCache(tableName, cancellationToken); + List list = await GetPartsInCache(tableName, cancellationToken).ConfigureAwait(false); if (list.Count == 0) return false; return list.Exists(c => partValues.SequenceEqual(c.PartValues) && c.PartBy.GetRange(c.FromDate).InRange(date)); } - private Task> GetPartsInCache(string tableName, CancellationToken cancellationToken) - => _cache.GetOrAdd(tableName, SelectPartsInDb, cancellationToken); + private async Task> GetPartsInCache(string tableName, CancellationToken cancellationToken) + { + // Check if we have a valid (non-failed) cached task. + if (_cache.TryGetValue(tableName, out var cached) && !cached.IsFaulted) + { + return await cached.ConfigureAwait(false); + } + + return await _cache.GetOrAdd(tableName, SelectPartsInDb, cancellationToken).ConfigureAwait(false); + } // search and set the cache duration based result set private async Task> SelectPartsInDb(string tableName, CancellationToken cancellationToken) @@ -40,7 +48,7 @@ private async Task> SelectPartsInDb(string tableName, Canc var tp = timeProvider ?? TimeProvider.System; DateTimeOffset from = (tp.GetUtcNow() - settings.CachedFromDate).StartOfDay(); - List list = await repository.GetPartsFromDate(tableName, from, cancellationToken); + List list = await repository.GetPartsFromDate(tableName, from, cancellationToken).ConfigureAwait(false); return list; } catch (Npgsql.PostgresException ex) when (PgErrorCodes.IsUndefinedTable(ex)) @@ -55,18 +63,22 @@ public async Task EnsureCache( StrOrNum[] partValues, CancellationToken cancellationToken = default) { - bool result = await InCache(tableName, date, partValues, cancellationToken); + bool result = await InCache(tableName, date, partValues, cancellationToken).ConfigureAwait(false); if (result) return true; - await repository.CreatePart(tableName, date, partValues, cancellationToken); + await repository.CreatePart(tableName, date, partValues, cancellationToken).ConfigureAwait(false); - await RemoveCache(tableName, cancellationToken); + await RemoveCache(tableName, cancellationToken).ConfigureAwait(false); - result = await InCache(tableName, date, partValues, cancellationToken); + result = await InCache(tableName, date, partValues, cancellationToken).ConfigureAwait(false); return result; } public Task RemoveCache(string tableName, CancellationToken cancellationToken = default) - => Task.FromResult(_cache.Remove(tableName, out _)); + { + // Remove both the cached task and any failed task to allow re-fetch on next access. + _cache.TryRemove(tableName, out _); + return Task.CompletedTask; + } } diff --git a/src/Sa.Partitional.PostgreSql/Cleaning/PartCleanupService.cs b/src/Sa.Partitional.PostgreSql/Cleaning/PartCleanupService.cs index 817721b2..396c2380 100644 --- a/src/Sa.Partitional.PostgreSql/Cleaning/PartCleanupService.cs +++ b/src/Sa.Partitional.PostgreSql/Cleaning/PartCleanupService.cs @@ -11,7 +11,7 @@ public async Task Clean(DateTimeOffset toDate, CancellationToken cancellati int cnt = 0; foreach (string tableName in sqlBuilder.Tables.Select(c => c.FullName)) { - cnt += await repository.DropPartsToDate(tableName, toDate, cancellationToken); + cnt += await repository.DropPartsToDate(tableName, toDate, cancellationToken).ConfigureAwait(false); } return cnt; } diff --git a/src/Sa.Partitional.PostgreSql/Configuration/Builder/TableBuilder.cs b/src/Sa.Partitional.PostgreSql/Configuration/Builder/TableBuilder.cs index bbe14be3..653c374c 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/Builder/TableBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/Builder/TableBuilder.cs @@ -155,13 +155,13 @@ public async Task GetParts(CancellationToken cancellationToken) if (getPartValues != null) { - StrOrNum[][] partItems = await getPartValues(cancellationToken); + StrOrNum[][] partItems = await getPartValues(cancellationToken).ConfigureAwait(false); result.AddRange(partItems); } if (original != null) { - StrOrNum[][] partItems = await original.GetParts(cancellationToken); + StrOrNum[][] partItems = await original.GetParts(cancellationToken).ConfigureAwait(false); result.AddRange(partItems); } diff --git a/src/Sa.Partitional.PostgreSql/Migration/MigrationJob.cs b/src/Sa.Partitional.PostgreSql/Migration/MigrationJob.cs index 5bb6290c..5ac07d5e 100644 --- a/src/Sa.Partitional.PostgreSql/Migration/MigrationJob.cs +++ b/src/Sa.Partitional.PostgreSql/Migration/MigrationJob.cs @@ -6,6 +6,6 @@ internal sealed class MigrationJob(IMigrationService service) : IJob { public async Task Execute(IJobContext context, CancellationToken cancellationToken) { - await service.Migrate(cancellationToken); + await service.Migrate(cancellationToken).ConfigureAwait(false); } } diff --git a/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs b/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs index 7de096d5..b6300756 100644 --- a/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs +++ b/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs @@ -7,45 +7,44 @@ IPartRepository repository , TimeProvider timeProvider , MigrationScheduleSettings settings) : IMigrationService, IDisposable { - private int s_triggered = 0; + private readonly SemaphoreSlim _migrationLock = new(1, 1); private readonly CancellationTokenSource _cts = new(); public CancellationToken OnMigrated => _cts.Token; - public void Dispose() => _cts.Dispose(); + public void Dispose() + { + _cts.Dispose(); + _migrationLock.Dispose(); + } public Task Migrate(DateTimeOffset[] dates, CancellationToken cancellationToken = default) => repository.Migrate(dates, cancellationToken); public async Task Migrate(CancellationToken cancellationToken = default) { - if (Interlocked.CompareExchange(ref s_triggered, 1, 0) == 0) + // Acquire exclusive lock with timeout to prevent indefinite spinning. + if (!await _migrationLock.WaitAsync(settings.WaitMigrationTimeout, cancellationToken).ConfigureAwait(false)) + return -1; + + try { - try - { - DateTimeOffset now = timeProvider.GetUtcNow().StartOfDay(); - DateTimeOffset[] dates = [.. Enumerable - .Range(0, settings.ForwardDays) - .Select(i => now.AddDays(i))]; - - int result = await repository.Migrate(dates, cancellationToken); - await _cts.CancelAsync(); - return result; - } - finally - { - Interlocked.CompareExchange(ref s_triggered, 0, 1); - } + DateTimeOffset now = timeProvider.GetUtcNow().StartOfDay(); + DateTimeOffset[] dates = [.. Enumerable + .Range(0, settings.ForwardDays) + .Select(i => now.AddDays(i))]; + + int result = await repository.Migrate(dates, cancellationToken).ConfigureAwait(false); + _cts.Cancel(); + return result; } - else + catch (OperationCanceledException) { - do - { - await Task.Delay(settings.WaitMigrationTimeout, cancellationToken); - } - while (s_triggered != 0); + return -1; + } + finally + { + _migrationLock.Release(); } - - return -1; } } diff --git a/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs b/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs index aa82f95c..ec3d6564 100644 --- a/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs +++ b/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs @@ -20,11 +20,11 @@ internal sealed partial class PartRepository( public async Task ExecuteDDL(string sql, CancellationToken cancellationToken) { - await _migrationSemaphore.WaitAsync(cancellationToken); + await _migrationSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); try { return await PgRetryStrategy.ExecuteWithRetry( - async t => await dataSource.ExecuteNonQuery(sql, t), + async t => await dataSource.ExecuteNonQuery(sql, t).ConfigureAwait(false), cancellationToken: cancellationToken); } finally @@ -42,7 +42,7 @@ public async Task CreatePart( ISqlTableBuilder builder = sqlBuilder[tableName] ?? throw new KeyNotFoundException(tableName); string sql = builder.CreateSql(date, partValues); - return await ExecuteDDL(sql, cancellationToken); + return await ExecuteDDL(sql, cancellationToken).ConfigureAwait(false); } public async Task Migrate( @@ -55,9 +55,9 @@ public async Task Migrate( int i = 0; - await foreach (string sql in sqlBuilder.MigrateSql(dates, resolve)) + await foreach (string sql in sqlBuilder.MigrateSql(dates, resolve).ConfigureAwait(false)) { - await ExecuteDDL(sql, cancellationToken); + await ExecuteDDL(sql, cancellationToken).ConfigureAwait(false); i++; } return i; @@ -77,7 +77,7 @@ public async Task Migrate(DateTimeOffset[] dates, CancellationToken cancell if (supMigration != null) { - return await supMigration.GetParts(cancellationToken); + return await supMigration.GetParts(cancellationToken).ConfigureAwait(false); } else { @@ -91,7 +91,7 @@ public async Task Migrate(DateTimeOffset[] dates, CancellationToken cancell return []; - }, cancellationToken); + }, cancellationToken).ConfigureAwait(false); return i; } @@ -104,10 +104,10 @@ public async Task> GetPartsFromDate( string sql = sqlBuilder.SelectPartsFromDateSql(tableName); long unixTime = fromDate.ToUniversalTime().StartOfDay().ToUnixTimeSeconds(); - return await GetPartsFormDateWithRetry(sql, unixTime, cancellationToken); + return await GetPartsFromDateWithRetry(sql, unixTime, cancellationToken).ConfigureAwait(false); } - private async Task> GetPartsFormDateWithRetry( + private async Task> GetPartsFromDateWithRetry( string sql, long unixTime, CancellationToken cancellationToken) { return await PgRetryStrategy.ExecuteWithRetry( @@ -118,7 +118,7 @@ private async Task> GetPartsFormDateWithRetry( return await dataSource.ExecuteReaderList( sql, ReadPartInfo, - [new NpgsqlParameter("from_date", unixTime)], t); + [new NpgsqlParameter("from_date", unixTime)], t).ConfigureAwait(false); } catch (PostgresException ex) when (UndefinedTable(ex)) { @@ -141,7 +141,7 @@ public async Task> GetPartsToDate( sql , ReadPartInfo , [new NpgsqlParameter("to_date", toDate.ToUnixTimeSeconds())] - , cancellationToken); + , cancellationToken).ConfigureAwait(false); } catch (PostgresException ex) when (UndefinedTable(ex)) { @@ -153,7 +153,7 @@ public async Task DropPartsToDate( string tableName, DateTimeOffset toDate, CancellationToken cancellationToken = default) { int droppedCount = 0; - List list = await GetPartsToDate(tableName, toDate, cancellationToken); + List list = await GetPartsToDate(tableName, toDate, cancellationToken).ConfigureAwait(false); LogStartingToDrop(tableName, toDate); @@ -166,7 +166,7 @@ public async Task DropPartsToDate( string sql = settings.DropPartSql(part.Id); try { - await ExecuteDDL(sql, cancellationToken); + await ExecuteDDL(sql, cancellationToken).ConfigureAwait(false); droppedCount++; LogSuccessfullyDropped(part.Id, part.RootTableName); } diff --git a/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlBuilder.cs b/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlBuilder.cs index 40ec2caa..4528db89 100644 --- a/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlBuilder.cs @@ -18,7 +18,7 @@ public async IAsyncEnumerable MigrateSql(DateTimeOffset[] dates, Func 0) { diff --git a/src/Sa.Schedule/Engine/JobController.cs b/src/Sa.Schedule/Engine/JobController.cs index 97b31e6f..4a0f9d21 100644 --- a/src/Sa.Schedule/Engine/JobController.cs +++ b/src/Sa.Schedule/Engine/JobController.cs @@ -199,6 +199,8 @@ public void ExecutionFailed(Exception exception) return; } + // Track retry count. The scheduler decides whether to re-enqueue the job + // based on FailedRetries < RetryCount. if (_context.FailedRetries < settings.ErrorHandling.RetryCount) { _context.FailedRetries++; @@ -212,6 +214,7 @@ public void ExecutionFailed(Exception exception) return; } + // All retries exhausted — delegate to the registered error handler. _context.ServiceProvider.GetService()?.HandleError(_context, error); } diff --git a/src/Sa.Utils.WorkQueue/SaWorkQueue.cs b/src/Sa.Utils.WorkQueue/SaWorkQueue.cs index f149a06a..998a4610 100644 --- a/src/Sa.Utils.WorkQueue/SaWorkQueue.cs +++ b/src/Sa.Utils.WorkQueue/SaWorkQueue.cs @@ -508,8 +508,10 @@ public void Shutdown() tasks = [.. _taskReaders]; } - // Use ConfigureAwait(false) to avoid potential deadlocks from blocking on async operations - Task.WhenAll(tasks).ConfigureAwait(false).GetAwaiter().GetResult(); + // Use synchronous wait only here — we already exited the async context. + // Readers are guaranteed to terminate because _shutdownCts is cancelled + // and the channel writer is completed. + Task.WaitAll(tasks, TimeSpan.FromSeconds(30)); ClearRemainingItems(); } @@ -529,6 +531,15 @@ private void ClearRemainingItems() OnStatusChanged(item.Input, SaWorkStatus.Faulted, err); MarkInactive(); } + + // Drain any orphaned task counters — if items were enqueued but not yet + // counted (race between Enqueue and Shutdown), decrement to prevent + // IsIdle() from returning false forever. + lock (_wiSync) + { + if (_taskCount > 0) + _taskCount = 0; + } } private void CompleteDispose() diff --git a/src/Sa/Classes/IProcessExecutor.cs b/src/Sa/Classes/IProcessExecutor.cs index 8f6a3c6b..586b3f22 100644 --- a/src/Sa/Classes/IProcessExecutor.cs +++ b/src/Sa/Classes/IProcessExecutor.cs @@ -281,7 +281,7 @@ public async Task ExecuteStdOutAsync( } finally { - await stdoutStream.DisposeAsync(); + await stdoutStream.DisposeAsync().ConfigureAwait(false); } await Task.WhenAll(backgroundTasks).ConfigureAwait(false); } @@ -333,11 +333,9 @@ private static async Task WriteToStdInAsync( try { // input to stdin - await using (inputStream.ConfigureAwait(false)) - { - await inputStream.CopyToAsync(process.StandardInput.BaseStream, cancellationToken) + await using var _ = inputStream; + await inputStream.CopyToAsync(process.StandardInput.BaseStream, cancellationToken) .ConfigureAwait(false); - } // Завершаем запись await process.StandardInput.FlushAsync(cancellationToken).ConfigureAwait(false); diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryConsumerGroupManagerTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryConsumerGroupManagerTests.cs index 0812139e..72b0fd7f 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryConsumerGroupManagerTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryConsumerGroupManagerTests.cs @@ -628,7 +628,7 @@ public async Task Manager_ConcurrentApplyAndGet_NoExceptions() const int iterations = 50; var exceptions = new List(); - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15)); // Publish messages var messages = Enumerable.Range(1, 20) @@ -647,7 +647,7 @@ public async Task Manager_ConcurrentApplyAndGet_NoExceptions() } catch (Exception ex) { - Interlocked.Exchange(ref exceptions, exceptions.Append(ex).ToList()); + Interlocked.Exchange(ref exceptions, [.. exceptions, ex]); } await Task.Delay(2, cts.Token); @@ -671,7 +671,7 @@ public async Task Manager_ConcurrentApplyAndGet_NoExceptions() } catch (Exception ex) { - Interlocked.Exchange(ref exceptions, exceptions.Append(ex).ToList()); + Interlocked.Exchange(ref exceptions, [.. exceptions, ex]); } await Task.Delay(2, cts.Token); @@ -691,7 +691,7 @@ public async Task Manager_ConcurrentApplyAndGet_NoExceptions() } catch (Exception ex) { - Interlocked.Exchange(ref exceptions, exceptions.Append(ex).ToList()); + Interlocked.Exchange(ref exceptions, [.. exceptions, ex]); } } }, cts.Token); From c2a97d5748434b31aaa1513594cb8a99d6357fab Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 1 Jul 2026 00:12:49 +0300 Subject: [PATCH 24/33] =?UTF-8?q?CancellationTokenSource=20=D0=BD=D0=B5=20?= =?UTF-8?q?=D0=BE=D1=81=D0=B2=D0=BE=D0=B1=D0=BE=D0=B6=D0=B4=D0=B0=D0=BB?= =?UTF-8?q?=D0=B8=D1=81=D1=8C=20=D0=BF=D1=80=D0=B8=20=D1=83=D0=B4=D0=B0?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B8=D0=B8=20=D1=87=D0=B8=D1=82=D0=B0=D1=82?= =?UTF-8?q?=D0=B5=D0=BB=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Sa.Utils.WorkQueue/SaWorkQueue.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Sa.Utils.WorkQueue/SaWorkQueue.cs b/src/Sa.Utils.WorkQueue/SaWorkQueue.cs index 998a4610..22c7b5f4 100644 --- a/src/Sa.Utils.WorkQueue/SaWorkQueue.cs +++ b/src/Sa.Utils.WorkQueue/SaWorkQueue.cs @@ -352,6 +352,9 @@ private void RemoveReader(CancellationTokenSource cts) _readerCount--; _concurrency = _readerCount; } + + // Освобождаем CTS после удаления из списка + cts.Dispose(); } private async Task ExecuteItemAsync(WorkItem item, CancellationToken ct) From 6f256b3a47f0d16db8db39774621162da0babb20 Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 1 Jul 2026 00:47:34 +0300 Subject: [PATCH 25/33] =?UTF-8?q?rename=20PostponeAt=20to=20PostponeDelay?= =?UTF-8?q?=20=E2=80=94=20clarify=20TimeSpan=20semantics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Commands/FinishDeliveryCommand.cs | 2 +- src/Sa.Outbox/Delivery/OutboxContext.cs | 18 ++++++++--------- src/Sa.Outbox/Exceptions/DeliveryException.cs | 4 ++-- src/Sa.Outbox/IOutboxContext.cs | 4 ++-- .../Commands/ErrorDeliveryGroupingTests.cs | 6 +++--- .../Sa.Outbox.Tests/DeliveryCourierTests.cs | 14 ++++++------- .../Sa.Outbox.Tests/FakeOutboxContext.cs | 20 +++++++++---------- 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/Sa.Outbox.PostgreSql/Commands/FinishDeliveryCommand.cs b/src/Sa.Outbox.PostgreSql/Commands/FinishDeliveryCommand.cs index 85ee7456..3c044a58 100644 --- a/src/Sa.Outbox.PostgreSql/Commands/FinishDeliveryCommand.cs +++ b/src/Sa.Outbox.PostgreSql/Commands/FinishDeliveryCommand.cs @@ -65,7 +65,7 @@ private static void AddContextParameters( var result = context.DeliveryResult; var message = GetErrorMessage(context.Exception, result.Message, errors); - var lockExpiresOn = (result.CreatedAt + context.PostponeAt).ToUnixTimeSeconds(); + var lockExpiresOn = (result.CreatedAt + context.PostponeDelay).ToUnixTimeSeconds(); var errorId = GetErrorId(context.Exception, errors); cmd diff --git a/src/Sa.Outbox/Delivery/OutboxContext.cs b/src/Sa.Outbox/Delivery/OutboxContext.cs index 24a32a5a..0490295a 100644 --- a/src/Sa.Outbox/Delivery/OutboxContext.cs +++ b/src/Sa.Outbox/Delivery/OutboxContext.cs @@ -22,14 +22,14 @@ internal sealed class OutboxContext( public DeliveryStatus DeliveryResult { get; private set; } - public TimeSpan PostponeAt { get; private set; } = TimeSpan.Zero; + public TimeSpan PostponeDelay { get; private set; } = TimeSpan.Zero; public Exception? Exception { get; private set; } - public void Postpone(TimeSpan postpone, string? message = null) - => SetDeliveryStatus(DeliveryStatusCode.Postpone, message, null, postpone); + public void Postpone(TimeSpan postponeDelay, string? message = null) + => SetDeliveryStatus(DeliveryStatusCode.Postpone, message, null, postponeDelay); - public void Retry(TimeSpan postpone, string? message = null) - => SetDeliveryStatus(DeliveryStatusCode.Retry, message, null, postpone); + public void Retry(TimeSpan postponeDelay, string? message = null) + => SetDeliveryStatus(DeliveryStatusCode.Retry, message, null, postponeDelay); public void Ok(string? message = null) => SetDeliveryStatus(DeliveryStatusCode.Ok, message); @@ -56,7 +56,7 @@ public void MovedPermanently(string? message = null) => SetDeliveryStatus(DeliveryStatusCode.MovedPermanently, message); - public void Warn(Exception exception, string? message = null, TimeSpan? postpone = null) + public void Warn(Exception exception, string? message = null, TimeSpan? postponeDelay = null) { ArgumentNullException.ThrowIfNull(exception); @@ -66,7 +66,7 @@ public void Warn(Exception exception, string? message = null, TimeSpan? postpone deliveryException?.StatusCode ?? DeliveryStatusCode.Warn, message ?? exception.Message, exception, - postpone ?? deliveryException?.PostponeAt); + postponeDelay ?? deliveryException?.PostponeDelay); } @@ -109,7 +109,7 @@ private void SetDeliveryStatus( DeliveryStatusCode statusCode, string? message = null, Exception? exception = null, - TimeSpan? postpone = null) + TimeSpan? postponeDelay = null) { DeliveryResult = new DeliveryStatus( statusCode, @@ -117,7 +117,7 @@ private void SetDeliveryStatus( GetUtcNow()); Exception = exception; - PostponeAt = postpone ?? TimeSpan.Zero; + PostponeDelay = postponeDelay ?? TimeSpan.Zero; } diff --git a/src/Sa.Outbox/Exceptions/DeliveryException.cs b/src/Sa.Outbox/Exceptions/DeliveryException.cs index d26ca8ae..36ba13d8 100644 --- a/src/Sa.Outbox/Exceptions/DeliveryException.cs +++ b/src/Sa.Outbox/Exceptions/DeliveryException.cs @@ -6,8 +6,8 @@ public class DeliveryException( string message, Exception? innerException, DeliveryStatusCode statusCode, - TimeSpan? postponeAt = null) : OutboxException(message, innerException) + TimeSpan? postponeDelay = null) : OutboxException(message, innerException) { public DeliveryStatusCode StatusCode => statusCode; - public TimeSpan? PostponeAt => postponeAt; + public TimeSpan? PostponeDelay => postponeDelay; } diff --git a/src/Sa.Outbox/IOutboxContext.cs b/src/Sa.Outbox/IOutboxContext.cs index 6ee15827..88a45616 100644 --- a/src/Sa.Outbox/IOutboxContext.cs +++ b/src/Sa.Outbox/IOutboxContext.cs @@ -40,9 +40,9 @@ public interface IOutboxContext Exception? Exception { get; } /// - /// Gets the duration for which the message processing is postponed. + /// Gets the delay duration before the message should be retried. /// - TimeSpan PostponeAt { get; } + TimeSpan PostponeDelay { get; } } diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Commands/ErrorDeliveryGroupingTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Commands/ErrorDeliveryGroupingTests.cs index 6b18d829..e5931dfc 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Commands/ErrorDeliveryGroupingTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Commands/ErrorDeliveryGroupingTests.cs @@ -93,7 +93,7 @@ private static IOutboxContext CreateMockContext( new OutboxPartInfo(0, "task-part", DateTimeOffset.UtcNow)), DeliveryResult: result, Exception: exception, - PostponeAt: TimeSpan.Zero); + PostponeDelay: TimeSpan.Zero); } /// @@ -106,7 +106,7 @@ private sealed class TestOutboxContext( OutboxTaskDeliveryInfo DeliveryInfo, DeliveryStatus DeliveryResult, Exception? Exception, - TimeSpan PostponeAt) : IOutboxContext + TimeSpan PostponeDelay) : IOutboxContext { public Guid OutboxId { get; } = OutboxId; public string PayloadId { get; } = PayloadId; @@ -114,6 +114,6 @@ private sealed class TestOutboxContext( public OutboxTaskDeliveryInfo DeliveryInfo { get; } = DeliveryInfo; public DeliveryStatus DeliveryResult { get; } = DeliveryResult; public Exception? Exception { get; } = Exception; - public TimeSpan PostponeAt { get; } = PostponeAt; + public TimeSpan PostponeDelay { get; } = PostponeDelay; } } diff --git a/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs b/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs index f417db97..577b79e8 100644 --- a/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs +++ b/src/Tests/Sa.Outbox.Tests/DeliveryCourierTests.cs @@ -118,7 +118,7 @@ public async Task Deliver_ProcessorThrows_MessageWarnedWithBackoff() Assert.Equal(0, result); Assert.Equal(DeliveryStatusCode.Warn, ctx.DeliveryResult.Code); Assert.Same(testException, ctx.Exception); - Assert.Equal(TimeSpan.FromSeconds(5), ctx.PostponeAt); + Assert.Equal(TimeSpan.FromSeconds(5), ctx.PostponeDelay); } [Fact] @@ -152,9 +152,9 @@ public async Task Deliver_ProcessorThrows_MultipleMessagesAllWarned() Assert.Equal(DeliveryStatusCode.Warn, ctx1.DeliveryResult.Code); Assert.Equal(DeliveryStatusCode.Warn, ctx2.DeliveryResult.Code); Assert.Equal(DeliveryStatusCode.Warn, ctx3.DeliveryResult.Code); - Assert.Equal(expectedBackoffs[0], ctx1.PostponeAt); - Assert.Equal(expectedBackoffs[1], ctx2.PostponeAt); - Assert.Equal(expectedBackoffs[2], ctx3.PostponeAt); + Assert.Equal(expectedBackoffs[0], ctx1.PostponeDelay); + Assert.Equal(expectedBackoffs[1], ctx2.PostponeDelay); + Assert.Equal(expectedBackoffs[2], ctx3.PostponeDelay); Assert.Same(testException, ctx1.Exception); Assert.Same(testException, ctx2.Exception); Assert.Same(testException, ctx3.Exception); @@ -181,7 +181,7 @@ public async Task Deliver_ProcessorThrows_UseDefaultRetryStrategy() Assert.Equal(0, result); Assert.Equal(DeliveryStatusCode.Warn, ctx.DeliveryResult.Code); Assert.NotNull(ctx.Exception); - Assert.True(ctx.PostponeAt > TimeSpan.Zero); + Assert.True(ctx.PostponeDelay > TimeSpan.Zero); } #endregion @@ -363,7 +363,7 @@ await courier.Deliver( Assert.Single(recordedAttempts); Assert.Equal(1, recordedAttempts[0]); // attempt 0 + 1 = 1 - Assert.Equal(TimeSpan.FromSeconds(5), ctx.PostponeAt); + Assert.Equal(TimeSpan.FromSeconds(5), ctx.PostponeDelay); } #endregion @@ -418,7 +418,7 @@ public async Task Deliver_ProcessorThrows_DeterministicStrategy_PredictableDelay CancellationToken.None); Assert.Equal(0, result); - Assert.Equal(TimeSpan.FromSeconds(1), ctx.PostponeAt); + Assert.Equal(TimeSpan.FromSeconds(1), ctx.PostponeDelay); } #endregion diff --git a/src/Tests/Sa.Outbox.Tests/FakeOutboxContext.cs b/src/Tests/Sa.Outbox.Tests/FakeOutboxContext.cs index 45e6418c..54dc7ff0 100644 --- a/src/Tests/Sa.Outbox.Tests/FakeOutboxContext.cs +++ b/src/Tests/Sa.Outbox.Tests/FakeOutboxContext.cs @@ -13,7 +13,7 @@ public sealed class FakeOutboxContext( DeliveryStatusCode initialStatus = DeliveryStatusCode.Pending) : IOutboxContextOperations { private DeliveryStatus _deliveryResult = new(initialStatus, payloadId, DateTimeOffset.UtcNow); - private TimeSpan _postponeAt; + private TimeSpan _postponeDelay; private Exception? _exception; public Guid OutboxId { get; set; } = Guid.NewGuid(); @@ -33,7 +33,7 @@ public OutboxTaskDeliveryInfo DeliveryInfo public DeliveryStatus DeliveryResult => _deliveryResult; public Exception? Exception => _exception; - public TimeSpan PostponeAt => _postponeAt; + public TimeSpan PostponeDelay => _postponeDelay; public FakeOutboxContext() : this("fake-msg", 0, DeliveryStatusCode.Pending) { } @@ -58,17 +58,17 @@ public void Aborted(string? message = null) public void MovedPermanently(string? message = null) => SetStatus(DeliveryStatusCode.MovedPermanently, message); - public void Postpone(TimeSpan postpone, string? message = null) - => SetStatus(DeliveryStatusCode.Postpone, message, postpone: postpone); + public void Postpone(TimeSpan postponeDelay, string? message = null) + => SetStatus(DeliveryStatusCode.Postpone, message, postponeDelay: postponeDelay); - public void Retry(TimeSpan postpone, string? message = null) - => SetStatus(DeliveryStatusCode.Retry, message, postpone: postpone); + public void Retry(TimeSpan postponeDelay, string? message = null) + => SetStatus(DeliveryStatusCode.Retry, message, postponeDelay: postponeDelay); - public void Warn(Exception exception, string? message = null, TimeSpan? postpone = null) + public void Warn(Exception exception, string? message = null, TimeSpan? postponeDelay = null) { ArgumentNullException.ThrowIfNull(exception); _exception = exception; - _postponeAt = postpone ?? TimeSpan.Zero; + _postponeDelay = postponeDelay ?? TimeSpan.Zero; _deliveryResult = new DeliveryStatus(DeliveryStatusCode.Warn, message ?? exception.Message, GetUtcNow()); } @@ -106,10 +106,10 @@ public void ErrorMaxAttempts() public DateTimeOffset GetUtcNow() => DateTimeOffset.UtcNow; - private void SetStatus(DeliveryStatusCode code, string? message, TimeSpan? postpone = null, Exception? exception = null) + private void SetStatus(DeliveryStatusCode code, string? message, TimeSpan? postponeDelay = null, Exception? exception = null) { _deliveryResult = new DeliveryStatus(code, message ?? "", GetUtcNow()); - _postponeAt = postpone ?? TimeSpan.Zero; + _postponeDelay = postponeDelay ?? TimeSpan.Zero; _exception = exception; } From 0577684535b4d706a06f29bac4985de9f4092e43 Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 1 Jul 2026 12:46:55 +0300 Subject: [PATCH 26/33] ~ readme Signed-off-by: dundich --- README-ru.md | 251 ++++++++++++ README.md | 309 ++++++++++++--- src/Sa.Configuration.PostgreSql/Readme-ru.md | 129 +++++++ src/Sa.Configuration.PostgreSql/Readme.md | 4 +- src/Sa.Configuration/CommandLine/Readme-ru.md | 208 ++++++++++ src/Sa.Configuration/CommandLine/Readme.md | 226 ++++++++--- src/Sa.Configuration/Readme-ru.md | 284 ++++++++++++++ src/Sa.Configuration/Readme.md | 70 ++-- src/Sa.Data.PostgreSql/Readme-ru.md | 188 +++++++++ src/Sa.Data.PostgreSql/Readme.md | 107 ++++-- src/Sa.Data.S3/README.md | 88 +++-- src/Sa.Data.S3/Readme-ru.md | 135 +++++++ src/Sa.HybridFileStorage/Readme-ru.md | 311 +++++++++++++++ src/Sa.HybridFileStorage/Readme.md | 296 ++++++++++++-- src/Sa.Media.FFmpeg/Readme-ru.md | 288 ++++++++++++++ src/Sa.Media.FFmpeg/Readme.md | 293 +++++++++++--- src/Sa.Media/Readme-ru.md | 184 +++++++++ src/Sa.Media/Readme.md | 18 + src/Sa.Outbox.PostgreSql/Readme-ru.md | 99 +++-- src/Sa.Outbox.PostgreSql/Readme.md | 4 + .../Delivery/IOutboxConsumerManager.cs | 9 +- src/Sa.Outbox/Delivery/Job/DeliveryJob.cs | 15 +- .../Delivery/OutboxConsumerManager.cs | 6 +- src/Sa.Outbox/Readme-ru.md | 251 ++++++++++++ src/Sa.Outbox/Readme.md | 253 +++++++----- src/Sa.Partitional.PostgreSql/Readme-ru.md | 267 +++++++++++++ src/Sa.Partitional.PostgreSql/Readme.md | 227 ++++++++++- src/Sa.Schedule/Readme-ru.md | 360 ++++++++++++++++++ src/Sa.Schedule/Readme.md | 14 +- src/Sa.Utils.WorkQueue/Readme-ru.md | 174 +++++++++ src/Sa.Utils.WorkQueue/Readme.md | 154 +++++--- .../Configuration.Web.csproj | 2 +- src/Samples/Configuration.Web/README.md | 157 ++++++++ src/Samples/Configuration.Web/Readme-ru.md | 157 ++++++++ src/Samples/FFMpeg.Console/Readme-ru.md | 100 +++++ src/Samples/README-ru.md | 236 ++++++++++++ src/Samples/README.md | 236 ++++++++++++ .../DeliveryConsumerGroupManagerTests.cs | 10 +- .../OutboxConsumerManagerTests.cs | 54 +-- 39 files changed, 5674 insertions(+), 500 deletions(-) create mode 100644 README-ru.md create mode 100644 src/Sa.Configuration.PostgreSql/Readme-ru.md create mode 100644 src/Sa.Configuration/CommandLine/Readme-ru.md create mode 100644 src/Sa.Configuration/Readme-ru.md create mode 100644 src/Sa.Data.PostgreSql/Readme-ru.md create mode 100644 src/Sa.Data.S3/Readme-ru.md create mode 100644 src/Sa.HybridFileStorage/Readme-ru.md create mode 100644 src/Sa.Media.FFmpeg/Readme-ru.md create mode 100644 src/Sa.Media/Readme-ru.md create mode 100644 src/Sa.Outbox/Readme-ru.md create mode 100644 src/Sa.Partitional.PostgreSql/Readme-ru.md create mode 100644 src/Sa.Schedule/Readme-ru.md create mode 100644 src/Sa.Utils.WorkQueue/Readme-ru.md create mode 100644 src/Samples/Configuration.Web/README.md create mode 100644 src/Samples/Configuration.Web/Readme-ru.md create mode 100644 src/Samples/FFMpeg.Console/Readme-ru.md create mode 100644 src/Samples/README-ru.md create mode 100644 src/Samples/README.md diff --git a/README-ru.md b/README-ru.md new file mode 100644 index 00000000..3bfdd6a4 --- /dev/null +++ b/README-ru.md @@ -0,0 +1,251 @@ +# Sa — Набор инфраструктурных библиотек для .NET 10 + +Серия переиспользуемых .NET 10-библиотек, сфокусированных на инфраструктурных паттернах для распределённых систем. Целевая платформа — **.NET 10.0**, используется **Native AOT**, применяется паттерн **Central Package Management (CPM)** через `Directory.Packages.props`. + +--- + +### [Sa.Outbox.PostgreSql](src/Sa.Outbox.PostgreSql) + +Реализация паттерна **Transactional Outbox** на PostgreSQL для гарантированной доставки сообщений в распределённых системах. Предотвращает потерю сообщений и гарантирует обработку даже при сбоях. + +- **Гарантированная доставка**: сообщения хранятся в БД до успешной обработки +- **Параллельная обработка**: несколько воркеров безопасно конкурируют за задачи через `SKIP LOCKED` +- **Мультитенантность**: изоляция и параллелизм по арендаторам +- **Авто-масштабирование**: runtime-изменение параллелизма без перезапуска +- **Планируемая очистка**: автоматическое удаление старых партиций +- **Self-bootstrapping**: авто-регистрация настроек консьюмера при первом запуске +- **Immutable настройки**: `OutboxConsumerSettings` record с fluent-билдером + +See [full README](src/Sa.Outbox.PostgreSql/Readme.md). + +--- + +### [Sa.Partitional.PostgreSql](src/Sa.Partitional.PostgreSql) — Декларативное партиционирование PostgreSQL + +Объявление партицирования таблиц PostgreSQL (range: день/месяц/год; list) с автоматической миграцией, планированием очистки и in-memory кэшем. + +- **Range-партиционирование** по дню, месяцу или году с авто-именованием по timestamp +- **List-партиционирование** по строковым/числовым ключам с иерархическими дочерними партициями +- **Fluent-билдер** для объявления таблиц, настройки fillfactor, кастомных ограничений и миграций +- **Автоматическая миграция** — предсоздание будущих партиций как фоновая задача +- **Автоматическая очистка** — удаление старых партиций по настраиваемому окну удержания +- **In-memory кэш** — избегает повторных запросов к каталогу; инвалидируется при runtime-изменениях +- **StrOrNum** — discriminated union для типобезопасных значений ключей партиций + +See the full [Guide](src/Sa.Partitional.PostgreSql/Guide.md) and [API Reference](src/Sa.Partitional.PostgreSql/ApiReference.md). + +--- + +### [Sa.Schedule](src/Sa.Schedule) — Планировщик задач + +Конфигурация и выполнение задач по расписанию — cron, интервалы, одноразовые запуски. + +| Возможность | Описание | +|-------------|----------| +| **Cron-тайминги** | Любое cron-выражение через `IJobTiming.FromCron()` | +| **Интервальное расписание** | `EverySeconds`, `EveryMinutes`, `EveryHours`, `EveryDays` | +| **Одноразовые задачи** | `RunOnce()` с опциональной начальной задержкой | +| **Стратегии ошибок** | `CloseApplication`, `AbortJob`, `StopAllJobs`, `Ignore` | +| **Повторные попытки** | Настраиваемое количество ретраев на ошибку | +| **Интерцепторы** | Кросс логика через `IJobInterceptor` | +| **Обработчики ошибок** | Глобальные `HandleError`-хендлеры на уровне планировщика | +| **DI-интеграция** | `AddSaSchedule(Action)` с `BackgroundService` | + +```csharp +builder.Services.AddSaSchedule(b => b + .AddJob() + .WithName("cleanup") + .EveryHours(1) + .ConfigureErrorHandling(eh => eh.IfErrorRetry(3).ThenAbortJob()) +); +``` + +--- + +### [Sa.HybridFileStorage](src/Sa.HybridFileStorage) — Гибридное файловое хранилище + +Абстракция файлового хранилища с автоматическим failover между провайдерами (FileSystem ↔ S3 ↔ PostgreSQL). + +| Возможность | Описание | +|-------------|----------| +| **Мультипровайдер** | FileSystem, S3 (Minio), PostgreSQL — подключайте сколько угодно | +| **Автоматический failover** | При недоступности одного провайдера — переход к следующему | +| **Интерцепторы** | `before`, `after`, `onError` хуки на каждом провайдере | +| **Пакетные операции** | `CopyToScopeBatchAsync` с параллелизмом и прогрессом | +| **Расширения** | `CopyFromFileAsync`, `CopyToBasketAsync` для удобства | +| **InMemory-провайдер** | Для тестирования: `AddSaInMemoryFileStorage()` | + +```csharp +builder.Services.AddSaHybridFileStorage(cfg => cfg + .ConfigureStorage((sp, c) => c + .AddStorage(new FileSystemStorage("disk")) + .AddStorage(new S3Storage("s3")) + ) +); +``` + +--- + +### [Sa.Configuration](src/Sa.Configuration) — Аргументы командной строки и секреты + +| Компонент | Назначение | +|-----------|------------| +| **Arguments** | Парсер CLI-аргументов в стиле dictionary — поддержка одиночных и множественных значений, типизированные геттеры (`GetBool`, `GetInt`, `GetTimeSpan` и т.д.) | +| **Secrets** | Безопасное управление секретами из файлов, переменных окружения и генерируемых host-key файлов. Поддержка chained stores и environment-aware загрузки | + +```csharp +var args = Arguments.CreateDefault(); +var dbPassword = args["db-password"]; // string? +var timeout = args.GetTimeSpan("timeout"); // TimeSpan? +``` + +--- + +### [Sa.Configuration.PostgreSql](src/Sa.Configuration.PostgreSql) — Динамическая конфигурация из PostgreSQL + +Добавляет источник конфигурации из БД — изменения отражаются в приложении без перекомпиляции и редеплоя. + +```csharp +builder.Configuration.AddSaPostgreSqlConfiguration(new PostgreSqlConfigurationOptions( + connectionString: "Host=localhost;Database=myapp", + selectSql: "SELECT config_key, config_value FROM app_config", + parameters: Array.Empty() +)); +``` + +--- + +### [Sa.Media](src/Sa.Media) — Асинхронное чтение WAV + +Памятно-эффективный асинхронный читатель WAV-файлов с конвертацией форматов. + +| Метод | Описание | +|-------|----------| +| `CreateFromFile(path)` | Открыть файл по пути | +| `GetHeaderAsync()` | Считать WAV-заголовок | +| `ReadSamplesPerChannelAsync()` | Потоковое чтение сэмплов по каналам | +| `ReadDoubleSamplesAsync()` | Нормализованные double-сэмплы [-1..1] | +| `ConvertToFormatAsync()` | Конвертация в PCM16/24/32, IEEE float | +| `ReadStreamableChunksAsync()` | Чанки фиксированного размера для streaming | + +```csharp +using var reader = AsyncWavReader.CreateFromFile("audio.wav"); +await foreach (var packet in reader.ReadDoubleSamplesAsync()) +{ + Console.WriteLine($"Ch{packet.ChannelId}: {packet.Sample:F4}"); +} +``` + +--- + +### [Sa.Media.FFmpeg](src/Sa.Media.FFmpeg) — Обёртка FFmpeg для .NET + +FFmpeg из коробки со встроенными бинарниками (Windows x64 + Linux) и DI. + +| Интерфейс | Назначение | +|-----------|------------| +| `IFFMpegExecutor` | Конвертация аудио/видео (PCM16LE, MP3, OGG) | +| `IFFProbeExecutor` | Извлечение метаданных (длина, каналы, частота, битрейт) | +| `IPcmS16LeChannelManipulator` | Разделение/объединение каналов | +| `IFFMpegLocator` | Автопоиск исполняемого FFmpeg | + +```csharp +builder.Services.AddSaFFMpeg(); + +var probe = IFFProbeExecutor.Default; +var meta = await probe.GetMetaInfo("input.mp3"); +Console.WriteLine($"Duration: {meta.Duration}s, Channels: {meta.Channels}"); +``` + +--- + +### [Sa.Data.PostgreSql](src/Sa.Data.PostgreSql) — Лёгкая обёртка Npgsql + +Без ORM-overhead, с DI, Native AOT и минимальными аллокациями. + +| Метод | Описание | +|-------|----------| +| `ExecuteNonQuery` | INSERT / UPDATE / DELETE / DDL с возвратом rowCount | +| `ExecuteScalar / ExecuteScalarTyped` | Одиночное значение с авто-кастомом | +| `ExecuteReader` | Потоковое чтение через callback (без загрузки в память) | +| `ExecuteReaderList` | Сборка всех строк в `List` | +| `ExecuteReaderFirst` | Первое значение первого столбца | +| `ExecuteReaderSingle` | Безопасное scalar-значение | +| `BeginBinaryImport` | Быстрый COPY BINARY для массового импорта | +| `PgRetryStrategy` | Повторы с jitter для transient-ошибок Npgsql | + +--- + +### [Sa.Utils.WorkQueue](src/Sa.Utils.WorkQueue) — Асинхронная очередь с ограничением параллелизма + +Высокопроизводительная очередь задач на базе `System.Threading.Channels` с ограниченной ёмкостью, динамическим контролем параллелизма и стратегиями масштабирования. + +| Возможность | Описание | +|-------------|----------| +| **Ограниченная очередь** | Back-pressure через `BoundedChannel` | +| **Динамический параллелизм** | Изменяйте `ConcurrencyLimit` на лету | +| **Стратегии масштабирования** | `Lifo` • `Fifo` • `RoundRobin` • `Random` | +| **Стратегии ошибок** | `Continue`, `StopReader`, `ShutdownQueue` | +| **Обратные вызовы статусов** | `Running` → `Completed` / `Faulted` / `Cancelled` / `Aborted` | +| **Логирование без аллокаций** | `[LoggerMessage]` source generator | + +See [full README](src/Sa.Utils.WorkQueue/Readme.md). + +--- + +## Образцы + +В `src/Samples/`: + +| Образец | Описание | +|---------|----------| +| [Configuration.Web](src/Samples/Configuration.Web) | CLI-аргументы + секреты в ASP.NET | +| [FFmpeg.Console](src/Samples/FFmpeg.Console) | Извлечение метаданных FFmpeg | +| [HybridFileStorage.Console](src/Samples/HybridFileStorage.Console) | Мульти-провайдерное хранилище | +| [Partitional.ConsoleApp](src/Samples/Partitional.ConsoleApp) | Декларативное партиционирование | +| [PgOutbox.ConsoleApp](src/Samples/PgOutbox.ConsoleApp) | Паттерн Outbox | +| [Schedule.Console](src/Samples/Schedule.Console) | Планировщик задач | +| [Storage.Tests](src/Samples/Storage.Tests) | Тесты гибридного хранилища | + +--- + +## Тесты + +В `src/Tests/`: 15 тестовых проектов на **xunit v3** с **Testcontainers** (PostgreSQL + Minio) для интеграционных тестов. + +--- + +## Сборка + +```powershell +# Полная сборка +.\build\do_build.ps1 + +# Запуск тестов +.\build\do_test.ps1 + +# Создание NuGet-пакетов +.\build\do_package.ps1 +``` + +Прямые команды dotnet: + +```powershell +dotnet restore src/Sa.slnx -c Release +dotnet build src/Sa.slnx -c Release -v n +dotnet test src/Sa.slnx -v n +``` + +--- + +## Архитектура + +- Целевая платформа — **.NET 10.0** с **Native AOT** +- **Central Package Management** — версии в `Directory.Packages.props` +- Общие утилиты в **Sa** линкуются в consuming-проект через `` +- Все пакеты используют SDK-style csproj с implicit usings, nullable и анализаторами +- Решение управляется через `.slnx` + +## Лицензия + +MIT diff --git a/README.md b/README.md index a1e45b79..dd5a2b02 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,115 @@ -# sa +# Sa — .NET 10 Infrastructure Libraries -dot net10 experimental aot project +Reusable infrastructure libraries for distributed .NET 10 systems — **Native AOT compatible**, **nullable enabled**, built on modern .NET primitives. +--- -## [Sa.Data.PostgreSql](src/Sa.Data.PostgreSql) +## Libraries -Лёгкая обёртка над Npgsql для типичных операций с PostgreSQL — без ORM overhead, с поддержкой DI, Native AOT и минимальными аллокациями. +### [Sa](src/Sa) — Shared Utilities -- **ExecuteNonQuery** — INSERT / UPDATE / DELETE / DDL с возвратом числа строк -- **ExecuteScalar / ExecuteScalarTyped** — получение одиночного значения с авто-кастом -- **ExecuteReader** — потоковое чтение строк через callback (без загрузки всего результата в память) -- **ExecuteReaderList** — сборка всех строк в `List` -- **ExecuteReaderFirst** — первое значение первого столбца (Guid, TimeSpan, DateTime, int, long и др.) -- **ExecuteReaderSingle / TryExecuteReaderSingle** — безопасное scalar-значение -- **ExecuteTransactionAsync** — атомарные операции с авто-commit/rollback -- **BeginBinaryImport** — быстрый COPY BINARY для массового импорта -- **PgDistributedLock** — распределённая блокировка на `pg_try_advisory_lock` -- **PgRetryStrategy** — повтор попыток с jitter для transient-ошибок Npgsql -- **DI-интеграция** — `AddSaPostgreSqlDataSource()` регистрирует всё автоматически +Common building blocks consumed by other packages via ``: -## [Sa.Outbox.PostgreSql](src/Sa.Outbox.PostgreSql) +- `LockRenewer` — automatic lock extension with configurable renewal interval +- `MurmurHash3` — compact hash for type identification and partitioning +- `Retry` — retry helpers with exponential backoff +- `ResetLazy` — lazily-evaluated, resettable cached value +- Extension methods: `DateTimeExtensions`, `EnumerableExtensions`, `ExceptionExtensions`, `SpanExtensions`, `StringExtensions`, `NumericExtensions`, `StrToExtensions`, `GuidExtensions` -Designed for implementing the Outbox pattern using PostgreSQL, which is used to ensure reliable message delivery in distributed systems. It helps prevent message loss and guarantees that messages will be processed even in the event of failures. +--- -- Reliable message delivery: Ensures that messages are stored in the database until they are successfully processed. -- Parallel processing: Enables messages to be processed in parallel, increasing system performance. -- Flexibility: Supports various types of messages and their handlers. -- Tenant support: Allows for even distribution of load. -- Data cleaning: scheduled deletion of old data. +### [Sa.Configuration](src/Sa.Configuration) — CLI Arguments & Secrets -## [Sa.Partitional.PostgreSql](src/Sa.Partitional.PostgreSql) +| Type | Purpose | +|------|---------| +| `Arguments` | Command-line argument parser — dictionary-like access, typed getters (`GetBool`, `GetInt`, `GetTimeSpan`, etc.) | +| `Secrets` | Secure secrets management from files, environment variables, and host-key files. Supports chained stores and templating (`${secret:key}`) | + +```csharp +// Arguments +var args = new Arguments(argsArray); +var port = args.GetInt("port") ?? 8080; + +// Secrets +var secrets = Secrets.CreateDefault(); +var populated = secrets.PopulateSecrets("Host={db_host};Password=${db_password}"); +``` + +--- + +### [Sa.Configuration.PostgreSql](src/Sa.Configuration.PostgreSql) — Dynamic DB Configuration + +PostgreSQL-backed `IConfigurationSource` — changes in the database reflect in-app without redeploy. + +```csharp +builder.Configuration.AddSaPostgreSqlConfiguration(new PostgreSqlConfigurationOptions( + connectionString: "Host=localhost;Database=myapp", + selectSql: "SELECT key, value FROM app_config", + parameters: Array.Empty())); +``` + +Supports parameterised queries and `PgRetryStrategy` for transient error handling. + +--- + +### [Sa.Data.PostgreSql](src/Sa.Data.PostgreSql) — Lightweight Npgsql Wrapper + +Thin wrapper over Npgsql for typical database operations — zero ORM overhead, Native AOT friendly. + +| Method | Description | +|--------|-------------| +| `ExecuteNonQueryAsync` | INSERT / UPDATE / DELETE / DDL with row count return | +| `ExecuteScalarAsync` / `ExecuteScalarTypedAsync` | Single value with auto-cast | +| `ExecuteReaderAsync` | Streaming row reading via callback (no full result in memory) | +| `ExecuteReaderListAsync` | Collect all rows into `List` | +| `ExecuteReaderFirstAsync` | First column of first row (Guid, TimeSpan, DateTime, int, long, etc.) | +| `ExecuteReaderSingleAsync` / `TryExecuteReaderSingleAsync` | Safe scalar with nullability | +| `ExecuteTransactionAsync` | Atomic operations with auto commit/rollback | +| `BeginBinaryImportAsync` | Fast COPY BINARY for bulk inserts | +| `PgRetryStrategy` | Retry with jitter for transient Npgsql errors | + +DI registration: `AddSaPostgreSqlDataSource()`. + +--- + +### [Sa.Data.S3](src/Sa.Data.S3) — S3 Data Client + +Minio-compatible S3 client for data operations. + +--- + +### [Sa.Outbox](src/Sa.Outbox) — Transactional Outbox Core + +Base infrastructure for the **Transactional Outbox** pattern — guarantees atomic message recording alongside business operations within a single database transaction, with reliable delivery, retries, blocking, multi-threading, and multi-tenancy support. + +Defines abstractions; concrete DB work (PostgreSQL, SQL Server, etc.) is implemented by providers (`Sa.Outbox.PostgreSql`, `Sa.Outbox.SqlServer`). + +| Key Type | Purpose | +|----------|---------| +| `IOutboxBuilder` | Fluent configuration builder | +| `IOutboxMessagePublisher` | Publish messages to outbox | +| `IConsumer` | Message consumer interface | +| `IOutboxContextOperations` | Delivery status change operations (`Ok`, `Error`, `Warn`, `Postpone`, etc.) | +| `OutboxConsumerSettings` | Immutable snapshot of consumer group settings (interval, batches, concurrency, retries…) | +| `OutboxConsumerSettingsBuilder` | Fluent builder for creating/updating `OutboxConsumerSettings` | +| `IOutboxConsumerManager` | Runtime manager: atomic swap, pause/resume, change subscriptions | +| `IDeliverySnapshot` | Read-only view of registered deliveries for diagnostics | +| `DeliveryStatus` / `DeliveryStatusCode` | HTTP-like delivery status codes | +| `ExponentialBackoffRetryStrategy` | Exponential backoff with jitter | + +See individual provider READMEs for full usage examples. + +--- + +### [Sa.Outbox.PostgreSql](src/Sa.Outbox.PostgreSql) — PostgreSQL Provider + +Production-ready PostgreSQL implementation with UUID v7 IDs, BINARY COPY bulk insertion, `SKIP LOCKED` concurrent consumption, advisory locks for offset coordination, and automated partition migration/cleanup. + +See [full README](src/Sa.Outbox.PostgreSql/Readme.md). + +--- + +### [Sa.Partitional.PostgreSql](src/Sa.Partitional.PostgreSql) — Declarative Partitioning Declarative PostgreSQL table partitioning for .NET 10 — range (day/month/year) and list partitioning with automated migration, cleanup scheduling, and in-memory caching. @@ -40,50 +120,175 @@ Declarative PostgreSQL table partitioning for .NET 10 — range (day/month/year) - **Automated cleanup** — drop old partitions past a configurable retention window - **In-memory cache** — avoids repeated catalog queries; auto-invalidates on runtime changes - **StrOrNum** discriminated union for type-safe partition key values -- See the full [Guide](src/Sa.Partitional.PostgreSql/Guide.md) and [API Reference](src/Sa.Partitional.PostgreSql/ApiReference.md). -## [Sa.Schedule](src/Sa.Schedule) +See [Guide](src/Sa.Partitional.PostgreSql/Guide.md) and [API Reference](src/Sa.Partitional.PostgreSql/ApiReference.md). + +--- + +### [Sa.Schedule](src/Sa.Schedule) — Scheduled Task Executor + +Configurable and executable scheduled tasks with failure strategies. + +| Feature | Description | +|---------|-------------| +| **Flexible timing** | Cron expressions, fixed intervals (seconds/minutes/hours/days), one-shot delays | +| **Failure strategies** | `CloseApplication`, `AbortJob`, `StopAllJobs`, or `Ignore` | +| **Retry on failure** | Configurable retry count per job | +| **Concurrency control** | Per-job `ConcurrencyLimit` and `MaxConcurrency` | +| **Interceptors** | `IJobInterceptor` for pre/post execution hooks | +| **Error handlers** | Global `Func` error handlers | +| **Runtime management** | Start, stop, restart individual schedulers via `IScheduler` | + +```csharp +builder.Services.AddSaSchedule(builder => builder + .UseHostedService() + .AddJob() + .WithName("cleanup") + .EveryHours(1) + .ConfigureErrorHandling(eh => eh.IfErrorRetry(3).ThenAbortJob())); +``` + +--- + +### [Sa.HybridFileStorage](src/Sa.HybridFileStorage) — Multi-Provider File Storage + +`IHybridFileStorage` abstracts file operations across multiple storage providers (FileSystem, S3, PostgreSQL) with automatic failover. + +| Capability | Description | +|------------|-------------| +| **Upload / Download / Delete** | Standard file operations with streaming | +| **Multi-provider** | Register any `IFileStorage` implementation | +| **Automatic failover** | Tries providers sequentially; aggregates errors if all fail | +| **Batch operations** | Parallel batch upload/download with progress reporting | +| **Interceptors** | Pre/post/on-error hooks per provider | +| **Built-in providers** | `InMemoryFileStorage` (testing), plus `FileSystem`, `S3`, `Postgres` in separate packages | + +```csharp +builder.Services.AddSaHybridFileStorage(cfg => cfg + .ConfigureStorage(sp => sp + .AddStorage(new FileSystemStorage("/data/uploads")) + .AddStorage(new S3Storage("s3-bucket")))); +``` + +Providers: [`Sa.HybridFileStorage.FileSystem`](src/Sa.HybridFileStorage.FileSystem), [`Sa.HybridFileStorage.S3`](src/Sa.HybridFileStorage.S3), [`Sa.HybridFileStorage.Postgres`](src/Sa.HybridFileStorage.Postgres). + +--- + +### [Sa.Media](src/Sa.Media) — Async WAV Reader + +Memory-efficient, fully async WAV file reader built on `System.IO.Pipelines`. + +| Method | Description | +|--------|-------------| +| `CreateFromFile` / `Create(Stream)` | Factory methods | +| `GetHeaderAsync` | Parse WAV header | +| `ReadSamplesPerChannelAsync` | Raw bytes per channel | +| `ReadDoubleSamplesAsync` | Normalized double samples | +| `ConvertToFormatAsync` | Convert to PCM16/24/32, IEEE float | +| `ReadStreamableChunksAsync` | Streaming chunks with configurable batch size | + +```csharp +using var reader = AsyncWavReader.CreateFromFile("audio.wav"); +var header = await reader.GetHeaderAsync(); +await foreach (var packet in reader.ReadDoubleSamplesAsync()) +{ + Console.WriteLine($"Ch{packet.ChannelId}: {packet.Sample}"); +} +``` + +--- + +### [Sa.Media.FFmpeg](src/Sa.Media.FFmpeg) — FFmpeg .NET Wrapper + +Ready-to-use FFmpeg integration with built-in binaries (Windows x64 + Linux) and DI support. + +| Interface | Purpose | +|-----------|---------| +| `IFFMpegExecutor` | Audio/video conversion (PCM16LE, MP3, OGG) | +| `IFFProbeExecutor` | Metadata extraction (duration, channels, sample rate, bitrate) | +| `IPcmS16LeChannelManipulator` | Channel split/join operations | +| `IFFMpegLocator` | Auto-discovery of FFmpeg executable | + +```csharp +builder.Services.AddSaFFMpeg(); + +var probe = IFFProbeExecutor.Default; +var meta = await probe.GetMetaInfo("input.mp3"); +Console.WriteLine($"Duration: {meta.Duration}s, Channels: {meta.Channels}"); +``` + +--- + +### [Sa.Utils.WorkQueue](src/Sa.Utils.WorkQueue) — Async Queue with Concurrency Limiting + +High-performance task queue built on `System.Threading.Channels` with bounded capacity, dynamic concurrency scaling, and multiple reader-scaling strategies. + +| Feature | Description | +|---------|-------------| +| **Bounded queue** | Back-pressure via `BoundedChannel` — overflow handled by `Wait`, `DropWrite`, `DropOldest` | +| **Dynamic concurrency** | Change `ConcurrencyLimit` at runtime | +| **Scaling strategies** | `Lifo` • `Fifo` • `RoundRobin` • `Random` | +| **Error strategies** | `Continue`, `StopReader`, `ShutdownQueue` | +| **Status callbacks** | `Running` → `Completed` / `Faulted` / `Cancelled` / `Aborted` | +| **Zero-allocation logging** | `[LoggerMessage]` source generator | + +See [full README](src/Sa.Utils.WorkQueue/Readme.md). + +--- + +## Samples -`Sa.Schedule` provides a way to configure and execute tasks on a schedule. +Located in `src/Samples/`: -- It allows you to manage a set of tasks that will be executed at specific times or at defined intervals. -- You can start and stop tasks. -- Define failure strategies: close the application, stop job, stop all jobs, or ignore the failure. +| Sample | Description | +|--------|-------------| +| [Configuration.Web](src/Samples/Configuration.Web) | CLI args + secrets in ASP.NET | +| [FFmpeg.Console](src/Samples/FFmpeg.Console) | FFmpeg metadata extraction | +| [HybridFileStorage.Console](src/Samples/HybridFileStorage.Console) | Multi-provider file storage | +| [Partitional.ConsoleApp](src/Samples/Partitional.ConsoleApp) | Declarative partitioning | +| [PgOutbox.ConsoleApp](src/Samples/PgOutbox.ConsoleApp) | Outbox pattern demo | +| [Schedule.Console](src/Samples/Schedule.Console) | Scheduled task executor | +| [Storage.Tests](src/Samples/Storage.Tests) | Hybrid file storage tests | -## [Sa.HybridFileStorage](src/Sa.HybridFileStorage) +--- -`IHybridFileStorage` - interface designed for hybrid file storage systems that facilitates the management of file operations, ensuring reliable and resilient access to files across multiple storage providers. +## Tests -- Supports file operations such as uploading, downloading, and deleting files. -- Integrates multiple storage providers (e.g., file system, s3, PostgreSQL) for enhanced reliability. -- Automatically switches between providers in case one becomes unavailable, ensuring continuous access to files. -- Promotes improved resilience and availability of file data in applications requiring dependable storage management. +Located in `src/Tests/`: 15 test projects using **xunit v3** and **Testcontainers** (PostgreSQL + Minio) for integration tests. -## [Sa.Configuration](src/Sa.Configuration) +--- -- `Arguments` class parses command-line arguments in a C# application, enabling easy retrieval of parameter values through a dictionary-like interface. It supports both single-value and multi-value parameters for flexible command-line configurations. -- `Secrets` class securely manages sensitive information, such as API keys and database passwords, from various sources. It can load secrets from files, environment variables, and dynamically generated host key files. +## Building -## [Sa.Configuration.PostgreSql](src/Sa.Configuration.PostgreSql) +```powershell +# Full build +.\build\do_build.ps1 -`AddPostgreSqlConfiguration` extension method allows you to add a PostgreSQL-based configuration source to an IConfigurationBuilder. +# Run tests +.\build\do_test.ps1 -- This setup allows for dynamic configuration management, where changes in the database can be reflected in the application without needing to recompile or redeploy. +# Package NuGet packages +.\build\do_package.ps1 +``` -## [Sa.Media](src/Sa.Media) +Direct dotnet commands: -- `AsyncWavReader` async and memory-efficient WAV file reader for .NET +```powershell +dotnet restore src/Sa.slnx -c Release +dotnet build src/Sa.slnx -c Release -v n +dotnet test src/Sa.slnx -v n +``` -## [Sa.Media.FFmpeg](src/Sa.Media.FFmpeg) +--- -FFmpeg .NET Wrapper - ready to use out of the box with minimal setup +## Architecture -- Extract metadata from media files (duration, channels, sample rate, etc.) -- Convert audio to: WAV, MP3, MP4, OGG .. -- Splits/Join audio file by channels -- Built-in FFmpeg binaries for Windows x64 and Linux -- Supports Dependency Injection (DI) via standard IServiceCollection integration +- Targets **.NET 10.0** with **Native AOT** +- Uses **Central Package Management** (`Directory.Packages.props`) +- Shared utilities in **Sa** are linked into consuming projects +- All packages use SDK-style csproj with implicit usings, nullable, and analyzers +- Solution managed via `.slnx` -## Sa.Utils +## License -- [Sa.Utils.WorkQueue](src/Sa.Utils.WorkQueue) - async Queue with Concurrency Limiting \ No newline at end of file +MIT diff --git a/src/Sa.Configuration.PostgreSql/Readme-ru.md b/src/Sa.Configuration.PostgreSql/Readme-ru.md new file mode 100644 index 00000000..4fe8a000 --- /dev/null +++ b/src/Sa.Configuration.PostgreSql/Readme-ru.md @@ -0,0 +1,129 @@ +# Sa.Configuration.PostgreSql + +Динамический источник конфигурации для .NET, загружающий настройки из PostgreSQL. Изменения в БД применяются к работающему приложению без перезапуска — достаточно вызвать `Reload()` на `IConfigurationRoot`. + +--- + +## Возможности + +- **Живая конфигурация**: значения хранятся в БД и могут быть изменены во время выполнения +- **Параметризированные SQL-запросы**: поддержка `@named_parameters` через `NpgsqlParameter` +- **Автоматические повторы**: встроенная стратегия повторов (`PgRetryStrategy`) с детекцией транзитных ошибок Npgsql +- **Обрезка ключей/значений**: пробелы автоматически обрезаются и у ключей, и у значений +- **Безопасная обработка NULL**: `NULL` в БД → `null` в конфиге; пустая строка → `string.Empty` + +--- + +## Публичный API + +| Тип | Назначение | +|-----|-----------| +| `PostgreSqlConfigurationOptions` | Immutable record: `ConnectionString`, `SelectSql`, `Parameters` | +| `DatabaseConfigurationSource` | Реализация `IConfigurationSource` | +| `DatabaseConfigurationProvider` | `ConfigurationProvider`, загружающий пары ключ-значение из БД | +| `Setup.AddSaPostgreSqlConfiguration()` | Метод-расширение для `IConfigurationBuilder` | + +--- + +## Быстрый старт + +```csharp +using Sa.Configuration.PostgreSql; + +var builder = WebApplication.CreateBuilder(args); + +builder.Configuration.AddSaPostgreSqlConfiguration(new PostgreSqlConfigurationOptions( + ConnectionString: "Host=localhost;Database=myapp;Username=app;Password=secret", + SelectSql: "SELECT key, value FROM app_settings" +)); + +var app = builder.Build(); + +// Чтение настроек +var theme = app.Configuration["theme"]; // → "dark" +var lang = app.Configuration["language"]; // → "en" +``` + +--- + +## Параметризированные запросы + +Используйте `@parameters` для фильтрации по клиенту/арендатору: + +```csharp +builder.Configuration.AddSaPostgreSqlConfiguration(new PostgreSqlConfigurationOptions( + ConnectionString: "...", + SelectSql: "SELECT key, value FROM client_settings WHERE client_id = @client_id", + Parameters: [new NpgsqlParameter("client_id", "acme-corp")] +)); +``` + +--- + +## Обновления живой конфигурации + +Когда строки в таблице `app_settings` изменяются, приложение может подхватить новые значения: + +```csharp +// После изменения строк в базе данных: +((IConfigurationRoot)app.Configuration).Reload(); + +// Или вручную: +provider.Reload(); // DatabaseConfigurationProvider реализует IConfigurationProvider +``` + +--- + +## Поведение загрузки + +| Сценарий | Результат | +|----------|----------| +| Ключ пустой или состоит только из пробелов | Пропускается | +| Значение `NULL` в БД | Сохраняется как `null` | +| Значение пустая строка в БД | Сохраняется как `string.Empty` | +| Ошибка подключения | `InvalidOperationException` с оригинальным исключением как `InnerException` | + +--- + +## Схема таблицы + +Минимальная таблица, необходимая для провайдера: + +```sql +CREATE TABLE app_settings ( + key VARCHAR PRIMARY KEY, + value TEXT +); + +-- Пример данных +INSERT INTO app_settings (key, value) VALUES + ('theme', 'dark'), + ('language', 'en'), + ('debug_mode', ''); -- пустая строка +``` + +--- + +## Зависимости + +- `Microsoft.Extensions.Configuration` +- `Sa.Data.PostgreSql` (обёртка Npgsql с PgRetryStrategy и IPgDataSource) + +--- + +## Структура проекта + +``` +src/Sa.Configuration.PostgreSql/ +├── PostgreSqlConfigurationOptions.cs # Record опций +├── DatabaseConfigurationSource.cs # IConfigurationSource +├── DatabaseConfigurationProvider.cs # ConfigurationProvider + повторы +├── Setup.cs # Метод-расширение AddSaPostgreSqlConfiguration() +└── Readme.md # ← вы здесь +``` + +--- + +## Лицензия + +MIT diff --git a/src/Sa.Configuration.PostgreSql/Readme.md b/src/Sa.Configuration.PostgreSql/Readme.md index 00c1586a..b66b721b 100644 --- a/src/Sa.Configuration.PostgreSql/Readme.md +++ b/src/Sa.Configuration.PostgreSql/Readme.md @@ -6,7 +6,7 @@ A dynamic configuration source for .NET that loads settings from PostgreSQL. Cha - **Live configuration**: values are stored in the database and can be changed at runtime - **Parameterized SQL queries**: supports `@named_parameters` via `NpgsqlParameter` -- **Automatic retry**: built-in retry strategy (PgRetryStrategy) with detection of Npgsql transaction errors +- **Automatic retry**: built-in retry strategy (`PgRetryStrategy`) with detection of Npgsql transient errors - **Key/value trimming**: whitespace is automatically trimmed from both keys and values - **Safe NULL handling**: `NULL` in DB → `null` in config; empty string → `string.Empty` @@ -15,6 +15,8 @@ A dynamic configuration source for .NET that loads settings from PostgreSQL. Cha | Type | Purpose | |------|---------| | `PostgreSqlConfigurationOptions` | Immutable record: `ConnectionString`, `SelectSql`, `Parameters` | +| `DatabaseConfigurationSource` | `IConfigurationSource` implementation | +| `DatabaseConfigurationProvider` | `ConfigurationProvider` that loads key-value pairs from DB | | `Setup.AddSaPostgreSqlConfiguration()` | Extension method for `IConfigurationBuilder` | ## Quick Start diff --git a/src/Sa.Configuration/CommandLine/Readme-ru.md b/src/Sa.Configuration/CommandLine/Readme-ru.md new file mode 100644 index 00000000..0c35bb1f --- /dev/null +++ b/src/Sa.Configuration/CommandLine/Readme-ru.md @@ -0,0 +1,208 @@ +# Arguments — Парсинг аргументов командной строки + +Парсинг и потребление аргументов командной строки в .NET-приложениях через простой dictionary-like API. Поддерживает форматы `--flag=value`, `--flag value`, короткие опции (`-x`) и типизированные геттеры. + +> **Важно:** парсер удаляет ведущие тире из имён параметров. При обращении к значению используйте ключ **без** лидирующих `-` или `--`. +> Пример: `--config_db` в CLI → `args["config_db"]` в коде. `-v` в CLI → `args["v"]` в коде. + +## Быстрый старт + +```csharp +using Sa.Configuration.CommandLine; + +// Парсим args (по умолчанию берёт Environment.GetCommandLineArgs()) +var args = new Arguments(args); + +// Доступ через индексатор (возвращает null, если ключ отсутствует) — ключи без ведущих тире +string? db = args["config_db"]; +string? file = args["config_file"]; +bool debug = args.IsPresent("debug"); // true если флаг присутствует и истинен + +// Типизированные помощники (возвращают nullable, null при отсутствии/невалидности) +int? port = args.GetInt("port"); +float? timeout = args.GetFloat("timeout"); +long offset = args.GetLong("offset"); +TimeSpan ttl = args.GetTimeSpan("ttl"); +bool verbose = args.GetBool("v"); // "true"/"1"/"yes"/"on" → true +``` + +### Минимальное консольное приложение + +```csharp +using Sa.Configuration.CommandLine; + +var arguments = new Arguments(args); + +Console.WriteLine($"БД: {arguments["db"] ?? "(по умолчанию)"}"); +Console.WriteLine($"Порт: {arguments.GetInt("port") ?? 5432}"); +Console.WriteLine($"Debug: {arguments.IsPresent("debug")}"); +Console.WriteLine($"TTL: {arguments.GetTimeSpan("ttl") ?? TimeSpan.Zero}"); +``` + +Запуск: + +```bash +dotnet run -- --db mydb --port 9999 --debug --ttl 00:05:00 -v +``` + +Вывод: + +``` +БД: mydb +Порт: 9999 +Debug: True +TTL: 00:05:00 +``` + +--- + +## Поддерживаемые форматы + +| Формат | Ввод в CLI | Ключ в словаре | Значение | +|--------|-----------|---------------|---------| +| Длинный флаг + пробел | `--key value` | `"key"` | `"value"` | +| Равно | `--key=value` | `"key"` | `"value"` | +| Короткий флаг + пробел | `-k value` | `"k"` | `"value"` | +| Короткое равно | `-k=v` | `"k"` | `"v"` | +| Булев флаг | `--debug` | `"debug"` | `"true"` | +| Значение в кавычках | `--name "hello world"` | `"name"` | `"hello world"` | + +--- + +## Справочник API + +### Конструктор + +```csharp +public Arguments(params IReadOnlyList args) +``` + +Создаёт экземпляр из списка строк аргументов. + +### Статическая фабрика + +```csharp +public static Arguments CreateDefault(string[]? args = null) +``` + +Шорткат, который использует `Environment.GetCommandLineArgs()` когда `args` равен null. + +```csharp +var args = Arguments.CreateDefault(); // читает Process.GetCurrentProcess().CommandLine +``` + +### Индексатор + +```csharp +public string? this[string param] { get; } +``` + +Возвращает значение по имени параметра или `null`, если не найдено. Ключи хранятся без ведущих тире. + +```csharp +var db = args["database"]; // null если --database никогда не передавали +``` + +### Contains / IsPresent + +```csharp +public bool Contains(string param) // true если ключ существует (даже если значение пустое) +public bool IsPresent(string param) // true если ключ существует И значение не null +``` + +`IsPresent` различает отсутствующий флаг и присутствующий, но пустой. + +### Типизированные геттеры + +Все возвращают `T?` (nullable) и дают `null` когда параметр отсутствует или не распарсивается. + +| Метод | Тип возврата | Пример | +|-------|-------------|--------| +| `GetBool(string)` | `bool?` | `args.GetBool("verbose")` — принимает `true/1/yes/on` | +| `GetInt(string)` | `int?` | `args.GetInt("port")` | +| `GetFloat(string)`| `float?`| `args.GetFloat("ratio")` | +| `GetLong(string)` | `long?` | `args.GetLong("offset")` | +| `GetTimeSpan(string)` | `TimeSpan?` | `args.GetTimeSpan("delay")` | + +Все числовые парсинги используют `CultureInfo.InvariantCulture`. + +### Исходные параметры + +```csharp +public IReadOnlyDictionary Parameters { get; } +``` + +Возвращает полный словарь распарсенных параметров. Ключи хранятся без ведущих тире. + +--- + +## Интеграция с Microsoft.Extensions.Configuration + +Регистрация аргументов командной строки как источника `IConfiguration`: + +```csharp +using Microsoft.Extensions.Configuration; +using Sa.Configuration.CommandLine; + +var configuration = new ConfigurationBuilder() + .AddSaCommandLine(args) // <-- добавляет CLI аргументы как источник конфига + .AddJsonFile("appsettings.json", optional: true) + .Build(); + +// Доступ через индексатор IConfiguration — ключи тоже без тире +var db = configuration["db"]; +var port = configuration["port"]; +``` + +Порядок важен: источники, зарегистрированные **позже**, переопределяют ранние. Размещайте `AddSaCommandLine` перед JSON/файловыми источниками, если хотите, чтобы CLI имел приоритет: + +```csharp +new ConfigurationBuilder() + .AddJsonFile("appsettings.json") // базовые значения по умолчанию + .AddSaCommandLine(args) // переопределения из CLI + .Build(); +``` + +--- + +## Запуск приложения + +### Из терминала + +```bash +# Длинные флаги с разделителем пробелом +dotnet run --project MyApp.dll --db production --port 5432 --debug + +# Синтаксис с равно +dotnet run --project MyApp.dll --db=production --port=5432 + +# Смешанные форматы +dotnet run -- -d --db=prod -p 3306 --ttl 30s +``` + +### Из Visual Studio / VS Code + +Установите аргументы в launchSettings.json: + +```json +{ + "profiles": { + "MyApp": { + "commandName": "Project", + "commandLineArgs": "--db test --port 9999 --debug --ttl 00:01:00" + } + } +} +``` + +--- + +## Краевые случаи + +| Ввод в CLI | Ключ в словаре | Значение | +|-----------|---------------|---------| +| `--flag` (без значения) | `"flag"` | `"true"` | +| `--flag=` (пустое) | `"flag"` | `""` | +| `--flag "quoted value"` | `"flag"` | `"quoted value"` | +| `-short=value` | `"short"` | `"value"` | +| Неизвестный формат | Игнорируется молча | — | diff --git a/src/Sa.Configuration/CommandLine/Readme.md b/src/Sa.Configuration/CommandLine/Readme.md index 8a497924..57ce9998 100644 --- a/src/Sa.Configuration/CommandLine/Readme.md +++ b/src/Sa.Configuration/CommandLine/Readme.md @@ -1,68 +1,208 @@ -# Arguments Class +# Arguments — Command-Line Parsing -The `Arguments` class provides a robust way to parse command-line arguments, making it easier to manage application configurations. It handles various parameter formats and provides helper methods for different data types. +Parse and consume command-line arguments in .NET apps with a simple dictionary-like API. Supports `--flag=value`, `--flag value`, short options (`-x`), and typed getters. -## Features +> **Key detail:** the parser strips leading dashes from parameter names. When you access a value, use the key **without** leading `-` or `--`. +> Example: `--config_db` in CLI → `args["config_db"]` in code. `-v` in CLI → `args["v"]` in code. -- Parse command-line arguments with support for various formats -- Access parameters by name using indexer syntax -- Helper methods for common data types (bool, int, float, long, TimeSpan) -- Support for both `--param=value` and `--param value` formats -- Support for short options like `-ip_override=127.0.0.1` +## Quick Start -## Usage - -### Basic Usage +```csharp +using Sa.Configuration.CommandLine; + +// Parse args (defaults to Environment.GetCommandLineArgs()) +var args = new Arguments(args); + +// Indexer access (returns null if absent) — keys have leading dashes stripped +string? db = args["config_db"]; +string? file = args["config_file"]; +bool debug = args.IsPresent("debug"); // true if flag present & truthy + +// Typed helpers (return nullable, null on missing/invalid) +int? port = args.GetInt("port"); +float? timeout = args.GetFloat("timeout"); +long offset = args.GetLong("offset"); +TimeSpan ttl = args.GetTimeSpan("ttl"); +bool verbose = args.GetBool("v"); // "true"/"1"/"yes"/"on" → true +``` -To use the Arguments class, create an instance and access parameters using the indexer syntax: +### Minimal Console App ```csharp -// Create an instance of the Arguments class, passing the command-line arguments +using Sa.Configuration.CommandLine; + var arguments = new Arguments(args); -// Retrieve values for specific parameters -string? configDb = arguments["--config_db"]; -string? configFile = arguments["--config_file"]; -string? configNLog = arguments["--config_nlog"]; -string? ipOverride = arguments["-ip_override"]; +Console.WriteLine($"DB: {arguments["db"] ?? "(default)"}"); +Console.WriteLine($"Port: {arguments.GetInt("port") ?? 5432}"); +Console.WriteLine($"Debug: {arguments.IsPresent("debug")}"); +Console.WriteLine($"TTL: {arguments.GetTimeSpan("ttl") ?? TimeSpan.Zero}"); +``` + +Run: + +```bash +dotnet run -- --db mydb --port 9999 --debug --ttl 00:05:00 -v +``` + +Output: -// Display the retrieved values -Console.WriteLine("Configuration Database: " + (configDb ?? "Not provided")); -Console.WriteLine("Configuration File: " + (configFile ?? "Not provided")); -Console.WriteLine("NLog Configuration: " + (configNLog ?? "Not provided")); -Console.WriteLine("IP Override: " + (ipOverride ?? "Not provided")); ``` +DB: mydb +Port: 9999 +Debug: True +TTL: 00:05:00 +``` + +--- + +## Supported Formats + +| Format | CLI Input | Dictionary Key | Value | +|--------|-----------|---------------|-------| +| Long flag + space | `--key value` | `"key"` | `"value"` | +| Equals sign | `--key=value` | `"key"` | `"value"` | +| Short flag + space | `-k value` | `"k"` | `"value"` | +| Short equals | `-k=v` | `"k"` | `"v"` | +| Boolean flag | `--debug` | `"debug"` | `"true"` | +| Quoted value | `--name "hello world"` | `"name"` | `"hello world"` | -### Advanced Usage +--- -The class also provides helper methods for different data types: +## API Reference + +### Constructor ```csharp -// Check if a parameter exists -if (arguments.Contains("--config_db")) -{ - // Get parameter as boolean - bool? nosjmp = arguments.GetBool("-nosjmp"); +public Arguments(params IReadOnlyList args) +``` - // Get parameter as integer - int? port = arguments.GetInt("--port"); +Creates an instance from a list of argument strings. - // Get parameter as float - float? timeout = arguments.GetFloat("--timeout"); +### Static Factory - // Get parameter as TimeSpan - TimeSpan? duration = arguments.GetTimeSpan("--duration"); -} +```csharp +public static Arguments CreateDefault(string[]? args = null) +``` + +Shortcut that uses `Environment.GetCommandLineArgs()` when `args` is null. + +```csharp +var args = Arguments.CreateDefault(); // reads Process.GetCurrentProcess().CommandLine +``` + +### Indexer + +```csharp +public string? this[string param] { get; } +``` + +Returns the value for a parameter name, or `null` if not found. Keys are stored without leading dashes. + +```csharp +var db = args["database"]; // null if --database was never passed +``` + +### Contains / IsPresent + +```csharp +public bool Contains(string param) // true if key exists (even if value is empty) +public bool IsPresent(string param) // true if key exists AND value is non-null ``` +`IsPresent` distinguishes between a missing flag and a present-but-empty flag. + +### Typed Getters + +All return `T?` (nullable) and yield `null` when the parameter is absent or unparsable. + +| Method | Return Type | Example | +|--------|-------------|---------| +| `GetBool(string)` | `bool?` | `args.GetBool("verbose")` — accepts `true/1/yes/on` | +| `GetInt(string)` | `int?` | `args.GetInt("port")` | +| `GetFloat(string)`| `float?`| `args.GetFloat("ratio")` | +| `GetLong(string)` | `long?` | `args.GetLong("offset")` | +| `GetTimeSpan(string)` | `TimeSpan?` | `args.GetTimeSpan("delay")` | + +All numeric parsing uses `CultureInfo.InvariantCulture`. + +### Raw Parameters + +```csharp +public IReadOnlyDictionary Parameters { get; } +``` + +Returns the full dictionary of parsed parameters. Keys are stored without leading dashes. + +--- + +## Integration with Microsoft.Extensions.Configuration + +Register command-line arguments as an `IConfiguration` source: + +```csharp +using Microsoft.Extensions.Configuration; +using Sa.Configuration.CommandLine; + +var configuration = new ConfigurationBuilder() + .AddSaCommandLine(args) // <-- adds CLI args as config source + .AddJsonFile("appsettings.json", optional: true) + .Build(); + +// Access via IConfiguration indexer — keys still have dashes stripped +var db = configuration["db"]; +var port = configuration["port"]; +``` + +Order matters: sources registered **later** override earlier ones. Place `AddSaCommandLine` before JSON/file sources if you want CLI to win: + +```csharp +new ConfigurationBuilder() + .AddJsonFile("appsettings.json") // base defaults + .AddSaCommandLine(args) // overrides from CLI + .Build(); +``` + +--- + ## Running the Application -To run the application with command-line arguments, you can use the command line or terminal. Here are examples: +### From terminal ```bash -# Standard usage -dotnet run Some.exe --config_db /opt/service_configs/config_db.json --config_file /opt/service_configs/appsettings.json --config_nlog /opt/service_configs/NLog.config -ip_override=127.0.0.1 -nosjmp +# Long flags with space separator +dotnet run --project MyApp.dll --db production --port 5432 --debug + +# Equals syntax +dotnet run --project MyApp.dll --db=production --port=5432 + +# Mixed formats +dotnet run -- -d --db=prod -p 3306 --ttl 30s +``` + +### From Visual Studio / VS Code + +Set arguments in launchSettings.json: + +```json +{ + "profiles": { + "MyApp": { + "commandName": "Project", + "commandLineArgs": "--db test --port 9999 --debug --ttl 00:01:00" + } + } +} +``` + +--- + +## Edge Cases -# Alternative format -dotnet run Some.exe --config_db=/opt/service_configs/config_db.json --config_file=/opt/service_configs/appsettings.json --config_nlog=/opt/service_configs/NLog.config -ip_override="127.0.0.1" -nosjmp -``` \ No newline at end of file +| CLI Input | Dictionary Key | Value | +|-----------|---------------|-------| +| `--flag` (no value) | `"flag"` | `"true"` | +| `--flag=` (empty) | `"flag"` | `""` | +| `--flag "quoted value"` | `"flag"` | `"quoted value"` | +| `-short=value` | `"short"` | `"value"` | +| Unknown format | Ignored silently | — | diff --git a/src/Sa.Configuration/Readme-ru.md b/src/Sa.Configuration/Readme-ru.md new file mode 100644 index 00000000..af5f7f3e --- /dev/null +++ b/src/Sa.Configuration/Readme-ru.md @@ -0,0 +1,284 @@ +# Sa.Configuration + +Безопасное управление секретами и парсер командной строки в экосистеме .NET `Microsoft.Extensions.Configuration`. Секреты автоматически подставляются в конфигурацию без ручного кода приложения. + +--- + +## Возможности + +- **Автоматическая подстановка секретов**: плейсхолдеры `{{key}}` заменяются реальными значениями из файлов, переменных окружения или аргументов командной строки +- **Защита от циклов**: встроенная защита от бесконечной рекурсии при разрешении плейсхолдеров +- **Опциональные плейсхолдеры**: `{{?key}}` — если секрет не найден, возвращается `null` вместо исключения +- **Цепочка хранилищ**: несколько источников секретов с приоритетным порядком +- **Парсер аргументов**: поддерживает форматы `--key value`, `--key=value`, `-flag` +- **Среды разработки**: автоматическая загрузка `secrets.{Environment}.txt` (Development/Staging/Production) + +--- + +## Быстрый старт + +### 1. Регистрация в `Program.cs` + +```csharp +using Sa.Configuration; + +var builder = WebApplication.CreateBuilder(args); + +// Подключение аргументов + секретов из файлов/ENV/CLI +builder.Configuration.AddSaConfiguration(); + +var app = builder.Build(); +``` + +### 2. Файл секретов (`secrets.txt`) + +```ini +# Postgres +sa_pg_host=localhost +sa_pg_user=postgres +sa_pg_port=5432 +sa_pg_database=myapp +sa_pg_schema=public +sa_pg_password=superSecret123 + +# API ключи +api_key=abc123xyz +jwt_secret=h8k2m9p0 +``` + +> ⚠️ Добавьте `secrets*.txt` в `.gitignore`! + +### 3. Плейсхолдеры в `appsettings.json` + +```json +{ + "secret": "{{sa_secret}}", + + "sa": { + "pg": { + "connection": "User ID={{sa_pg_user}};Password={{sa_pg_password}};Host={{sa_pg_host}};Port={{sa_pg_port}};Database={{sa_pg_database}};Pooling=true;SearchPath={{sa_pg_schema}};Command Timeout=180;" + } + }, + + "ExternalApi": { + "ApiKey": "{{api_key}}" + } +} +``` + +### 4. Чтение конфигурации + +```csharp +var pgConn = app.Configuration["sa:pg:connection"]; +// → "User ID=postgres;Password=superSecret123;Host=localhost;..." +``` + +--- + +## Приоритет секретов + +Секреты ищутся в порядке убывания приоритета: + +| # | Источник | Пример файла | +|---|----------|-------------| +| 1 | Базовый файл секретов | `secrets.txt` | +| 2 | Файл конкретной среды | `secrets.Development.txt` | +| 3 | Переменные окружения | `SA_PG_PASSWORD=...` | +| 4 | Аргументы командной строки | `--sa_pg_password=...` | + +Первый источник, имеющий значение, побеждает. Это позволяет переопределять секреты для каждой среды. + +--- + +## Опциональные плейсхолдеры + +Используйте `{{?key}}` вместо `{{key}}`, чтобы избежать ошибки при отсутствии секрета: + +```json +{ + "optional_feature": "{{?feature_flag}}" +} +``` + +Если `feature_flag` не найден ни в одном хранилище, возвращается `null`. + +--- + +## Использование с Sa.Configuration.PostgreSql + +```csharp +using Sa.Configuration; +using Sa.Configuration.PostgreSql; + +var builder = WebApplication.CreateBuilder(args); + +// Сначала стандартные источники (appsettings.json, secrets.txt) +builder.Configuration.AddSaConfiguration(); + +// Затем динамические настройки из базы данных +builder.Configuration.AddSaPostgreSqlConfiguration(new PostgreSqlConfigurationOptions( + ConnectionString: "...", + SelectSql: "SELECT key, value FROM app_settings" +)); + +var app = builder.Build(); +``` + +--- + +## Аргументы — Парсер командной строки + +```csharp +using Sa.Configuration.CommandLine; + +// some.exe --config_db /share/data.db --debug +var args = new Arguments(args); + +string? configDb = args["config_db"]; // → "/share/data.db" +bool? debug = args.GetBool("debug"); // → true +int? port = args.GetInt("port"); // → null +TimeSpan? timeout = args.GetTimeSpan("timeout"); +``` + +Поддерживаемые форматы: + +``` +--key value +--key=value +-key value +-key=value +-flag → flag=true (булев флаг) +``` + +Типизированные методы возвращают `null`, когда параметр отсутствует или невалиден: + +| Метод | Возвращаемый тип | Преобразование | +|-------|-----------------|----------------| +| `GetBool()` | `bool?` | `"true"/"1"/"yes"/"on"` → `true` | +| `GetInt()` | `int?` | `int.TryParse(..., InvariantCulture)` | +| `GetFloat()` | `float?` | то же самое | +| `GetLong()` | `long?` | то же самое | +| `GetTimeSpan()` | `TimeSpan?` | `TimeSpan.TryParse(..., InvariantCulture)` | + +Дополнительные методы: + +| Метод | Возвращаемый тип | Описание | +|-------|-----------------|---------| +| `Contains(param)` | `bool` | Проверяет наличие параметра | +| `IsPresent(param)` | `bool` | Параметр существует И имеет непустое значение | + +--- + +## Секреты — Управление секретами + +### Создание по умолчанию + +```csharp +using Sa.Configuration.SecretStore; + +// Стандартная цепочка: File → File.Env → EnvVar → CommandLine +var secrets = Secrets.CreateDefault(); +``` + +### Пользовательская цепочка + +```csharp +var secrets = new Secrets( + new FileSecretStore("my-secrets.txt"), + new EnvironmentVariableSecretStore(), + new InMemorySecretStore(new Dictionary { + { "override_key", "override_value" } + }) +); +``` + +### Добавление на лету + +```csharp +secrets.AddStore(new FileSecretStore("additional-secrets.txt")); +``` + +### Подстановка плейсхолдеров + +```csharp +string template = "Server={{host}};Password={{password}}"; +string result = secrets.PopulateSecrets(template); +// → "Server=localhost;Password=s3cret!" +``` + +### Получение одного секрета + +```csharp +string? password = secrets.GetSecret("sa_pg_password"); +``` + +### Определение имени среды + +```csharp +string env = Secrets.GetEnvironmentName(); +// → "Development", "Staging", "Production" и т.д. +``` + +--- + +## Публичный API + +### Пространство имён `Sa.Configuration` + +| Тип | Назначение | +|-----|-----------| +| `Setup.AddSaConfiguration()` | Главная точка входа: подключение аргументов + обработка секретов | + +### Пространство имён `Sa.Configuration.CommandLine` + +| Тип | Назначение | +|-----|-----------| +| `Arguments` | Парсер аргументов командной строки | +| `Arguments.CreateDefault()` | Создаёт из `Environment.GetCommandLineArgs()` | +| `Setup.AddSaCommandLine()` | Метод-расширение для `IConfigurationBuilder` | + +### Пространство имён `Sa.Configuration.SecretStore` + +| Тип | Назначение | +|-----|-----------| +| `Secrets` | Основной класс управления секретами, реализует `ISecretService` | +| `Secrets.CreateDefault()` | Стандартная цепочка хранилищ | +| `Secrets.GetEnvironmentName()` | Определяет среду (`DOTNET_ENVIRONMENT` / `ASPNETCORE_ENVIRONMENT`) | +| `SecretOptions` | Опции для `CreateDefault()`: `FileName`, `Args`, `EnvironmentName` | +| `ISecretService` | Интерфейс: `PopulateSecrets()` + `GetSecret()` | +| `ISecretStore` | Интерфейс: `GetSecret(string key)` | +| `Setup.AddSaPostSecretProcessing()` | Метод-расширение: применяет `ISecretService` к конфигу ПОСЛЕ загрузки других источников | + +### Хранилища секретов (`Sa.Configuration.SecretStore.Stories`) + +| Класс | Описание | +|-------|---------| +| `FileSecretStore` | Загружает `key=value` из текстового файла (пропускает комментарии `#`) | +| `EnvironmentVariableSecretStore` | Читает из `Environment.GetEnvironmentVariable()` | +| `CommandLineArgsSecretStore` | Берёт секреты из `Arguments` | +| `InMemorySecretStore` | Словарь в памяти, fluent `.AddSecret()` | + +--- + +## Как это работает + +``` +┌──────────────────────────────────────────────────────┐ +│ 1. appsettings.json содержит: │ +│ "connection": "Host={{sa_pg_host}};Password={{...}}"│ +├──────────────────────────────────────────────────────┤ +│ 2. secrets.txt содержит: │ +│ sa_pg_host=localhost │ +│ sa_pg_password=s3cret! │ +├──────────────────────────────────────────────────────┤ +│ 3. AddSaPostSecretProcessing подставляет плейсхолдеры:│ +│ IConfiguration["sa:pg:connection"] │ +│ → "Host=localhost;Password=s3cret!;..." │ +└──────────────────────────────────────────────────────┘ +``` + +--- + +## Лицензия + +MIT diff --git a/src/Sa.Configuration/Readme.md b/src/Sa.Configuration/Readme.md index 5f5ea7b2..bef68115 100644 --- a/src/Sa.Configuration/Readme.md +++ b/src/Sa.Configuration/Readme.md @@ -2,6 +2,8 @@ Secure secrets management and command-line argument parsing within the .NET `Microsoft.Extensions.Configuration` ecosystem. Secrets are automatically substituted into configuration without manual application code. +--- + ## Features - **Automatic secret substitution**: `{{key}}` placeholders are replaced with real values from files, environment variables, or command-line arguments @@ -11,6 +13,8 @@ Secure secrets management and command-line argument parsing within the .NET `Mic - **Argument parser**: supports `--key value`, `--key=value`, `-flag` formats - **Environments**: automatic loading of `secrets.{Environment}.txt` (Development/Staging/Production) +--- + ## Quick Start ### 1. Register in `Program.cs` @@ -69,6 +73,8 @@ var pgConn = app.Configuration["sa:pg:connection"]; // → "User ID=postgres;Password=superSecret123;Host=localhost;..." ``` +--- + ## Secret Priority Order Secrets are looked up in descending priority order: @@ -82,6 +88,8 @@ Secrets are looked up in descending priority order: The first source that has a value wins. This allows overriding secrets per environment. +--- + ## Optional Placeholders Use `{{?key}}` instead of `{{key}}` to avoid an error when a secret is missing: @@ -94,6 +102,8 @@ Use `{{?key}}` instead of `{{key}}` to avoid an error when a secret is missing: If `feature_flag` is not found in any store, `null` is returned. +--- + ## Usage with Sa.Configuration.PostgreSql ```csharp @@ -114,6 +124,8 @@ builder.Configuration.AddSaPostgreSqlConfiguration(new PostgreSqlConfigurationOp var app = builder.Build(); ``` +--- + ## Arguments — Command-Line Argument Parser ```csharp @@ -148,6 +160,15 @@ Typed methods return `null` when the parameter is absent or invalid: | `GetLong()` | `long?` | same as above | | `GetTimeSpan()` | `TimeSpan?` | `TimeSpan.TryParse(..., InvariantCulture)` | +Additional methods: + +| Method | Return Type | Description | +|--------|------------|-------------| +| `Contains(param)` | `bool` | Checks if parameter exists | +| `IsPresent(param)` | `bool` | Parameter exists AND has a non-null value | + +--- + ## Secrets — Secrets Management ### Creating Defaults @@ -171,6 +192,12 @@ var secrets = new Secrets( ); ``` +### Fluent Addition at Runtime + +```csharp +secrets.AddStore(new FileSecretStore("additional-secrets.txt")); +``` + ### Placeholder Substitution ```csharp @@ -185,6 +212,15 @@ string result = secrets.PopulateSecrets(template); string? password = secrets.GetSecret("sa_pg_password"); ``` +### Environment Name Resolution + +```csharp +string env = Secrets.GetEnvironmentName(); +// → "Development", "Staging", "Production", etc. +``` + +--- + ## Public API ### Namespace `Sa.Configuration` @@ -222,6 +258,8 @@ string? password = secrets.GetSecret("sa_pg_password"); | `CommandLineArgsSecretStore` | Pulls secrets from `Arguments` | | `InMemorySecretStore` | Dictionary in memory, fluent `.AddSecret()` | +--- + ## How It Works ``` @@ -239,32 +277,8 @@ string? password = secrets.GetSecret("sa_pg_password"); └──────────────────────────────────────────────────────┘ ``` -## Project Layout +--- -``` -src/Sa.Configuration/ -├── Setup.cs # AddSaConfiguration() -├── CommandLine/ -│ ├── Arguments.cs # Argument parser -│ ├── Arguments.partial.cs # Typed GetXxx() methods -│ ├── ArgumentsConfigurationProvider.cs # IConfigurationProvider -│ └── Setup.cs # AddSaCommandLine() -├── SecretStore/ -│ ├── Secrets.cs # Secrets management -│ ├── SecretOptions.cs # CreateDefault() options -│ ├── ISecretService.cs # Service interface -│ ├── ISecretStore.cs # Store interface -│ ├── Engine/ -│ │ ├── ChainedSecretStore.cs # Store stacking -│ │ ├── ChainedSecrets.cs # Chained + Service -│ │ └── SecretService.cs # Placeholder substitution -│ ├── Stories/ -│ │ ├── FileSecretStore.cs # Text file -│ │ ├── EnvironmentVariableSecretStore.cs # ENV vars -│ │ ├── CommandLineArgsSecretStore.cs # Args parser -│ │ └── InMemorySecretStore.cs # Dictionary in memory -│ ├── PostSecretProcessingConfigurationProvider.cs # IConfigurationProvider -│ ├── PostSecretProcessingConfigurationSource.cs # IConfigurationSource -│ └── Setup.cs # AddSaPostSecretProcessing() -└── Readme.md # ← you are here -``` +## License + +MIT diff --git a/src/Sa.Data.PostgreSql/Readme-ru.md b/src/Sa.Data.PostgreSql/Readme-ru.md new file mode 100644 index 00000000..ff4be082 --- /dev/null +++ b/src/Sa.Data.PostgreSql/Readme-ru.md @@ -0,0 +1,188 @@ +# Sa.Data.PostgreSql + +Лёгкая обёртка над Npgsql для типичных операций с PostgreSQL — без ORM overhead, с поддержкой DI, AOT и минимальными аллокациями. + +--- + +## Быстрый старт + +```csharp +// Вариант 1: прямое создание +var dataSource = IPgDataSource.Create("Host=db;Database=mydb;Username=usr;Password=pwd"); + +// Вариант 2: через DI +services.AddSaPostgreSqlDataSource(b => b.WithConnectionString("Host=db;Database=mydb;Username=usr;Password=pwd")); +// или с factory (например, из IConfiguration): +services.AddSaPostgreSqlDataSource(b => b.WithConnectionString(sp => + sp.GetRequiredService().GetConnectionString("Default"))); +``` + +--- + +## ExecuteNonQuery + +Выполняет SQL-команду, которая не возвращает данные (INSERT / UPDATE / DELETE / DDL), и возвращает число затронутых строк. + +```csharp +// Простой запрос +int affected = await dataSource.ExecuteNonQuery("DELETE FROM sessions WHERE expired = true"); + +// С параметрами +int affected = await dataSource.ExecuteNonQuery(""" + INSERT INTO users (name, age) VALUES (@p0, @p1); + """, [ + new NpgsqlParameter { ParameterName = "p0", Value = "Tom" }, + new NpgsqlParameter { ParameterName = "p1", Value = 18 } + ]); +``` + +--- + +## ExecuteScalar / ExecuteScalarTyped + +Возвращает первое значение первой строки результата. `ExecuteScalarTyped` автоматически кастует результат, включая поддержку `Guid`, `DateTime`, `DateTimeOffset` и `DateOnly → DateTime`. + +```csharp +// object? перегрузка +object? count = await dataSource.ExecuteScalar("SELECT COUNT(*) FROM users"); + +// Типизированная перегрузка +int count = await dataSource.ExecuteScalarTyped("SELECT COUNT(*) FROM users"); +long id = await dataSource.ExecuteScalarTyped("SELECT nextval('users_id_seq')"); +Guid tenantId = await dataSource.ExecuteScalarTyped("SELECT tenant_uuid FROM tenants LIMIT 1"); +``` + +--- + +## ExecuteReader + +Потоковое чтение строк с callback'ом — идеально для обработки больших результатов без загрузки в память. + +```csharp +int processed = 0; +await dataSource.ExecuteReader("SELECT id, name FROM users", (reader, rowIndex) => +{ + int id = reader.GetInt32(0); + string name = reader.GetString(1); + Console.WriteLine($"{rowIndex}: {id} → {name}"); + processed++; +}); +Console.WriteLine($"Processed {processed} rows"); +``` + +--- + +## ExecuteReaderList + +Читает все строки и собирает их в `List`. + +```csharp +// Простая проекция +var names = await dataSource.ExecuteReaderList( + "SELECT name FROM users ORDER BY name", + reader => reader.GetString(0)); + +// С параметрами +var activeUsers = await dataSource.ExecuteReaderList<(int Id, string Name)>( + """SELECT id, name FROM users WHERE active = @active ORDER BY name""", + reader => (reader.GetInt32(0), reader.GetString(1)), + [new NpgsqlParameter { ParameterName = "active", Value = true }]); +``` + +--- + +## ExecuteReaderFirst + +Возвращает первое значение из первого столбца первой строки. Возвращает `default(T)` если результат пуст. + +```csharp +string name = await dataSource.ExecuteReaderFirst("SELECT name FROM users WHERE id = 42"); +// → "Tom" или default(string) если не найдено +``` + +Поддерживаемые типы: `int`, `long`, `short`, `bool`, `double`, `decimal`, `char`, `string`, `DateTime`, `Guid`, `DateTimeOffset`. + +--- + +## BeginBinaryImport + +Быстрый бинарный импорт через COPY BINARY. + +```csharp +ulong imported = await dataSource.BeginBinaryImport( + "COPY users (name, email) FROM stdin BINARY", + async (writer, ct) => + { + foreach (var user in users) + { + await writer.StartRowAsync(ct); + await writer.WriteAsync(user.Name, ct); + await writer.WriteAsync(user.Email, ct); + } + return await writer.CompleteAsync(ct); + }, + cancellationToken); + +Console.WriteLine($"Imported {imported} rows"); +``` + +--- + +## PgRetryStrategy + +Повторы с jitter для транзитных ошибок Npgsql. + +```csharp +using Sa.Data.PostgreSql; + +// Автоматически повторяет при транзитных ошибках (сброс соединения, таймаут и т.д.) +var result = await PgRetryStrategy.ExecuteWithRetry( + async ct => + { + using var conn = await dataSource.OpenDbConnection(ct); + return await conn.OpenAsync(ct); + }, + retryCount: 5, + initialDelay: 530); +``` + +--- + +## DbCommandExtensions + INamePrefixProvider + +Оптимизированный API для добавления параметризированных команд с предварительно закэшированными именами параметров (минимальные аллокации). + +```csharp +// Объявите провайдер префиксов +public class UserParams : INamePrefixProvider +{ + public static string[] GetPrefixes() => ["name", "age", "email"]; + public static int MaxIndex => 10; +} + +// Используйте — имена генерируются как @name0, @name1, ..., @age0, ... +var cmd = new NpgsqlCommand("SELECT * FROM users WHERE name = @name0 AND age > @age0") + .AddParam("name", "Tom", 0) + .AddParam("age", 18, 0); +``` + +--- + +## Сравнение методов + +| Метод | Возврат | Когда использовать | +|-------|---------|-------------------| +| `ExecuteNonQuery` | `int` (строки) | INSERT / UPDATE / DELETE / DDL | +| `ExecuteScalar` | `object?` | Одиночное значение, нужна ручная casts | +| `ExecuteScalarTyped` | `T` | Одиночное значение с авто-кастомом (Guid, DateTime, DateTimeOffset, DateOnly) | +| `ExecuteReader` | `int` (строки) | Потоковая обработка, много строк | +| `ExecuteReaderList` | `List` | Маленький результат, собрать всё | +| `ExecuteReaderFirst` | `T` | Одна строка, одна колонка | +| `BeginBinaryImport` | `ulong` (строки) | Массовый импорт через COPY BINARY | +| `ExecuteTransactionAsync` | `void` | Атомарные операции с rollback | + +--- + +## Лицензия + +MIT diff --git a/src/Sa.Data.PostgreSql/Readme.md b/src/Sa.Data.PostgreSql/Readme.md index 46c1d088..6882926d 100644 --- a/src/Sa.Data.PostgreSql/Readme.md +++ b/src/Sa.Data.PostgreSql/Readme.md @@ -1,29 +1,33 @@ # Sa.Data.PostgreSql -Лёгкая обёртка над Npgsql для типичных операций с PostgreSQL — без ORM overhead, с поддержкой DI, AOT и минимальными аллокациями. +Lightweight Npgsql wrapper for common PostgreSQL operations — no ORM overhead, with DI, Native AOT support, and minimal allocations. -## Быстрый старт +--- + +## Quick Start ```csharp -// Вариант 1: прямой создание +// Option 1: direct creation var dataSource = IPgDataSource.Create("Host=db;Database=mydb;Username=usr;Password=pwd"); -// Вариант 2: через DI +// Option 2: via DI services.AddSaPostgreSqlDataSource(b => b.WithConnectionString("Host=db;Database=mydb;Username=usr;Password=pwd")); -// или с factory (например, из IConfiguration): +// or with factory (e.g., from IConfiguration): services.AddSaPostgreSqlDataSource(b => b.WithConnectionString(sp => sp.GetRequiredService().GetConnectionString("Default"))); ``` +--- + ## ExecuteNonQuery -Выполняет SQL-команду, которая не возвращает данные (INSERT / UPDATE / DELETE / DDL), и возвращает число затронутых строк. +Executes a SQL command that doesn't return data (INSERT / UPDATE / DELETE / DDL) and returns the number of affected rows. ```csharp -// Простой запрос +// Simple query int affected = await dataSource.ExecuteNonQuery("DELETE FROM sessions WHERE expired = true"); -// С параметрами +// With parameters int affected = await dataSource.ExecuteNonQuery(""" INSERT INTO users (name, age) VALUES (@p0, @p1); """, [ @@ -32,9 +36,11 @@ int affected = await dataSource.ExecuteNonQuery(""" ]); ``` +--- + ## ExecuteScalar / ExecuteScalarTyped -Возвращает первое значение первой строки результата. `ExecuteScalarTyped` автоматически кастует результат, включая поддержку `Guid`, `DateTime`, `DateTimeOffset` и `DateOnly → DateTime`. +Returns the first value of the first row in the result. `ExecuteScalarTyped` automatically casts the result, including support for `Guid`, `DateTime`, `DateTimeOffset`, and `DateOnly → DateTime`. ```csharp // object? overload @@ -46,9 +52,11 @@ long id = await dataSource.ExecuteScalarTyped("SELECT nextval('users_id_se Guid tenantId = await dataSource.ExecuteScalarTyped("SELECT tenant_uuid FROM tenants LIMIT 1"); ``` +--- + ## ExecuteReader -Потоковое чтение строк с callback'ом — идеально для обработки больших результатов без загрузки в память. +Streaming row reading with a callback — ideal for processing large results without loading into memory. ```csharp int processed = 0; @@ -62,57 +70,63 @@ await dataSource.ExecuteReader("SELECT id, name FROM users", (reader, rowIndex) Console.WriteLine($"Processed {processed} rows"); ``` +--- + ## ExecuteReaderList -Читает все строки и собирает их в `List`. +Reads all rows and collects them into `List`. ```csharp -// Простая проекция +// Simple projection var names = await dataSource.ExecuteReaderList( "SELECT name FROM users ORDER BY name", reader => reader.GetString(0)); -// С параметрами +// With parameters var activeUsers = await dataSource.ExecuteReaderList<(int Id, string Name)>( """SELECT id, name FROM users WHERE active = @active ORDER BY name""", reader => (reader.GetInt32(0), reader.GetString(1)), [new NpgsqlParameter { ParameterName = "active", Value = true }]); ``` +--- + ## ExecuteReaderFirst -Возвращает первое значение из первого столбца первой строки. Возвращает `default(T)` если результат пуст. +Returns the first value from the first column of the first row. Returns `default(T)` if the result is empty. -Поддерживаемые типы: `int`, `long`, `short`, `bool`, `double`, `decimal`, `char`, `string`, `DateTime`, `Guid`, `DateTimeOffset`. +Supported types: `int`, `long`, `short`, `bool`, `double`, `decimal`, `char`, `string`, `DateTime`, `Guid`, `DateTimeOffset`. ```csharp -// Вернёт 0 если таблица пуста +// Returns 0 if the table is empty int errorCount = await dataSource.ExecuteReaderFirst( "SELECT COUNT(*) FROM outbox_errors"); -// Guid — работает автоматически +// Guid — automatic casting works Guid firstTenantId = await dataSource.ExecuteReaderFirst( "SELECT tenant_id FROM tenants LIMIT 1"); ``` +--- + ## ExecuteTransactionAsync -Атомарная транзакция с автоматическим rollback при ошибке. +Atomic transaction with automatic rollback on error. ```csharp await dataSource.ExecuteTransactionAsync(async (transaction, ct) => { - // Все команды внутри используют одну транзакцию + // All commands inside use one transaction await dataSource.ExecuteNonQuery( "INSERT INTO accounts (balance) VALUES (0)", ct); await dataSource.ExecuteNonQuery( "INSERT INTO transactions (account_id, amount) VALUES (1, 100)", ct); - // При успехе — авто-commit + // On success — auto-commit }, IsolationLevel.ReadCommitted, ct); -// При любом исключении — авто-rollback +// On any exception — auto-rollback try { await dataSource.ExecuteTransactionAsync(async (tx, ct) => @@ -122,13 +136,15 @@ try } catch (InvalidOperationException) { - // Транзакция откатилась автоматически + // Transaction rolled back automatically } ``` +--- + ## BeginBinaryImport -Быстрый бинарный импорт данных через COPY — в разы быстрее поштучных INSERT'ов. +Fast binary data import via COPY — orders of magnitude faster than individual INSERTs. ```csharp ulong imported = await dataSource.BeginBinaryImport( @@ -149,15 +165,16 @@ ulong imported = await dataSource.BeginBinaryImport( Console.WriteLine($"Imported {imported} rows"); ``` +--- ## PgRetryStrategy -Повтор попыток с jitter для transient-ошибок Npgsql. +Retry with jitter for transient Npgsql errors. ```csharp using Sa.Data.PostgreSql; -// Автоматически повторяет при transient-ошибках (connection reset, timeout и т.п.) +// Automatically retries on transient errors (connection reset, timeout, etc.) var result = await PgRetryStrategy.ExecuteWithRetry( async ct => { @@ -168,33 +185,43 @@ var result = await PgRetryStrategy.ExecuteWithRetry( initialDelay: 530); ``` +--- + ## DbCommandExtensions + INamePrefixProvider -Оптимизированный API для добавления параметризованных команд с пред-кэшированными именами параметров (минимальные аллокации). +Optimized API for adding parameterized commands with pre-cached parameter names (minimal allocations). ```csharp -// Объявите провайдер префиксов +// Declare a prefix provider public class UserParams : INamePrefixProvider { public static string[] GetPrefixes() => ["name", "age", "email"]; public static int MaxIndex => 10; } -// Используйте — имена генерируются как @name0, @name1, ..., @age0, ... +// Use — names are generated as @name0, @name1, ..., @age0, ... var cmd = new NpgsqlCommand("SELECT * FROM users WHERE name = @name0 AND age > @age0") .AddParam("name", "Tom", 0) .AddParam("age", 18, 0); ``` -## Сравнение методов - -| Метод | Возврат | Когда использовать | -|---|---|---| -| `ExecuteNonQuery` | `int` (строки) | INSERT / UPDATE / DELETE / DDL | -| `ExecuteScalar` | `object?` | Одно значение, нужна ручная конвертация | -| `ExecuteScalarTyped` | `T` | Одно значение с авто-кастом (Guid, DateTime, DateTimeOffset, DateOnly) | -| `ExecuteReader` | `int` (строки) | Потоковая обработка, много строк | -| `ExecuteReaderList` | `List` | Небольшой результат, нужно собрать всё | -| `ExecuteReaderFirst` | `T` | Одна строка одного столбца | -| `BeginBinaryImport` | `ulong` (строки) | Массовый импорт COPY BINARY | -| `ExecuteTransactionAsync` | `void` | Атомарные операции с rollback | +--- + +## Method Comparison + +| Method | Return | When to use | +|--------|--------|-------------| +| `ExecuteNonQuery` | `int` (rows) | INSERT / UPDATE / DELETE / DDL | +| `ExecuteScalar` | `object?` | Single value, manual cast needed | +| `ExecuteScalarTyped` | `T` | Single value with auto-cast (Guid, DateTime, DateTimeOffset, DateOnly) | +| `ExecuteReader` | `int` (rows) | Streaming processing, many rows | +| `ExecuteReaderList` | `List` | Small result, collect everything | +| `ExecuteReaderFirst` | `T` | One row, one column | +| `BeginBinaryImport` | `ulong` (rows) | Mass import via COPY BINARY | +| `ExecuteTransactionAsync` | `void` | Atomic operations with rollback | + +--- + +## License + +MIT diff --git a/src/Sa.Data.S3/README.md b/src/Sa.Data.S3/README.md index 5a6e220b..a88e5ca8 100644 --- a/src/Sa.Data.S3/README.md +++ b/src/Sa.Data.S3/README.md @@ -1,14 +1,18 @@ # Sa.Data.S3 -Обёртка над `HttpClient` для работы с S3-совместимыми хранилищами (Minio, AWS S3, DigitalOcean Spaces и др.). Полностью собственная реализация AWS Signature Version 4 — **без зависимостей от AWS SDK или Minio SDK**. +Wrapper over `HttpClient` for working with S3-compatible storage systems (Minio, AWS S3, DigitalOcean Spaces, etc.). Fully self-implemented AWS Signature Version 4 — **no dependencies on AWS SDK or Minio SDK**. -## Мотивация +--- -Это форк https://github.com/teoadal/Storage. Мотивация — клиенты [AWS SDK for .NET](https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/welcome.html) (4.x) и [Minio .NET](https://github.com/minio/minio-dotnet) (6.x) потребляли слишком много памяти. Результат: скорость почти как у AWS, а потребление памяти в ~150 раз меньше чем Minio SDK и в ~17 раз меньше AWS SDK. +## Motivation -## Создание клиента +This is a fork of https://github.com/teoadal/Storage. Motivation: the [AWS SDK for .NET](https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/welcome.html) (4.x) and [Minio .NET](https://github.com/minio/minio-dotnet) (6.x) clients consumed too much memory. Result: speed is comparable to AWS, while memory consumption is ~150× lower than Minio SDK and ~17× lower than AWS SDK. -### Без DI +--- + +## Creating a Client + +### Without DI ```csharp var client = new S3BucketClient(new HttpClient(), new S3BucketClientSetupSettings @@ -20,7 +24,7 @@ var client = new S3BucketClient(new HttpClient(), new S3BucketClientSetupSetting }); ``` -### С DI +### With DI ```csharp services.AddSaS3BucketClient(new S3BucketClientSetupSettings @@ -31,27 +35,31 @@ services.AddSaS3BucketClient(new S3BucketClientSetupSettings SecretKey = "ChangeMe123", TotalRequestTimeout = TimeSpan.FromSeconds(180), ConnectionPoolLifetime = TimeSpan.FromMinutes(15), - HandlerLifetime = Timeout.InfiniteTimeSpan // или TimeSpan.FromHours(2) для периодического обновления handler + HandlerLifetime = Timeout.InfiniteTimeSpan // or TimeSpan.FromHours(2) for periodic handler refresh }); -// Использование: +// Usage: var client = serviceProvider.GetRequiredService(); ``` -## Настройки - -| Свойство | Описание | По умолчанию | -|---|---|---| -| `AccessKey` | Ключ доступа S3 | *(обязательно)* | -| `SecretKey` | Секретный ключ S3 | *(обязательно)* | -| `Bucket` | Имя бакета | *(обязательно)* | -| `Endpoint` | URL S3-хранилища | *(обязательно)* | -| `Region` | Регион для SigV4 | `"us-east-1"` | -| `Service` | Сервис для SigV4 | `"s3"` | -| `UseHttp2` | Принудительный HTTP/2 | `false` | -| `TotalRequestTimeout` | Таймаут каждого запроса | `180 сек` | -| `ConnectionPoolLifetime` | Время жизни пула соединений | `15 мин` | -| `HandlerLifetime` | Время жизни HttpClient handler | `∞` (бесконечность) | +--- + +## Settings + +| Property | Description | Default | +|----------|-------------|---------| +| `AccessKey` | S3 access key | *(required)* | +| `SecretKey` | S3 secret key | *(required)* | +| `Bucket` | Bucket name | *(required)* | +| `Endpoint` | S3 storage URL | *(required)* | +| `Region` | Region for SigV4 | `"us-east-1"` | +| `Service` | Service name for SigV4 | `"s3"` | +| `UseHttp2` | Force HTTP/2 | `false` | +| `TotalRequestTimeout` | Per-request timeout | `180 sec` | +| `ConnectionPoolLifetime` | Connection pool lifetime | `15 min` | +| `HandlerLifetime` | HttpClient handler lifetime | `∞` (infinite) | + +--- ## API @@ -62,7 +70,7 @@ public interface IBucketOperations { Task CreateBucket(CancellationToken ct); Task DeleteBucket(CancellationToken ct); - Task DeleteBucket(bool forceDelete, CancellationToken ct); // force: удалить все объекты перед удалением bucket + Task DeleteBucket(bool forceDelete, CancellationToken ct); // force: delete all objects before removing bucket Task IsBucketExists(CancellationToken ct); } ``` @@ -79,22 +87,22 @@ public interface IFileOperations Task GetFileStream(string fileName, CancellationToken ct); Task GetFileUrl(string fileName, TimeSpan expiration, CancellationToken ct); Task IsFileExists(string fileName, CancellationToken ct); - IAsyncEnumerable List(string? prefix, CancellationToken ct); // с pagination + IAsyncEnumerable List(string? prefix, CancellationToken ct); // with pagination Task UploadFile(string fileName, string contentType, byte[] data, CancellationToken ct); - Task UploadFile(string fileName, string contentType, CancellationToken ct); // ручной multipart + Task UploadFile(string fileName, string contentType, CancellationToken ct); // manual multipart Task UploadFile(string fileName, string contentType, Stream data, CancellationToken ct); } ``` -### Rучной Multipart Upload +### Manual Multipart Upload -Для файлов > 5MB автоматически выбирается multipart upload. Для ручного управления: +For files > 5MB, multipart upload is selected automatically. For manual control: ```csharp using var uploader = await client.UploadFile("large-file.bin", "application/octet-stream", ct); uploader.AddPart(chunkData, ct); -uploader.AddPart(chunkData, offset, length, ct); // перегрузка с offset +uploader.AddPart(chunkData, offset, length, ct); // overload with offset uploader.AddParts(fullDataStream, ct); uploader.AddParts(fullByteArray, ct); @@ -108,12 +116,20 @@ else } ``` -## Особенности реализации +--- + +## Implementation Details + +- **AWS SigV4** — full manual implementation of request signing (SHA256 + HMAC-SHA256 chain) +- **ArrayPool.Shared** — buffer pooling to minimize GC pressure +- **ref struct ValueStringBuilder** — stack-based string builder with zero allocations +- **stackalloc** — wherever possible to avoid heap allocation +- **Buffered XML parser** — efficient reading of S3 responses (ListObjects, Multipart IDs) +- **Pagination** — automatic handling of `IsTruncated` / `NextContinuationToken` in `List()` +- **CancellationToken** — supported in all async operations + +--- + +## License -- **AWS SigV4** — полная ручная реализация подписывания запросов (SHA256 + HMAC-SHA256 chain) -- **ArrayPool.Shared** — пулинг буферов для минимизации GC pressure -- **ref struct ValueStringBuilder** — стек-based строковый билдер без аллокаций -- **stackalloc** — везде где возможно для избежания heap allocation -- **Буферизированный XML парсер** — эффективное чтение ответов S3 (ListObjects, Multipart IDs) -- **Pagination** — автоматическая обработка `IsTruncated` / `NextContinuationToken` в `List()` -- **CancellationToken** — поддерживается во всех async операциях +MIT diff --git a/src/Sa.Data.S3/Readme-ru.md b/src/Sa.Data.S3/Readme-ru.md new file mode 100644 index 00000000..2f949535 --- /dev/null +++ b/src/Sa.Data.S3/Readme-ru.md @@ -0,0 +1,135 @@ +# Sa.Data.S3 + +Обёртка над `HttpClient` для работы с S3-совместимыми хранилищами (Minio, AWS S3, DigitalOcean Spaces и др.). Полностью собственная реализация AWS Signature Version 4 — **без зависимостей от AWS SDK или Minio SDK**. + +--- + +## Мотивация + +Это форк https://github.com/teoadal/Storage. Мотивация: клиенты [AWS SDK for .NET](https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/welcome.html) (4.x) и [Minio .NET](https://github.com/minio/minio-dotnet) (6.x) потребляли слишком много памяти. Результат: скорость почти как у AWS, а потребление памяти в ~150 раз меньше Minio SDK и в ~17 раз меньше AWS SDK. + +--- + +## Создание клиента + +### Без DI + +```csharp +var client = new S3BucketClient(new HttpClient(), new S3BucketClientSetupSettings +{ + Bucket = "mybucket", + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123" +}); +``` + +### С DI + +```csharp +services.AddSaS3BucketClient(new S3BucketClientSetupSettings +{ + Bucket = "mybucket", + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123", + TotalRequestTimeout = TimeSpan.FromSeconds(180), + ConnectionPoolLifetime = TimeSpan.FromMinutes(15), + HandlerLifetime = Timeout.InfiniteTimeSpan // или TimeSpan.FromHours(2) для периодического обновления handler +}); + +// Использование: +var client = serviceProvider.GetRequiredService(); +``` + +--- + +## Настройки + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `AccessKey` | Ключ доступа S3 | *(обязательно)* | +| `SecretKey` | Секретный ключ S3 | *(обязательно)* | +| `Bucket` | Имя бакета | *(обязательно)* | +| `Endpoint` | URL S3-хранилища | *(обязательно)* | +| `Region` | Регион для SigV4 | `"us-east-1"` | +| `Service` | Сервис для SigV4 | `"s3"` | +| `UseHttp2` | Принудительный HTTP/2 | `false` | +| `TotalRequestTimeout` | Таймаут каждого запроса | `180 сек` | +| `ConnectionPoolLifetime` | Время жизни пула соединений | `15 мин` | +| `HandlerLifetime` | Время жизни HttpClient handler | `∞` (бесконечность) | + +--- + +## API + +### IBucketOperations + +```csharp +public interface IBucketOperations +{ + Task CreateBucket(CancellationToken ct); + Task DeleteBucket(CancellationToken ct); + Task DeleteBucket(bool forceDelete, CancellationToken ct); // force: удалить все объекты перед удалением bucket + Task IsBucketExists(CancellationToken ct); +} +``` + +### IFileOperations + +```csharp +public interface IFileOperations +{ + string BuildFileUrl(string fileName); + string BuildFileUrl(string fileName, TimeSpan expiration); + Task DeleteFile(string fileName, CancellationToken ct); + Task GetFile(string fileName, CancellationToken ct); + Task GetFileStream(string fileName, CancellationToken ct); + Task GetFileUrl(string fileName, TimeSpan expiration, CancellationToken ct); + Task IsFileExists(string fileName, CancellationToken ct); + IAsyncEnumerable List(string? prefix, CancellationToken ct); // с pagination + Task UploadFile(string fileName, string contentType, byte[] data, CancellationToken ct); + Task UploadFile(string fileName, string contentType, CancellationToken ct); // ручной multipart + Task UploadFile(string fileName, string contentType, Stream data, CancellationToken ct); +} +``` + +### Ручной Multipart Upload + +Для файлов > 5MB автоматически выбирается multipart upload. Для ручного управления: + +```csharp +using var uploader = await client.UploadFile("large-file.bin", "application/octet-stream", ct); + +uploader.AddPart(chunkData, ct); +uploader.AddPart(chunkData, offset, length, ct); // перегрузка с offset +uploader.AddParts(fullDataStream, ct); +uploader.AddParts(fullByteArray, ct); + +if (await uploader.Complete(ct)) +{ + Console.WriteLine($"Uploaded {uploader.Written} bytes"); +} +else +{ + await uploader.Abort(ct); +} +``` + +--- + +## Особенности реализации + +- **AWS SigV4** — полная ручная реализация подписывания запросов (SHA256 + HMAC-SHA256 chain) +- **ArrayPool.Shared** — пулинг буферов для минимизации GC pressure +- **ref struct ValueStringBuilder** — стек-based строковый билдер без аллокаций +- **stackalloc** — везде где возможно для избежания heap allocation +- **Буферизированный XML парсер** — эффективное чтение ответов S3 (ListObjects, Multipart IDs) +- **Pagination** — автоматическая обработка `IsTruncated` / `NextContinuationToken` в `List()` +- **CancellationToken** — поддерживается во всех async операциях + +--- + +## Лицензия + +MIT diff --git a/src/Sa.HybridFileStorage/Readme-ru.md b/src/Sa.HybridFileStorage/Readme-ru.md new file mode 100644 index 00000000..5fc77428 --- /dev/null +++ b/src/Sa.HybridFileStorage/Readme-ru.md @@ -0,0 +1,311 @@ +# Sa.HybridFileStorage + +Гибридная абстракция файлового хранилища с автоматическим переключением между провайдерами. Объединяет несколько бэкендов (FileSystem, S3, PostgreSQL) под единым устойчивым API — если один провайдер становится недоступен, система переключается на другой. + +--- + +## Поддерживаемые провайдеры + +| Провайдер | Класс | Сценарий использования | +|-----------|-------|----------------------| +| **Файловая система** | `FileSystemStorage` | Локальная разработка, on-premise развёртывания | +| **S3-совместимое** | `S3FileStorage` | Облачное хранилище (AWS S3, MinIO и др.) | +| **PostgreSQL** | `PostgresFileStorage` | Файлы внутри БД, транзакционная согласованность | +| **In-Memory** | `InMemoryFileStorage` | Тестирование, эфемерные сценарии | + +--- + +## Ключевые возможности + +- ✅ **Единый API** — Один интерфейс для всех провайдеров хранения +- ✅ **Изоляция Basket/Tenant** — Многопользовательская поддержка со scoped корзинами +- ✅ **Режим «только чтение»** — Защита от случайных модификаций +- ✅ **Потоковая передача** — Эффективная работа с памятью при передаче файлов +- ✅ **Native AOT готово** — Полная совместимость с .NET 10 Native AOT +- ✅ **Пакетные операции** — Эффективная массовая обработка файлов с параллелизмом +- ✅ **Перехватчики (Interceptors)** — Хуки жизненного цикла загрузки/скачивания/удаления + +--- + +## Формат File ID + +Все файлы идентифицируются через унифицированный URI-подобный формат: + +``` +{storageType}://{basket}/{tenantId}/{fileName} +``` + +**Примеры:** +- `s3://share/42/document.pdf` +- `fs://root/100/report.xlsx` +- `pg://files/7/1773210911/some/data.bin` +- `mem://share/42/temp.txt` + +--- + +## Быстрый старт + +### Без DI + +```csharp +using var memory = new InMemoryFileStorage(new InMemoryFileStorageOptions("share")); +var container = new HybridFileStorageContainer([memory]); +var storage = new HybridFileStorage(container, InterceptorContainer.Empty); + +var stream = "Hello, HybridFileStorage!".ToStream(); +var result = await storage.UploadAsync( + "share", + new UploadFileInput { FileName = "file.txt", TenantId = 42 }, + stream, + ct); + +await storage.DownloadAsync(result.FileId, async (fs, t) => +{ + var content = await fs.ToStrAsync(t); + Console.WriteLine(content); +}); +``` + +### С DI + +```csharp +builder.Services.AddSaHybridFileStorage(configure => configure + .AddStorage(InMemoryFileStorage.New("share")) + .AddLogging()); + +// Или регистрация отдельных провайдеров: +builder.Services.AddSaFileSystemFileStorage(new FileSystemStorageSettings +{ + BasePath = @"C:\data\files", + Basket = "documents" +}); + +builder.Services.AddSaS3FileStorage(new S3FileStorageOptions +{ + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123", + Bucket = "mybucket", + Basket = "uploads" +}); + +// Использование: +var storage = serviceProvider.GetRequiredService(); +``` + +--- + +## Настройки + +### FileSystemStorageSettings + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `BasePath` | Корневая директория для файлов | *(обязательно)* | +| `Basket` | Имя области хранения | `"share"` | +| `StorageType` | Префикс схемы в File ID | `"fs"` | +| `IsReadOnly` | Запрет записи | `false` | +| `BufferSize` | Размер буфера чтения/записи | `256 КБ` | + +### S3FileStorageOptions + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `Endpoint` | URL S3-эндпоинта | *(обязательно)* | +| `AccessKey` | Ключ доступа S3 | *(обязательно)* | +| `SecretKey` | Секретный ключ S3 | *(обязательно)* | +| `Bucket` | Имя бакета | *(обязательно)* | +| `Basket` | Имя области хранения | `"share"` | +| `Region` | Регион для SigV4 | `"eu-central-1"` | +| `IsReadOnly` | Запрет записи | `false` | + +### PostgresFileStorageOptions + +| Свойство | Описание | +|----------|----------| +| `SchemaName` | Схема PostgreSQL | +| `TableName` | Имя таблицы для данных файлов | +| `PartOptions.PgPartBy` | Стратегия партиционирования (day/month/year/list/range) | +| `CleanupOptions.ExpireDays` | Порог автоочистки | +| `StorageOptions.IsReadOnly` | Запрет записи | + +### InMemoryFileStorageOptions + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `Basket` | Имя области хранения | `"share"` | +| `IsReadOnly` | Запрет записи | `false` | + +--- + +## Пакетные операции + +`HybridFileStorageExtensions` предоставляет высокоуровневые методы для массовой обработки файлов с встроенным параллелизмом, обработкой ошибок и отчётами о прогрессе. + +```csharp +// Копирование из локальной файловой системы +var result = await storage.CopyFromFileAsync( + @"C:\temp\document.pdf", + "archive", + new UploadFileInput { FileName = "archived.pdf", TenantId = 42 }); + +// Копирование между корзинами/областями +var moved = await storage.CopyToBasketAsync( + "s3://share/42/doc.pdf", + "backup"); + +// Пакетное копирование с параллелизмом и прогрессом +var batchResult = await storage.CopyToScopeBatchAsync( + fileIds: ["s3://share/1/a.txt", "s3://share/2/b.txt"], + basket: "archive", + options: new BatchOptions + { + MaxDegreeOfParallelism = 8, + ContinueOnError = true, + Progress = new Progress() + }); + +foreach (var ok in batchResult.Succeeded) + Console.WriteLine($"Скопировано: {ok.FileId}"); + +foreach (var err in batchResult.Failed) + Console.WriteLine($"Ошибка #{err.Index}: {err.FileId} — {err.Exception.Message}"); +``` + +### BatchResult + +| Член | Тип | Описание | +|------|-----|----------| +| `Succeeded` | `IReadOnlyList` | Успешные результаты | +| `Failed` | `IReadOnlyList` | Ошибки с File ID и исключением | +| `Total` | `int` | Всего обработанных элементов | +| `HasErrors` | `bool` | Были ли ошибки | +| `ThrowIfHasErrors()` | `void` | Выбрасывает `BatchOperationException` при наличии ошибок | + +### BatchOptions + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `MaxDegreeOfParallelism` | Одновременные операции | `4` | +| `ContinueOnError` | Продолжать после ошибок | `true` | +| `OperationTimeout` | Таймаут на операцию | `0` (бесконечность) | +| `Progress` | Отчётчик прогресса | `null` | + +--- + +## Перехватчики (Interceptors) + +Хуки жизненного цикла для операций загрузки/скачивания/удаления. + +```csharp +public interface IUploadInterceptor +{ + ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken ct); + ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken ct); + ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken ct); +} + +public interface IDownloadInterceptor { /* аналогично Can/After/Error */ } +public interface IDeleteInterceptor { /* аналогично Can/After/Error */ } +``` + +Регистрация перехватчиков через fluent builder: + +```csharp +services.AddSaHybridFileStorage(cfg => cfg.ConfigureInterceptors((sp, container) => +{ + container.AddUploadInterceptor(myCustomInterceptor); + container.AddDownloadInterceptor(loggingInterceptor); +})); +``` + +Встроенный `LoggingInterceptor` доступен через `.AddLogging()`. + +--- + +## Режим «только чтение» + +Установите `IsReadOnly = true` для любого провайдера хранилища, чтобы запретить запись. Попытки записи вызывают `HybridFileStorageWritableException`: + +```csharp +builder.Services.AddSaFileSystemFileStorage(settings => +{ + settings.BasePath = @"C:\readonly\data"; + settings.IsReadOnly = true; +}); +``` + +--- + +## Доменные типы + +### StorageResult + +```csharp +public sealed record StorageResult( + string FileId, + string AbsoluteUrl, + string StorageType, + DateTimeOffset UploadedAt); +``` + +### UploadFileInput + +```csharp +public sealed record UploadFileInput +{ + public int TenantId { get; init; } + public string FileName { get; init; } + public static UploadFileInput Empty { get; } +} +``` + +### FileMetadata + +```csharp +public sealed class FileMetadata +{ + public required string Basket { get; init; } + public required string FileName { get; init; } + public int TenantId { get; init; } + public required string StorageType { get; init; } +} +``` + +--- + +## Исключения + +| Исключение | Когда выбрасывается | +|------------|-------------------| +| `HybridFileStorageNoAvailableException` | Не найдено хранилище для запрошенной корзины | +| `HybridFileStorageWritableException` | Попытка записи в хранилище «только чтение» | +| `HybridFileStorageAggregateException` | Несколько ошибок провайдеров агрегированы | +| `BatchOperationException` | Пакет с ошибками и `ContinueOnError = false` | + +--- + +## Структура проекта + +``` +src/Sa.HybridFileStorage/ +├── IHybridFileStorage.cs # Главный интерфейс +├── HybridFileStorage.cs # Реализация с failover +├── HybridFileStorageContainer.cs # Контейнер провайдеров +├── HybridStorageBuilder.cs # Fluent builder +├── HybridFileStorageExtensions.cs # Пакетные операции +├── Setup.cs # DI расширения +├── FileMetadata.cs # DTO метаданных +├── BatchResult.cs # Типы результатов пакетной обработки +└── Interceptors/ # Хуки загрузки/скачивания/удаления + +src/Sa.HybridFileStorage.FileSystem/ # Провайдер файловой системы +src/Sa.HybridFileStorage.S3/ # Провайдер S3 +src/Sa.HybridFileStorage.Postgres/ # Провайдер PostgreSQL +``` + +--- + +## Лицензия + +MIT diff --git a/src/Sa.HybridFileStorage/Readme.md b/src/Sa.HybridFileStorage/Readme.md index a607c6d7..42fda1f9 100644 --- a/src/Sa.HybridFileStorage/Readme.md +++ b/src/Sa.HybridFileStorage/Readme.md @@ -1,10 +1,8 @@ -# Hybrid File Storage +# Sa.HybridFileStorage -## IHybridFileStorage Interface +Hybrid file storage abstraction with automatic provider failover. Unifies multiple storage backends (FileSystem, S3, PostgreSQL) under a single resilient API — if one provider becomes unavailable, the system switches to another. -The `IHybridFileStorage` interface enhances the resilience and availability of file data in applications that require reliable storage management. - -This interface defines a contract for hybrid file storage systems capable of handling file operations such as uploading, downloading, and deleting files. The integration of multiple storage providers (such as file system, S3, and PostgreSQL) ensures reliable file storage, as the system can automatically switch between different providers in the event that one becomes unavailable. +--- ## Supported Storage Providers @@ -13,20 +11,21 @@ This interface defines a contract for hybrid file storage systems capable of han | **File System** | `FileSystemStorage` | Local development, on-premise deployments | | **S3 Compatible** | `S3FileStorage` | Cloud storage (AWS S3, MinIO, etc.) | | **PostgreSQL** | `PostgresFileStorage` | Database-embedded files, transactional consistency | +| **In-Memory** | `InMemoryFileStorage` | Testing, ephemeral scenarios | + +--- ## Key Features - ✅ **Unified API** — Single interface for all storage providers -- ✅ **Basket-Tenant-based isolation** — Multi-tenant support +- ✅ **Basket-Tenant-based isolation** — Multi-tenant support with scoped buckets - ✅ **Read-only mode** — Protect storage from accidental modifications - ✅ **Streaming support** — Memory-efficient file transfers - ✅ **Native AOT ready** — Full compatibility with .NET 10 Native AOT -- ✅ **Batch operations** — Efficient bulk file processing - -## Batch Operations - -The `HybridFileStorageExtensions` class provides high-level methods for bulk file operations with built-in parallelism, error handling, and progress reporting. +- ✅ **Batch operations** — Efficient bulk file processing with parallelism +- ✅ **Interceptors** — Upload/download/delete lifecycle hooks +--- ## File ID Format @@ -40,34 +39,273 @@ All files are identified using a unified URI-like format: - `s3://share/42/document.pdf` - `fs://root/100/report.xlsx` - `pg://files/7/1773210911/some/data.bin` +- `mem://share/42/temp.txt` + +--- + +## Quick Start + +### Without DI + +```csharp +using var memory = new InMemoryFileStorage(new InMemoryFileStorageOptions("share")); +var container = new HybridFileStorageContainer([memory]); +var storage = new HybridFileStorage(container, InterceptorContainer.Empty); -## Installation +var stream = "Hello, HybridFileStorage!".ToStream(); +var result = await storage.UploadAsync( + "share", + new UploadFileInput { FileName = "file.txt", TenantId = 42 }, + stream, + ct); + +await storage.DownloadAsync(result.FileId, async (fs, t) => +{ + var content = await fs.ToStrAsync(t); + Console.WriteLine(content); +}); +``` + +### With DI + +```csharp +builder.Services.AddSaHybridFileStorage(configure => configure + .AddStorage(InMemoryFileStorage.New("share")) + .AddLogging()); + +// Or register individual providers: +builder.Services.AddSaFileSystemFileStorage(new FileSystemStorageSettings +{ + BasePath = @"C:\data\files", + Basket = "documents" +}); + +builder.Services.AddSaS3FileStorage(new S3FileStorageOptions +{ + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123", + Bucket = "mybucket", + Basket = "uploads" +}); -```bash -dotnet add package Sa.HybridFileStorage +// Usage: +var storage = serviceProvider.GetRequiredService(); ``` +--- + +## Settings + +### FileSystemStorageSettings -## Usage Example +| Property | Description | Default | +|----------|-------------|---------| +| `BasePath` | Root directory for files | *(required)* | +| `Basket` | Storage scope name | `"share"` | +| `StorageType` | Scheme prefix in File ID | `"fs"` | +| `IsReadOnly` | Prevent writes | `false` | +| `BufferSize` | Read/write buffer size | `256 KB` | + +### S3FileStorageOptions + +| Property | Description | Default | +|----------|-------------|---------| +| `Endpoint` | S3 endpoint URL | *(required)* | +| `AccessKey` | S3 access key | *(required)* | +| `SecretKey` | S3 secret key | *(required)* | +| `Bucket` | Bucket name | *(required)* | +| `Basket` | Storage scope name | `"share"` | +| `Region` | Region for SigV4 | `"eu-central-1"` | +| `IsReadOnly` | Prevent writes | `false` | + +### PostgresFileStorageOptions + +| Property | Description | +|----------|-------------| +| `SchemaName` | PostgreSQL schema | +| `TableName` | Table name for file data | +| `PartOptions.PgPartBy` | Partitioning strategy (day/month/year/list/range) | +| `CleanupOptions.ExpireDays` | Auto-cleanup threshold | +| `StorageOptions.IsReadOnly` | Prevent writes | + +### InMemoryFileStorageOptions + +| Property | Description | Default | +|----------|-------------|---------| +| `Basket` | Storage scope name | `"share"` | +| `IsReadOnly` | Prevent writes | `false` | + +--- + +## Batch Operations + +`HybridFileStorageExtensions` provides high-level methods for bulk file operations with built-in parallelism, error handling, and progress reporting. ```csharp +// Copy from local filesystem +var result = await storage.CopyFromFileAsync( + @"C:\temp\document.pdf", + "archive", + new UploadFileInput { FileName = "archived.pdf", TenantId = 42 }); -// di -builder.AddStorage(new InMemoryFileStorage()) -builder.Services.AddSaHybridStorage(); +// Copy between baskets/scopes +var moved = await storage.CopyToBasketAsync( + "s3://share/42/doc.pdf", + "backup"); -// some test -using var stream = "Hello, HybridFileStorage!".ToStream(); +// Batch copy with parallelism and progress +var batchResult = await storage.CopyToScopeBatchAsync( + fileIds: ["s3://share/1/a.txt", "s3://share/2/b.txt"], + basket: "archive", + options: new BatchOptions + { + MaxDegreeOfParallelism = 8, + ContinueOnError = true, + Progress = new Progress() + }); -await storage.UploadAsync( - "basket", - new UploadFileInput { FileName = "file.txt" }, - stream, - cancellationToken); +foreach (var ok in batchResult.Succeeded) + Console.WriteLine($"Copied: {ok.FileId}"); + +foreach (var err in batchResult.Failed) + Console.WriteLine($"Failed #{err.Index}: {err.FileId} — {err.Exception.Message}"); +``` + +### BatchResult + +| Member | Type | Description | +|--------|------|-------------| +| `Succeeded` | `IReadOnlyList` | Successful results | +| `Failed` | `IReadOnlyList` | Errors with file ID and exception | +| `Total` | `int` | Total items processed | +| `HasErrors` | `bool` | Whether any operation failed | +| `ThrowIfHasErrors()` | `void` | Throws `BatchOperationException` if failures occurred | + +### BatchOptions + +| Property | Description | Default | +|----------|-------------|---------| +| `MaxDegreeOfParallelism` | Concurrent operations | `4` | +| `ContinueOnError` | Keep going after failures | `true` | +| `OperationTimeout` | Per-operation timeout | `0` (infinite) | +| `Progress` | Progress reporter | `null` | + +--- + +## Interceptors -await storage.DownloadAsync( - result.FileId, - async (fs, t) => actual = await fs.ToStrAsync(t), - cancellationToken); +Lifecycle hooks for upload/download/delete operations. +```csharp +public interface IUploadInterceptor +{ + ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken ct); + ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken ct); + ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken ct); +} + +public interface IDownloadInterceptor { /* analogous Can/After/Error */ } +public interface IDeleteInterceptor { /* analogous Can/After/Error */ } +``` + +Register interceptors via fluent builder: + +```csharp +services.AddSaHybridFileStorage(cfg => cfg.ConfigureInterceptors((sp, container) => +{ + container.AddUploadInterceptor(myCustomInterceptor); + container.AddDownloadInterceptor(loggingInterceptor); +})); +``` + +Built-in `LoggingInterceptor` is available via `.AddLogging()`. + +--- + +## Read-Only Mode + +Set `IsReadOnly = true` on any storage provider to prevent writes. Attempted writes throw `HybridFileStorageWritableException`: + +```csharp +builder.Services.AddSaFileSystemFileStorage(settings => +{ + settings.BasePath = @"C:\readonly\data"; + settings.IsReadOnly = true; +}); +``` + +--- + +## Domain Types + +### StorageResult + +```csharp +public sealed record StorageResult( + string FileId, + string AbsoluteUrl, + string StorageType, + DateTimeOffset UploadedAt); ``` + +### UploadFileInput + +```csharp +public sealed record UploadFileInput +{ + public int TenantId { get; init; } + public string FileName { get; init; } + public static UploadFileInput Empty { get; } +} +``` + +### FileMetadata + +```csharp +public sealed class FileMetadata +{ + public required string Basket { get; init; } + public required string FileName { get; init; } + public int TenantId { get; init; } + public required string StorageType { get; init; } +} +``` + +--- + +## Exceptions + +| Exception | When thrown | +|-----------|------------| +| `HybridFileStorageNoAvailableException` | No storage found for the requested basket | +| `HybridFileStorageWritableException` | Write attempted on read-only storage | +| `HybridFileStorageAggregateException` | Multiple provider errors aggregated | +| `BatchOperationException` | Batch with failures and `ContinueOnError = false` | + +--- + +## Project Structure + +``` +src/Sa.HybridFileStorage/ +├── IHybridFileStorage.cs # Main interface +├── HybridFileStorage.cs # Implementation with failover +├── HybridFileStorageContainer.cs # Provider container +├── HybridStorageBuilder.cs # Fluent builder +├── HybridFileStorageExtensions.cs # Batch operations +├── Setup.cs # DI extensions +├── FileMetadata.cs # Metadata DTO +├── BatchResult.cs # Batch result types +└── Interceptors/ # Upload/download/delete hooks + +src/Sa.HybridFileStorage.FileSystem/ # Filesystem provider +src/Sa.HybridFileStorage.S3/ # S3 provider +src/Sa.HybridFileStorage.Postgres/ # PostgreSQL provider +``` + +--- + +## License + +MIT diff --git a/src/Sa.Media.FFmpeg/Readme-ru.md b/src/Sa.Media.FFmpeg/Readme-ru.md new file mode 100644 index 00000000..89194d08 --- /dev/null +++ b/src/Sa.Media.FFmpeg/Readme-ru.md @@ -0,0 +1,288 @@ +# Sa.Media.FFmpeg + +Кроссплатформенная обёртка .NET над FFmpeg (Windows x64, Linux) со **встроенными статическими бинарниками** — работает сразу без установки в систему. Упрощает обработку аудио: извлечение метаданных, конвертация форматов, разделение/объединение каналов и DI-интеграция. + +--- + +## Возможности + +- 🎵 **Извлечение метаданных** — длительность, битрейт, формат, частота дискретизации, каналы через `ffprobe` +- 🔊 **Конвертация аудио** — PCM S16 LE WAV, MP3, OGG Vorbis/Opus +- 🎛️ **Манипуляция каналами** — разделение стерео на монофайлы, объединение двух моно в стерео +- 📦 **Встроенные бинарники FFmpeg** — Windows x64/arm64, Linux x64/arm64, macOS x64 (fallback на linux-x64) +- 💉 **Поддержка DI** — стандартная интеграция с `IServiceCollection` и конфигурацией опций +- ⚡ **Потоковый I/O** — передача аудио напрямую из потоков без промежуточных файлов + +--- + +## Быстрый старт + +### Дефолтные экземпляры (без настройки) + +```csharp +using Sa.Media.FFmpeg; + +// Извлечение метаданных +var meta = await IFFProbeExecutor.Default.GetMetaInfo("input.mp3"); +Console.WriteLine($"Duration: {meta.Duration}s, Channels: {meta.Channels}"); + +// Конвертация аудио +await IFFMpegExecutor.Default.ConvertToPcmS16Le( + "input.mp3", + "output.wav", + outputSampleRate: 16000, + outputChannelCount: 1); + +// Получение поддерживаемых форматов/кодеков +var formats = await IFFMpegExecutor.Default.GetFormats(); +var codecs = await IFFMpegExecutor.Default.GetCodecs(); +``` + +### Разделение каналов (стерео → монофайлы) + +```csharp +var splitter = new PcmS16LeChannelManipulator(); + +var resultFiles = await splitter.SplitAsync( + inputFileName: "stereo.mp3", + outputFileName: "output", + outputSampleRate: 16000, + isOverwrite: true); + +// Создаёт: +// output_channel_0.wav — левый канал +// output_channel_1.wav — правый канал +``` + +### Объединение каналов (моно → стерео) + +```csharp +var merger = new PcmS16LeChannelManipulator(); + +var joined = await merger.JoinAsync( + leftFileName: "left.wav", + rightFileName: "right.wav", + outputFileName: "stereo_output.wav", + outputSampleRate: 16000); +``` + +### Потоковая конвертация (без промежуточных файлов) + +```csharp +await using var inputStream = File.OpenRead("input.mp3"); + +await IFFMpegExecutor.Default.ConvertToPcmS16Le( + inputStream, + inputFormat: "mp3", + onOutput: async (stream, ct) => + { + // Обрабатываем WAV-поток напрямую — например, подаём в AsyncWavReader + await using var reader = new AsyncWavReader(stream); + await foreach (var packet in reader.ReadDoubleSamplesAsync(ct)) + { + Console.WriteLine($"Sample: {packet.Sample:F4}"); + } + }, + outputSampleRate: 16000, + outputChannelCount: 1); +``` + +--- + +## С DI + +```csharp +builder.Services.AddSaFFMpeg(configure: options => +{ + options.ExecutablePath = @"C:\tools\ffmpeg.exe"; // опциональный override + options.WritableDirectory = @"C:\temp\output"; + options.TimeoutSeconds = 300; // 5 минут +}); + +// Использование: +var executor = serviceProvider.GetRequiredService(); +var probe = serviceProvider.GetRequiredService(); +var manip = serviceProvider.GetRequiredService(); +``` + +Привязка секции конфигурации: + +```csharp +builder.Services.AddSaFFMpeg(configSectionPath: "Ffmpeg"); + +// appsettings.json: +// { +// "Ffmpeg": { +// "ExecutablePath": "/usr/bin/ffmpeg", +// "WritableDirectory": "/tmp/output", +// "TimeoutSeconds": 300 +// } +// } +``` + +--- + +## Поддерживаемые конвертации + +| Источник | Целевой | Метод | Примечание | +|----------|---------|-------|-----------| +| Любой, поддерживаемый FFmpeg | **PCM S16 LE WAV** | `ConvertToPcmS16Le()` | Настраиваемая частота (по умолч. 16 кГц), кол-во каналов | +| Любой | **PCM S16 LE WAV** | `ConvertToPcmS16LePreservingFormat()` | Сохраняет исходную частоту и каналы | +| Любой | **MP3** | `ConvertToMp3()` | 16 кГц, 128 kbps, libmp3lame | +| Любой | **OGG Vorbis** | `ConvertToOgg(isLibopus: false)` | Стандартный Vorbis | +| Любой | **OGG Opus** | `ConvertToOgg(isLibopus: true)` | Кодек Opus (только Linux) | + +--- + +## Настройки + +### FFMpegOptions + +| Свойство | Тип | Описание | По умолчанию | +|----------|-----|----------|-------------| +| `ExecutablePath` | `string?` | Полный путь к бинарнику ffmpeg/ffprobe | Автопоиск (встроенный → PATH) | +| `WritableDirectory` | `string?` | Директория для выходных файлов | Текущая рабочая директория | +| `TimeoutSeconds` | `int?` | Таймаут операции в секундах | `300` (5 минут) | + +Вызовите `options.Validate()` для проверки существования `WritableDirectory` и неотрицательности таймаута. + +--- + +## Справочник публичного API + +### IFFMpegExecutor + +| Свойство/Метод | Возврат | Описание | +|----------------|---------|----------| +| `Default` | `IFFMpegExecutor` | Статический дефолтный экземпляр (использует встроенный бинарник) | +| `Executor` | `IFFRawExecutor` | Внутренний низкоуровневый процессор | +| `GetVersion()` | `Task` | Строка версии FFmpeg | +| `GetFormats()` | `Task` | Все поддерживаемые форматы | +| `GetCodecs()` | `Task` | Все поддерживаемые кодеки | +| `ConvertToPcmS16Le(file, file, ...)` | `Task` | Конвертация в WAV-файл | +| `ConvertToPcmS16LePreservingFormat(file, file, ...)` | `Task` | Конвертация с сохранением формата | +| `ConvertToPcmS16Le(stream, func, ...)` | `Task` | Потоковая конвертация | +| `ConvertToMp3(file, file, ...)` | `Task` | Конвертация в MP3 | +| `ConvertToOgg(file, file, ...)` | `Task` | Конвертация в OGG (Vorbis или Opus) | + +### IFFProbeExecutor + +| Свойство/Метод | Возврат | Описание | +|----------------|---------|----------| +| `Default` | `IFFProbeExecutor` | Статический дефолтный экземпляр | +| `Executor` | `IFFRawExecutor` | Внутренний низкоуровневый процессор | +| `GetChannelsAndSampleRate()` | `Task<(int?, int?)>` | Сырая пара канал/частота | +| `GetMetaInfo(file)` | `Task` | Полные метаданные из пути к файлу | +| `GetMetaInfo(stream, format)` | `Task` | Полные метаданные из потока | + +### IPcmS16LeChannelManipulator + +| Метод | Возврат | Описание | +|-------|---------|----------| +| `SplitAsync(input, output, ...)` | `Task>` | Разделить стерео → несколько моно WAV | +| `JoinAsync(left, right, output, ...)` | `Task` | Объединить два моно → стерео WAV | + +### IFFRawExecutor + +| Свойство/Метод | Возврат | Описание | +|----------------|---------|----------| +| `ExecutablePath` | `string` | Путь к бинарнику ffmpeg | +| `DefaultTimeout` | `TimeSpan` | Дефолтный таймаут операции | +| `ExecuteAsync(args, ...)` | `Task` | Выполнить FFmpeg с аргументами | +| `ExecuteStdOutAsync(args, stream, func, ...)` | `Task` | Пропустить stdin/stdout через FFmpeg | + +--- + +## Доменные типы + +### MediaMetadata + +```csharp +public sealed record MediaMetadata( + double? Duration = null, + string? FormatName = null, + int? BitRate = null, + int? Size = null) +{ + public static readonly MediaMetadata Empty = new(); +} +``` + +### ProcessExecutionResult + +```csharp +public record ProcessExecutionResult( + int ExitCode, + string StandardOutput, + string StandardError); +``` + +--- + +## Исключения + +| Исключение | Когда выбрасывается | +|------------|-------------------| +| `ProcessExecutionException` | FFmpeg завершается с ненулевым кодом | +| `ProcessExecutionResultException` | Обёртка над `ProcessExecutionResult` с форматированным сообщением | +| `ProcessStartException` | Не удалось запустить процесс FFmpeg | +| `ProcessTimeoutException` | Операция превысила таймаут | + +--- + +## Встроенные бинарники + +Статические сборки FFmpeg встраиваются на этапе билда и распаковываются в `sa/native/` во время выполнения. Установка в систему не требуется. + +**Поддерживаемые RID:** `win-x64`, `win-arm64`, `linux-x64`, `linux-arm64`, `osx-x64` (macOS fallback на linux-x64). + +**Порядок поиска:** +1. `AppContext.BaseDirectory/sa/native/ffmpeg` +2. `AppContext.BaseDirectory/ffmpeg` +3. Системный `PATH` + +--- + +## Нативные зависимости (Linux) + +Ubuntu/Debian: + +```bash +sudo apt update && sudo apt install libmp3lame0 libopus0 libvorbis0a libvorbisenc2 +``` + +Alpine Linux: + +```bash +sudo apk add lame-libs opus libvorbis +``` + +--- + +## Структура проекта + +``` +src/Sa.Media.FFmpeg/ +├── IFFMpegExecutor.cs # Интерфейс конвертации аудио +├── IFFProbeExecutor.cs # Интерфейс извлечения метаданных +├── IFFRawExecutor.cs # Низкоуровневое выполнение процессов +├── IFFMpegExecutorFactory.cs # Фабрика создания экzekторов +├── IFFMpegLocator.cs # Поиск бинарников +├── IPcmS16LeChannelManipulator.cs # Операции split/join +├── FFMpegOptions.cs # Опции конфигурации +├── MediaMetadata.cs # DTO результата probe +├── Services/ +│ ├── ProcessExecutor.cs # Запускщик процессов + исключения +│ ├── FFMpegExecutor.cs # Реализация +│ ├── FFProbeExecutor.cs # Реализация +│ └── ... # Внутренние парсеры, сериализаторы +├── buildTransitive/ +│ └── Sa.Media.FFmpeg.targets # MSBuild: распаковка нативных бинарников +└── sa/ # Локальные ZIP-архивы (только для разработки) +``` + +--- + +## Лицензия + +MIT diff --git a/src/Sa.Media.FFmpeg/Readme.md b/src/Sa.Media.FFmpeg/Readme.md index 4db82a9c..caa00872 100644 --- a/src/Sa.Media.FFmpeg/Readme.md +++ b/src/Sa.Media.FFmpeg/Readme.md @@ -1,73 +1,254 @@ -# Sa.Media.FFmpeg +# Sa.Media.FFmpeg -## FFmpeg .NET Wrapper - ready to use out of the box with minimal setup +Cross-platform .NET wrapper for FFmpeg (Windows x64, Linux) with **bundled static binaries** — works out of the box without system-wide installation. Simplifies audio processing: metadata extraction, format conversion, channel split/join, and DI integration. -A cross-platform .NET wrapper for FFmpeg (Windows x64 and Linux), designed to simplify audio and video processing in .NET applications. The library provides a bundled static FFmpeg build when it is not installed system-wide, ensuring smooth operation without external dependencies. +--- -- Extract metadata from media files (duration, channels, sample rate, etc.) -- Convert audio to: wav, mp3, mp4, ogg, ac3, mov .. -- Splits/Join audio file by channels -- Built-in FFmpeg binaries for Windows x64 and Linux -- Supports Dependency Injection (DI) via standard IServiceCollection integration +## Features +- 🎵 **Metadata extraction** — duration, bitrate, format, sample rate, channels via `ffprobe` +- 🔊 **Audio conversion** — PCM S16 LE WAV, MP3, OGG Vorbis/Opus +- 🎛️ **Channel manipulation** — split stereo to mono files, join two monos into stereo +- 📦 **Bundled FFmpeg binaries** — Windows x64/arm64, Linux x64/arm64, macOS x64 (falls back to linux-x64) +- 💉 **DI support** — standard `IServiceCollection` integration with options configuration +- ⚡ **Streaming I/O** — pipe audio directly from streams without intermediate files -Interfaces: +--- -- `IFFMpegExecutor` — perform audio conversion tasks by `ffmpeg` -- `IFFProbeExecutor` — retrieve stream info and metadata by `ffprobe` +## Quick Start +### Default instances (no setup required) -## Example Usage +```csharp +using Sa.Media.FFmpeg; -Audio Conversion +// Metadata extraction +var meta = await IFFProbeExecutor.Default.GetMetaInfo("input.mp3"); +Console.WriteLine($"Duration: {meta.Duration}s, Channels: {meta.Channels}"); -```csharp - var ffmpeg = Sa.Media.FFmpeg.IFFMpegExecutor.Default; - - var codecs = await ffmpeg.GetCodecs(); - Console.WriteLine(codecs); - - await ffmpeg.ConvertToPcmS16Le( - "data/input.mp3", - "data/output.wav", - outputChannelCount: 1); -``` +// Audio conversion +await IFFMpegExecutor.Default.ConvertToPcmS16Le( + "input.mp3", + "output.wav", + outputSampleRate: 16000, + outputChannelCount: 1); +// Get supported formats/codecs +var formats = await IFFMpegExecutor.Default.GetFormats(); +var codecs = await IFFMpegExecutor.Default.GetCodecs(); +``` -Splits input audio file by channels +### Channel split (stereo → mono files) ```csharp var splitter = new PcmS16LeChannelManipulator(); var resultFiles = await splitter.SplitAsync( - inputFileName: "input.mp3", - outputFileName: "output.wav", - sampleRate: 16000, - isOverwrite: true -); + inputFileName: "stereo.mp3", + outputFileName: "output", + outputSampleRate: 16000, + isOverwrite: true); + +// Produces: +// output_channel_0.wav — left channel +// output_channel_1.wav — right channel ``` -This will produce: +### Channel join (mono → stereo) + +```csharp +var merger = new PcmS16LeChannelManipulator(); +var joined = await merger.JoinAsync( + leftFileName: "left.wav", + rightFileName: "right.wav", + outputFileName: "stereo_output.wav", + outputSampleRate: 16000); ``` -output_channel_0.wav -output_channel_1.wav + +### Streaming conversion (no intermediate files) + +```csharp +await using var inputStream = File.OpenRead("input.mp3"); + +await IFFMpegExecutor.Default.ConvertToPcmS16Le( + inputStream, + inputFormat: "mp3", + onOutput: async (stream, ct) => + { + // Process WAV stream directly — e.g., feed into AsyncWavReader + await using var reader = new AsyncWavReader(stream); + await foreach (var packet in reader.ReadDoubleSamplesAsync(ct)) + { + Console.WriteLine($"Sample: {packet.Sample:F4}"); + } + }, + outputSampleRate: 16000, + outputChannelCount: 1); ``` +--- -## Check library dependencies -To see all missing dependencies: +## With DI -```bash -cd bin/Debug/net10.0/sa/native/ -ldd ffmpeg +```csharp +builder.Services.AddSaFFMpeg(configure: options => +{ + options.ExecutablePath = @"C:\tools\ffmpeg.exe"; // optional override + options.WritableDirectory = @"C:\temp\output"; + options.TimeoutSeconds = 300; // 5 minutes +}); + +// Usage: +var executor = serviceProvider.GetRequiredService(); +var probe = serviceProvider.GetRequiredService(); +var manip = serviceProvider.GetRequiredService(); +``` + +Configuration section binding: + +```csharp +builder.Services.AddSaFFMpeg(configSectionPath: "Ffmpeg"); + +// appsettings.json: +// { +// "Ffmpeg": { +// "ExecutablePath": "/usr/bin/ffmpeg", +// "WritableDirectory": "/tmp/output", +// "TimeoutSeconds": 300 +// } +// } +``` + +--- + +## Supported Conversions + +| Source | Target | Method | Notes | +|--------|--------|--------|-------| +| Any FFmpeg-supported | **PCM S16 LE WAV** | `ConvertToPcmS16Le()` | Custom sample rate (default 16 kHz), channel count | +| Any | **PCM S16 LE WAV** | `ConvertToPcmS16LePreservingFormat()` | Preserves original sample rate & channels | +| Any | **MP3** | `ConvertToMp3()` | 16 kHz, 128 kbps, libmp3lame | +| Any | **OGG Vorbis** | `ConvertToOgg(isLibopus: false)` | Standard Vorbis | +| Any | **OGG Opus** | `ConvertToOgg(isLibopus: true)` | Opus codec (Linux only) | + +--- + +## Settings + +### FFMpegOptions + +| Property | Type | Description | Default | +|----------|------|-------------|---------| +| `ExecutablePath` | `string?` | Full path to ffmpeg/ffprobe binary | Auto-discovery (bundled → PATH) | +| `WritableDirectory` | `string?` | Output directory for generated files | Current working directory | +| `TimeoutSeconds` | `int?` | Operation timeout in seconds | `300` (5 minutes) | + +Call `options.Validate()` to verify `WritableDirectory` exists and timeout is non-negative. + +--- + +## Public API Reference + +### IFFMpegExecutor + +| Property/Method | Returns | Description | +|-----------------|---------|-------------| +| `Default` | `IFFMpegExecutor` | Static default instance (uses bundled binary) | +| `Executor` | `IFFRawExecutor` | Underlying raw process executor | +| `GetVersion()` | `Task` | FFmpeg version string | +| `GetFormats()` | `Task` | All supported formats | +| `GetCodecs()` | `Task` | All supported codecs | +| `ConvertToPcmS16Le(file, file, ...)` | `Task` | Convert to WAV file | +| `ConvertToPcmS16LePreservingFormat(file, file, ...)` | `Task` | Convert preserving original format | +| `ConvertToPcmS16Le(stream, func, ...)` | `Task` | Stream-based conversion | +| `ConvertToMp3(file, file, ...)` | `Task` | Convert to MP3 | +| `ConvertToOgg(file, file, ...)` | `Task` | Convert to OGG (Vorbis or Opus) | + +### IFFProbeExecutor + +| Property/Method | Returns | Description | +|-----------------|---------|-------------| +| `Default` | `IFFProbeExecutor` | Static default instance | +| `Executor` | `IFFRawExecutor` | Underlying raw process executor | +| `GetChannelsAndSampleRate()` | `Task<(int?, int?)>` | Raw channel/sample-rate pair | +| `GetMetaInfo(file)` | `Task` | Full metadata from file path | +| `GetMetaInfo(stream, format)` | `Task` | Full metadata from stream | + +### IPcmS16LeChannelManipulator + +| Method | Returns | Description | +|--------|---------|-------------| +| `SplitAsync(input, output, ...)` | `Task>` | Split stereo → multiple mono WAVs | +| `JoinAsync(left, right, output, ...)` | `Task` | Join two monos → stereo WAV | + +### IFFRawExecutor + +| Property/Method | Returns | Description | +|-----------------|---------|-------------| +| `ExecutablePath` | `string` | Path to the ffmpeg binary | +| `DefaultTimeout` | `TimeSpan` | Default operation timeout | +| `ExecuteAsync(args, ...)` | `Task` | Execute FFmpeg with arguments | +| `ExecuteStdOutAsync(args, stream, func, ...)` | `Task` | Stream stdin/stdout through FFmpeg | + +--- + +## Domain Types + +### MediaMetadata + +```csharp +public sealed record MediaMetadata( + double? Duration = null, + string? FormatName = null, + int? BitRate = null, + int? Size = null) +{ + public static readonly MediaMetadata Empty = new(); +} +``` + +### ProcessExecutionResult + +```csharp +public record ProcessExecutionResult( + int ExitCode, + string StandardOutput, + string StandardError); ``` +--- + +## Exceptions + +| Exception | When thrown | +|-----------|------------| +| `ProcessExecutionException` | FFmpeg exits with non-zero code | +| `ProcessExecutionResultException` | Wraps `ProcessExecutionResult` with formatted message | +| `ProcessStartException` | Failed to start FFmpeg process | +| `ProcessTimeoutException` | Operation exceeded timeout | + +--- + +## Bundled Binaries + +FFmpeg static builds are embedded at build time and unpacked into `sa/native/` at runtime. No system installation required. + +**Supported RIDs:** `win-x64`, `win-arm64`, `linux-x64`, `linux-arm64`, `osx-x64` (macOS falls back to linux-x64). + +**Discovery order:** +1. `AppContext.BaseDirectory/sa/native/ffmpeg` +2. `AppContext.BaseDirectory/ffmpeg` +3. System `PATH` + +--- + +## Native Dependencies (Linux) + On Ubuntu/Debian: ```bash -sudo apt update -sudo apt install libmp3lame0 libopus0 libvorbis0a libvorbisenc2 +sudo apt update && sudo apt install libmp3lame0 libopus0 libvorbis0a libvorbisenc2 ``` On Alpine Linux: @@ -76,12 +257,32 @@ On Alpine Linux: sudo apk add lame-libs opus libvorbis ``` +--- -wsl build -``` -# WSL: +## Project Layout -dotnet nuget locals all --clear -dotnet restore -r linux-x64 -dotnet build -c Debug -r linux-x64 ``` +src/Sa.Media.FFmpeg/ +├── IFFMpegExecutor.cs # Audio conversion interface +├── IFFProbeExecutor.cs # Metadata extraction interface +├── IFFRawExecutor.cs # Low-level process execution +├── IFFMpegExecutorFactory.cs # Factory for creating executors +├── IFFMpegLocator.cs # Binary discovery +├── IPcmS16LeChannelManipulator.cs # Split/join operations +├── FFMpegOptions.cs # Configuration options +├── MediaMetadata.cs # Probe result DTO +├── Services/ +│ ├── ProcessExecutor.cs # Process runner + exceptions +│ ├── FFMpegExecutor.cs # Implementation +│ ├── FFProbeExecutor.cs # Implementation +│ └── ... # Internal parsers, serializers +├── buildTransitive/ +│ └── Sa.Media.FFmpeg.targets # MSBuild: unpack native binaries +└── sa/ # Local ZIP archives (dev only) +``` + +--- + +## License + +MIT diff --git a/src/Sa.Media/Readme-ru.md b/src/Sa.Media/Readme-ru.md new file mode 100644 index 00000000..825d0c15 --- /dev/null +++ b/src/Sa.Media/Readme-ru.md @@ -0,0 +1,184 @@ +# Sa.Media + +Асинхронный, экономичный по памяти WAV-ридер для .NET 10+. Создан для совместимости с Native AOT с нулевыми аллокациями на горячих путях. + +--- + +## Возможности + +- **Полностью асинхронный** — потоковая передача на основе `PipeReader`, без блокирующего I/O +- **Экономия памяти** — повторное использование буферов `ArrayPool`/`MemoryPool`, минимальная нагрузка на GC +- **Мультиформатная поддержка** — PCM 8/16/24/32-bit, IEEE Float 32/64-bit +- **Расширяемость** — поддерживает чанки `WAVE_FORMAT_EXTENSIBLE` +- **Обрезка по времени** — читайте только нужную часть через `TimeRange` +- **Канало-ориентированный** — перечисление сэмплов по каналам с отслеживанием позиции +- **Автоматический пропуск чанков** — `JUNK`, `LIST` и другие метаданные пропускаются прозрачно + +--- + +## Быстрый старт + +### Чтение заголовка + +```csharp +using var stream = File.OpenRead("test.wav"); +var reader = new AsyncWavReader(stream); + +var header = await reader.GetHeaderAsync(); +Console.WriteLine($"{header.NumChannels}ch @ {header.SampleRate}Hz, " + + $"{header.BitsPerSample}-bit {header.AudioFormat}"); +``` + +### Чтение сырых сэмплов по каналам + +```csharp +await using var reader = AsyncWavReader.CreateFromFile("test.wav"); + +await foreach (var packet in reader.ReadSamplesPerChannelAsync( + cancellationToken: ct)) +{ + Console.WriteLine($"Ch#{packet.ChannelId}: {packet.Sample.Length} bytes at pos {packet.Position}"); +} +``` + +### Чтение нормализованных double сэмплов [-1.0 … 1.0] + +```csharp +await using var reader = AsyncWavReader.CreateFromFile("test.wav"); + +await foreach (var packet in reader.ReadDoubleSamplesAsync(cancellationToken: ct)) +{ + Console.WriteLine($"Ch#{packet.ChannelId}: {packet.Sample:F4}"); +} +``` + +### Потоковые батчи (идеально для аудио-пайплайнов) + +```csharp +await using var reader = AsyncWavReader.CreateFromFile("test.wav"); + +await foreach (var batch in reader.ReadStreamableChunksAsync( + samplesPerBatch: 4096, + cancellationToken: ct)) +{ + // Каждый yield создаёт независимые данные — безопасно обрабатывать асинхронно +} +``` + +### Обрезка по временному диапазону + +```csharp +await using var reader = AsyncWavReader.CreateFromFile("test.wav"); + +// Читаем только секунды 5–15 +var range = TimeRange.Seconds(5, 15); +await foreach (var packet in reader.ReadDoubleSamplesAsync(range, cancellationToken: ct)) +{ + // Только сэмплы из обрезанного диапазона +} +``` + +### Конвертация в другой формат + +```csharp +await using var reader = AsyncWavReader.CreateFromFile("input.wav"); + +// Конвертация в 24-bit PCM +await foreach (var packet in reader.ConvertToFormatAsync( + AudioEncoding.Pcm24BitSigned, + cancellationToken: ct)) +{ + // Сырые 24-битные PCM байты на каждый семпл +} +``` + +--- + +## Поддерживаемые форматы + +| Формат | Чтение | Запись | +|--------|--------|--------| +| PCM 8-bit (unsigned) | ✅ | ✅ | +| PCM 16-bit (signed) | ✅ | ✅ | +| PCM 24-bit (signed) | ✅ | ✅ | +| PCM 32-bit (signed) | ✅ | ✅ | +| IEEE Float 32-bit | ✅ | ✅ | +| IEEE Float 64-bit | ✅ | ✅ | + +Все форматы поддерживают моно и стерео. Неизвестные чанки (`JUNK`, `LIST` и т.д.) автоматически пропускаются. + +--- + +## Справочник публичного API + +### Основные типы + +| Тип | Описание | +|-----|----------| +| `AsyncWavReader` | Главный асинхронный WAV-ридер — создаётся из `Stream` или пути к файлу | +| `WavHeader` | Разобранный RIFF/WAV заголовок с вычисляемыми свойствами (`IsPcm`, `IsStereo`, `Duration`) | +| `AudioPacket` | Record: `(ChannelId, Sample, Position, IsEof)` — сырые/конвертированные байты | +| `AudioNormalizedPacket` | Record: `(ChannelId, Sample, Position, IsEof)` — нормализованные double [-1.0, 1.0] | +| `TimeRange` | Record: `(From, To)` — обрезка по времени с фабричными методами | +| `AudioEncoding` | Enum: PCM 8/16/24/32, IEEE Float 32/64 | +| `WaveFormatType` | Enum: `Pcm`, `Adpcm`, `IeeeFloat`, `Extensible` | + +### Ключевые методы `AsyncWavReader` + +| Метод | Возврат | Описание | +|-------|---------|----------| +| `Create(Stream)` | `AsyncWavReader` | Фабрика из потока | +| `CreateFromFile(string)` | `AsyncWavReader` | Фабрика из пути к файлу | +| `GetHeaderAsync()` | `Task` | Потокобезопасное ленивое разбирание заголовка | +| `ReadSamplesPerChannelAsync()` | `IAsyncEnumerable` | Сырые сэмплы по каналам | +| `ReadDoubleSamplesAsync()` | `IAsyncEnumerable` | Нормализованные double сэмплы | +| `ConvertToFormatAsync()` | `IAsyncEnumerable` | Конвертация в целевую кодировку | +| `ReadStreamableChunksAsync()` | `IAsyncEnumerable` | Пакетные сэмплы для пайплайнов | + +### Фабрики `TimeRange` + +| Метод | Пример | Описание | +|-------|--------|----------| +| `TimeRange.Create(from, to)` | Базовый конструктор | From/to TimeSpan | +| `TimeRange.Ms(from, to)` | В миллисекундах | Точность до мс | +| `TimeRange.Seconds(from, to)` | В секундах | Точность до double секунд | +| `TimeRange.RangeFromDuration(from, dur)` | От начала + длительность | Построение от смещения | +| `TimeRange.Default` | `[0, ∞)` | Весь файл, без обрезки | + +--- + +## Примечания по производительности + +- `allowBufferReuse=true` (по умолчанию) повторно использует пуленные буферы между yields — вызывающий должен скопировать перед следующей итерацией +- `allowBufferReuse=false` выделяет новый массив на каждый семпл — безопаснее для параллельных потребителей +- `ReadStreamableChunksAsync` принудительно устанавливает `allowBufferReuse:false` внутри для предотвращения алиасинга буферов +- Все внутренние await используют `ConfigureAwait(false)` — безопасно в любом синхронизационном контексте + +--- + +## Структура проекта + +``` +src/Sa.Media/ +├── AsyncWavReader.cs # Основной класс ридера +├── AsyncWavWriter.cs # Внутренний WAV-писатель +├── AudioEncoding.cs # Enum форматов +├── AudioEncodingExtensions.cs +├── AudioPacket.cs # Record сырого семпла +├── AudioNormalizedPacket.cs # Record нормализованного семпла +├── BinaryPipeReader.cs # Little-endian бинарный ридер +├── PipeReaderExtensions.cs # Помощники пропуска +├── SampleConverter.cs # Конвертация PCM ↔ double +├── TimeRange.cs # Диапазон обрезки +├── TimeRangeExtensions.cs # Расширение, объединение, сортировка +├── WavHeader.cs # Модель RIFF заголовка +├── WavHeaderReader.cs # Парсер заголовка +├── WaveFormatType.cs # Enum типа формата +└── WaveFormatTypeExtensions.cs +``` + +--- + +## Лицензия + +MIT diff --git a/src/Sa.Media/Readme.md b/src/Sa.Media/Readme.md index 5803a21a..ed06e62b 100644 --- a/src/Sa.Media/Readme.md +++ b/src/Sa.Media/Readme.md @@ -2,6 +2,8 @@ Async, memory-efficient WAV file reader for .NET 10+. Designed for Native AOT compatibility with zero allocations on hot paths. +--- + ## Features - **Fully asynchronous** — `PipeReader`-based streaming, no blocking I/O @@ -12,6 +14,8 @@ Async, memory-efficient WAV file reader for .NET 10+. Designed for Native AOT co - **Channel-aware** — per-channel sample enumeration with position tracking - **Automatic chunk skipping** — `JUNK`, `LIST`, and other metadata chunks are transparently skipped +--- + ## Quick Start ### Read header @@ -88,6 +92,8 @@ await foreach (var packet in reader.ConvertToFormatAsync( } ``` +--- + ## Supported Formats | Format | Read | Write | @@ -101,6 +107,8 @@ await foreach (var packet in reader.ConvertToFormatAsync( All formats support mono and stereo. Unknown chunks (`JUNK`, `LIST`, etc.) are automatically skipped. +--- + ## Public API Reference ### Core types @@ -137,6 +145,8 @@ All formats support mono and stereo. Unknown chunks (`JUNK`, `LIST`, etc.) are a | `TimeRange.RangeFromDuration(from, dur)` | From start + duration | Build from offset | | `TimeRange.Default` | `[0, ∞)` | Full file, no trim | +--- + ## Performance Notes - `allowBufferReuse=true` (default) reuses pooled buffers across yields — caller must copy before next iteration @@ -144,6 +154,8 @@ All formats support mono and stereo. Unknown chunks (`JUNK`, `LIST`, etc.) are a - `ReadStreamableChunksAsync` forces `allowBufferReuse:false` internally to prevent buffer aliasing - All internal awaits use `ConfigureAwait(false)` — safe in any synchronization context +--- + ## Project Layout ``` @@ -164,3 +176,9 @@ src/Sa.Media/ ├── WaveFormatType.cs # Format type enum └── WaveFormatTypeExtensions.cs ``` + +--- + +## License + +MIT diff --git a/src/Sa.Outbox.PostgreSql/Readme-ru.md b/src/Sa.Outbox.PostgreSql/Readme-ru.md index 23e76814..4fc6efcf 100644 --- a/src/Sa.Outbox.PostgreSql/Readme-ru.md +++ b/src/Sa.Outbox.PostgreSql/Readme-ru.md @@ -24,8 +24,13 @@ IHost host = Host.CreateDefaultBuilder() .WithDeliveries(b => b .AddDeliveryScoped((_, s) => { - s.ScheduleSettings.WithIntervalSeconds(5).WithImmediate(); - s.ConsumeSettings.WithMaxBatchSize(16); + s + .WithInterval(TimeSpan.FromSeconds(5)) + .StartImmediately() + .WithMaxBatchSize(16) + .WithMaxDeliveryAttempts(3) + .WithLockDuration(TimeSpan.FromSeconds(10)) + .WithLookbackInterval(TimeSpan.FromDays(7)); }) ) ) @@ -61,7 +66,7 @@ public sealed record OrderCreated(string PayloadId, string ProductName); public sealed class OrderConsumer : IConsumer { public async ValueTask Consume( - ConsumerGroupSettings settings, + OutboxConsumerSettings settings, OutboxMessageFilter filter, ReadOnlyMemory> messages, CancellationToken ct) @@ -81,6 +86,7 @@ public sealed class OrderConsumer : IConsumer - Конкурентную обработку через `SKIP LOCKED` - Повтор / отложенный повтор / фиксацию ошибок - Автоматическую очистку старых партиций +- Self-bootstrapping настроек консьюмера в `IOutboxConsumerManager` --- @@ -125,46 +131,77 @@ public sealed class OrderConsumer : IConsumer // С настройками .AddDeliveryScoped((_, settings) => { - // Расписание опроса - settings.ScheduleSettings + settings .WithInterval(TimeSpan.FromSeconds(5)) - .WithImmediate(); // стартовать сразу - - // Ограничения потребления - settings.ConsumeSettings - .WithMaxBatchSize(16) // макс сообщений в батче - .WithMaxDeliveryAttempts(3) // стоп ретраев после N попыток - .WithBatchingWindow(TimeSpan.FromSeconds(2)) - .WithLockDuration(TimeSpan.FromMinutes(10)); + .StartImmediately() // стартовать сразу + .WithMaxBatchSize(16) // макс сообщений в батче + .WithMaxDeliveryAttempts(3) // стоп ретраев после N попыток + .WithLockDuration(TimeSpan.FromSeconds(10)) + .WithLookbackInterval(TimeSpan.FromDays(7)); }) ) // ... ) ``` -#### Справочник ConsumeSettings +#### Справочник настроек `OutboxConsumerSettingsBuilder` + +Единый fluent-билдер для всех параметров consumer group: | Метод | По умолчанию | Описание | |---|---|---| -| `WithInterval(interval)` | 5 с | Частота опроса | -| `WithImmediate()` | — | Не ждать первый интервал | -| `WithMaxBatchSize(n)` | 16 | Макс сообщений в батче | -| `WithMaxDeliveryAttempts(n)` | ∞ | Стоп ретраев после N попыток | -| `WithBatchingWindow(span)` | 2 с | Ждать до этого времени для заполнения батча | -| `WithLockDuration(span)` | 10 м | TTL блокировки задачи | -| `WithSingleIteration()` | — | Обработать один раз (для тестов) | +| `WithInterval(span)` | 1 мин | Период опроса | +| `StartImmediately()` | — | Старт без ожидания первого интервала | +| `WithMaxBatchSize(n)` | 16 | Макс. сообщений за батч | +| `WithMaxDeliveryAttempts(n)` | 3 | Стоп-повторы после N попыток | +| `WithLockDuration(span)` | 10 с | TTL блокировки сообщения | +| `WithLockRenewal(span)` | 3 с | Период продления блокировки | +| `WithLookbackInterval(span)` | 7 дн | История поиска необработанных | +| `WithBatchingWindow(span)` | 3 с | Окно агрегации сообщений | | `WithNoBatchingWindow()` | — | Взять всё доступное сейчас | - -#### Динамические изменения внутри `Consume()` +| `WithConcurrencyLimit(n)` | 1 | Одновременных задач | +| `WithMaxConcurrency(n)` | 48 | Макс. параллельных процессоров | +| `WithRetryCountOnError(n)` | 0 | Повторы при ошибке (-1 = бесконечно) | +| `WithNoRetries()` | — | Отключить повторы | +| `WithInfiniteRetries()` | — | Бесконечные повторы | +| `WithMaxProcessingIterations(n)` | 10 | Итераций за цикл (-1 = безлимитно) | +| `WithSingleIteration()` | — | Одна итерация (тестирование) | +| `WithUnlimitedIterations()` | — | Безлимитные итерации | +| `WithIterationDelay(span)` | 0 с | Задержка между итерациями | +| `WithPerTenantTimeout(span)` | 0 | Таймаут обработки одного тенанта | +| `WithPerTenantMaxDegreeOfParallelism(n)` | 1 | Параллельность по тенантам (1 = последовательно, -1 = все ядра) | +| `WithSequentialProcessing()` | 1 | Последовательная обработка по тенантам | +| `WithMaxParallelism()` | -1 | Максимальная параллельность по тенантам | +| `Paused(bool)` | false | Пауза consumer group | +| `Resumed()` | — | Снять с паузы | +| `AsSingleton(bool)` | true | Singleton (один на кластер) vs Scoped | + +#### Runtime-управление настройками + +Настройки автоматически регистрируются в `IOutboxConsumerManager` при первом запуске job'а. Для runtime-изменений: ```csharp -public async ValueTask Consume(ConsumerGroupSettings settings, ...) +var manager = host.Services.GetRequiredService(); + +// Atomic swap — новый снимок применяется на следующей итерации +manager.Apply("cg_order_consumer", s => s with { MaxBatchSize = 64 }); + +// Pause / Resume +manager.Pause("cg_order_consumer"); +manager.Resume("cg_order_consumer"); + +// Подписка на изменения +using var sub = manager.Subscribe("cg_order_consumer", updated => { - // Изменить поведение во время обработки - settings.ConsumeSettings.WithMaxProcessingIterations(100); -} + // реакция на изменение настроек +}); + +// Проверка состояния +bool paused = manager.IsPaused("cg_order_consumer"); ``` +> **Self-bootstrapping:** `DeliveryJob` автоматически регистрирует настройки в `IOutboxConsumerManager` при первом запуске. Используется `TryRegister` для безопасной конкурентной регистрации — если несколько инстансов приложения стартуют одновременно, только один успешно зарегистрирует настройки, остальные прочитают канонический снимок из менеджера. + ### 3. Подключение к PostgreSQL Внутри `AddSaOutboxUsingPostgreSql`: @@ -194,7 +231,7 @@ public async ValueTask Consume(ConsumerGroupSettings settings, ...) #### AOT-совместимый сериализатор -Для Native AOT избегайте рефлексии: +Для Native AHT избегайте рефлексии: ```csharp [JsonSourceGenerationOptions(GenerationMode = JsonSourceGenerationMode.Metadata)] @@ -262,7 +299,7 @@ public class OrderSerializer : IOutboxMessageSerializer settings.CleanupSettings.ExecutionInterval = TimeSpan.FromHours(4); // ── Минимальное смещение (без повторной обработки) ─ - settings.ConsumeSettings.WithMinOffset(DateTimeOffset.Now); + // settings.ConsumeSettings.WithMinOffset(DateTimeOffset.Now); }) ) ``` @@ -335,11 +372,15 @@ if (!migrationService.OnMigrated.IsCancellationRequested) | Метод | Когда использовать | Что дальше | |---|---|---| | `msg.Ok()` | Всё прошло успешно | Задача удалена | +| `msg.Created()` | Создан побочный ресурс | Задача удалена | +| `msg.Accepted()` | Принято в асинхронную обработку | Задача удалена | +| `msg.NoContent()` | Обработано, данных нет | Задача удалена | | `msg.Error(ex)` | Неустранимая ошибка | Запись в `__error$`, без повтора | | `msg.Warn(ex)` | Временная проблема (сеть, таймаут) | Повтор при следующем опросе | | `msg.Postpone(ts)` | Нужно подождать перед повтором | Повтор после `ts` | | `msg.Retry(ts, reason)` | Повтор с метаданными | Повтор с информацией о попытке | | `msg.Aborted(reason)` | Намеренно пропустить | Отмечено как пропущенное, без повтора | +| `msg.ErrorMaxAttempts()` | Исчерпан максимум попыток | Запись в `__error$`, без повтора | --- diff --git a/src/Sa.Outbox.PostgreSql/Readme.md b/src/Sa.Outbox.PostgreSql/Readme.md index e5610f71..0cae0631 100644 --- a/src/Sa.Outbox.PostgreSql/Readme.md +++ b/src/Sa.Outbox.PostgreSql/Readme.md @@ -363,11 +363,15 @@ After processing each message, call exactly one method: | Method | When to use | Next action | |---|---|---| | `msg.Ok()` | Everything went fine | Task removed | +| `msg.Created()` | Side-effect resource created | Task removed | +| `msg.Accepted()` | Accepted for async processing | Task removed | +| `msg.NoContent()` | Processed, no data to return | Task removed | | `msg.Error(ex)` | Unrecoverable failure | Logged to `__error$`, no retry | | `msg.Warn(ex)` | Transient issue (network, timeout) | Requeued, processed next poll | | `msg.Postpone(ts)` | Need to wait before retry | Requeued after `ts` | | `msg.Retry(ts, reason)` | Retry with metadata | Requeued with attempt info | | `msg.Aborted(reason)` | Intentionally skip | Marked skipped, no retry | +| `msg.ErrorMaxAttempts()` | Max attempts exhausted | Logged to `__error$`, no retry | --- diff --git a/src/Sa.Outbox/Delivery/IOutboxConsumerManager.cs b/src/Sa.Outbox/Delivery/IOutboxConsumerManager.cs index 91086586..9f3d7705 100644 --- a/src/Sa.Outbox/Delivery/IOutboxConsumerManager.cs +++ b/src/Sa.Outbox/Delivery/IOutboxConsumerManager.cs @@ -16,11 +16,14 @@ public interface IOutboxConsumerManager /// A function that receives the current snapshot and returns the updated one. Use this with { ... } expressions. void Apply(string consumerGroupId, Func transform); + + /// - /// Registers a consumer group with initial settings. - /// Unlike , this does not require prior registration. + /// Attempts to register a consumer group only if it is not already registered. + /// Thread-safe and idempotent — concurrent callers will see a consistent result. + /// Returns true if the group was newly registered, false if it already existed. /// - internal void Register(string consumerGroupId, OutboxConsumerSettings settings); + bool TryRegister(string consumerGroupId, OutboxConsumerSettings settings); /// /// Retrieves the current immutable settings snapshot. Thread-safe. diff --git a/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs b/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs index 498ba213..93442d5d 100644 --- a/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs +++ b/src/Sa.Outbox/Delivery/Job/DeliveryJob.cs @@ -17,11 +17,22 @@ public async Task Execute(IJobContext context, CancellationToken cancellationTok if (settings is null) { // Auto-bootstrap: first execution hasn't been registered yet. - settings = context.Settings.Properties.GetConsumerGroupSettings() + // Use TryRegister to avoid race conditions when multiple application + // instances start simultaneously — only one will succeed, others skip. + var bootstrapped = context.Settings.Properties.GetConsumerGroupSettings() ?? throw new InvalidOperationException( $"No OutboxConsumerSettings for consumer group '{context.JobName}'."); - settingsManager.Register(context.JobName, settings); + bool registered = settingsManager.TryRegister(context.JobName, bootstrapped); + if (!registered) + { + // Another instance registered us concurrently — read the canonical snapshot. + settings = settingsManager.Get(context.JobName)!; + } + else + { + settings = bootstrapped; + } } await processor.ProcessMessages(settings, cancellationToken).ConfigureAwait(false); diff --git a/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs b/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs index c484c782..90a4c37a 100644 --- a/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs +++ b/src/Sa.Outbox/Delivery/OutboxConsumerManager.cs @@ -11,7 +11,7 @@ internal sealed class OutboxConsumerManager : IOutboxConsumerManager private readonly Lock _lock = new(); /// - public void Register(string consumerGroupId, OutboxConsumerSettings settings) + public bool TryRegister(string consumerGroupId, OutboxConsumerSettings settings) { if (string.IsNullOrWhiteSpace(consumerGroupId)) throw new ArgumentException("Consumer group ID cannot be null or empty.", nameof(consumerGroupId)); @@ -20,6 +20,9 @@ public void Register(string consumerGroupId, OutboxConsumerSettings settings) lock (_lock) { + if (_settings.ContainsKey(consumerGroupId)) + return false; + _settings[consumerGroupId] = settings; if (!_listeners.ContainsKey(consumerGroupId)) @@ -30,6 +33,7 @@ public void Register(string consumerGroupId, OutboxConsumerSettings settings) // Notify subscribers OUTSIDE the lock to avoid deadlocks NotifyListeners(consumerGroupId, settings); + return true; } /// diff --git a/src/Sa.Outbox/Readme-ru.md b/src/Sa.Outbox/Readme-ru.md new file mode 100644 index 00000000..b95d822b --- /dev/null +++ b/src/Sa.Outbox/Readme-ru.md @@ -0,0 +1,251 @@ +# Sa.Outbox + +Базовая инфраструктурная библиотека для реализации паттерна **Transactional Outbox** в распределённых .NET-системах. Гарантирует атомарную запись сообщений вместе с бизнес-операциями в рамках одной транзакции БД, надёжную доставку, повторы, блокировки, многопоточность и поддержку мультитенантности. + +Определяет абстракции и базовую логику — конкретная работа с БД (PostgreSQL, SQL Server и т.д.) реализуется в пакетах-провайдерах (`Sa.Outbox.PostgreSql`, `Sa.Outbox.SqlServer` и др.). + +--- + +## Быстрый старт + +### 1. Установите пакет провайдера + +```bash +dotnet add package Sa.Outbox.PostgreSql +``` + +### 2. Настройте DI + +```csharp +builder.Services + .AddSaOutbox(builder => builder + .WithTenants((_, ts) => ts.WithTenantIds(1, 2, 3)) + .WithDeliveries(b => b.AddDelivery()) + ) + // Регистрация провайдера (пример — PostgreSQL) + .AddSaOutboxUsingPostgreSql(cfg => cfg + .WithDataSource(ds => ds.WithConnectionString("Host=localhost;Database=outbox")) + ); +``` + +--- + +## Архитектура + +``` +┌──────────────┐ Publish ┌─────────────┐ +│ Application │ ───────────────► │ outbox__msg$│ +│ │ │ (source) │ +│ IConsumer │ └──────┬──────┘ +│ │ │ RentDelivery (SKIP LOCKED) +│ │ ▼ +└──────────────┘ ┌─────────────┐ + ▲ │ outbox │ + └── Ack/Warn/Error ◄─────┤ (queue) │ + └─────────────┘ +``` + +### Два этапа жизненного цикла + +| Этап | Описание | +|------|---------| +| **Publication** | Сообщения записываются в таблицу outbox внутри транзакции бизнес-операции через `IOutboxBulkWriter.InsertBulk()` | +| **Delivery** | Фоновые задания (`Sa.Schedule`) захватывают заблокированные сообщения, вызывают консьюмеры, обновляют статус | + +--- + +## Основные типы + +| Тип | Назначение | +|-----|-----------| +| `IOutboxBuilder` | Fluent-билдер конфигурации | +| `IOutboxMessagePublisher` | Публикация сообщений в outbox | +| `IConsumer\` | Интерфейс консьюмера сообщений | +| `IOutboxContextOperations\` | Операции изменения статуса доставки (`Ok`, `Error`, `Warn`, `Postpone` и т.д.) | +| `OutboxConsumerSettings` | Immutable-снимок настроек группы консьюмера (интервал, батчи, параллелизм, повторы…) | +| `OutboxConsumerSettingsBuilder` | Fluent-билдер для создания и частичного обновления `OutboxConsumerSettings` | +| `IOutboxConsumerManager` | Runtime-менеджер: атомарный свап, пауза/возобновление, изменение подписок | +| `IDeliverySnapshot` | Read-only представление зарегистрированных доставок для диагностики | +| `DeliveryStatus` / `DeliveryStatusCode` | HTTP-подобные коды статуса доставки | +| `ExponentialBackoffRetryStrategy` | Экспоненциальный backoff с jitter | +| `OutboxPartInfo` | Информация о партиции: TenantId, PartName | + +--- + +## Коды статуса доставки + +Полный набор HTTP-подобных кодов статуса: + +| Код | Статус | Значение | +|-----|--------|---------| +| 200 | `Ok()` | Успешно обработано | +| 201 | `Created()` | Создан ресурс побочного эффекта | +| 202 | `Accepted()` | Принято для асинхронной обработки | +| 203 | `Ok203()` | Non-Authoritative Information | +| 204 | `NoContent()` | Обработано, нет данных | +| 299 | `Aborted()` | Намеренно пропущено | +| 301 | `MovedPermanently()` | Перемещено в другую очередь | +| 400 | `Warn()` | Временная ошибка → повтор | +| 500–507 | `ErrorXXX()` | Постоянная ошибка | +| 508 | `ErrorMaxAttempts()` | Исчерпан макс. число попыток | +| 103 | `Postpone()` | Отложенная обработка | +| 104 | `Retry()` | Повторить сейчас | + +--- + +## Методы статуса + +После обработки каждого сообщения вызовите ровно один метод из `IOutboxContextOperations`: + +| Метод | Описание | +|-------|---------| +| `msg.Ok(message?)` | Успешно обработано (200 OK) | +| `msg.Created(message?)` | Создан ресурс побочного эффекта (201 Created) | +| `msg.Accepted(message?)` | Принято для асинхронной обработки (202 Accepted) | +| `msg.NoContent(message?)` | Обработано, нет данных (204 No Content) | +| `msg.Aborted(message?)` | Намеренно пропущено (299 Aborted) | +| `msg.Warn(exception, message?, postpone?)` | Временная ошибка → повтор (400 Warn) | +| `msg.Error(exception, message?)` | Постоянная ошибка (500 Error) | +| `msg.ErrorMaxAttempts()` | Исчерпан макс. число попыток (508) | +| `msg.Postpone(delay, message?)` | Отложить обработку (103 Postpone) | +| `msg.Retry(delay, message?)` | Повторить с метаданными (104 Retry) | + +--- + +## Конфигурация + +### Регистрация консьюмеров + +```csharp +builder.Services.AddSaOutbox(builder => builder + .WithDeliveries(d => d + // Singleton delivery (один инстанс на всё приложение) + .AddDelivery("orders", (sp, cs) => { + cs + .WithMaxBatchSize(32) + .WithLockDuration(TimeSpan.FromSeconds(10)) + .WithMaxDeliveryAttempts(5) + .WithInterval(TimeSpan.FromSeconds(30)) + .WithInitialDelay(TimeSpan.FromSeconds(5)); + }) + // Scoped delivery (DI-scoped на каждую доставку) + .AddDeliveryScoped("events") + ) +); +``` + +> **Self-bootstrapping:** `DeliveryJob` автоматически регистрирует настройки в `IOutboxConsumerManager` при первом выполнении. Отдельный сервис инициализации не нужен — каждое задание читает живые снимки из менеджера, включая runtime-изменения через `Apply()`. + +### Управление настройками во время выполнения + +`IOutboxConsumerManager` позволяет менять настройки без перезапуска: + +```csharp +// Атомарный свап — новый снимок применяется атомарно +manager.Apply("orders", s => s with { MaxBatchSize = 64 }); + +// Пауза / Возобновление +manager.Pause("orders"); +manager.Resume("orders"); + +// Подписка на изменения +using var sub = manager.Subscribe("orders", updated => +{ + // реакция на изменение настроек +}); + +// Проверка состояния +bool paused = manager.IsPaused("orders"); +bool registered = manager.IsRegistered("orders"); + +// Удаление (удаляет настройки И.detach внешний контроль) +manager.Unregister("orders"); + +// Список всех групп +var allGroups = manager.GetAllConsumerGroupIds(); +``` + +### Настройки потребления + +`OutboxConsumerSettings` — единый immutable record. Все параметры задаются через `OutboxConsumerSettingsBuilder`: + +| Параметр | По умолчанию | Описание | +|----------|-------------|---------| +| `ConsumerGroupId` | — | Уникальный идентификатор группы | +| `AsSingleton` | true | Singleton (один на кластер) vs Scoped | +| `Interval` | 1 мин | Период выполнения между итерациями | +| `InitialDelay` | 10 сек | Задержка перед первым выполнением | +| `ConcurrencyLimit` | 1 | Количество параллельных воркеров | +| `MaxConcurrency` | 48 | Абсолютный потолок процессоров | +| `IterationDelay` | 0 сек | Задержка между итерациями в цикле | +| `MaxProcessingIterations` | 10 | Итераций за цикл (-1 = бесконечно) | +| `LockDuration` | 10 сек | TTL блокировки записи | +| `LockRenewal` | 3 сек | Интервал продления блокировки | +| `LookbackInterval` | 7 дней | Окно поиска истории необработанных сообщений | +| `MaxDeliveryAttempts` | 3 | Макс. попыток доставки перед DLQ | +| `MaxBatchSize` | 16 | Макс. сообщений в батче | +| `BatchingWindow` | 3 сек | Окно агрегации сообщений | +| `PerTenantTimeout` | 0 | Таймаут на обработку одного арендатора | +| `PerTenantMaxDegreeOfParallelism` | 1 | Параллелизм по арендаторам (1 = последовательно, -1 = все ядра) | +| `RetryCountOnError` | 0 | Повторы при ошибке (-1 = бесконечно) | +| `Paused` | false | Флаг паузы | + +--- + +### Мультитенантность + +```csharp +.WithTenants((_, ts) => ts + .WithTenantIds(1, 2, 3) // Явный список + .WithAutoDetect() // Автодетект из сообщений в рантайме + .WithTenantDetector() // Кастомный детектор + .WithTenantParallelProcessing(3) // Параллельная обработка по арендаторам +) +``` + +--- + +### Метаданные сообщений + +```csharp +// Вариант 1: явный partName и резолвер PayloadId +options.AddMetadata(partName: "orders", getPayloadId: m => m.Id); + +// Вариант 2: вывод из IOutboxPublishable +options.AddMetadata(); +``` + +--- + +## Доступные провайдеры + +| Провайдер | Пакет | Статус | +|-----------|-------|--------| +| PostgreSQL | `Sa.Outbox.PostgreSql` | ✅ production-ready | +| SQL Server | `Sa.Outbox.SqlServer` | 🔧 в разработке | +| Redis | `Sa.Outbox.Redis` | 🔧 в разработке | + +--- + +## Требования к провайдерам + +Провайдер должен реализовать три ключевых интерфейса: + +| Интерфейс | Назначение | +|-----------|-----------| +| `IOutboxBulkWriter` | Массовая вставка сообщений в БД | +| `IOutboxDeliveryManager` | Блокировки и диспетчеризация сообщений | +| `ITenantSource` | Источник ID арендатора | + +--- + +## Зависимости + +- **Sa.Schedule** — планировщик фоновых заданий +- Базовые классы из **Sa** (LockRenewer, MurmurHash3, Retry, расширения) + +--- + +## Лицензия + +MIT diff --git a/src/Sa.Outbox/Readme.md b/src/Sa.Outbox/Readme.md index 279ee957..881eaeaa 100644 --- a/src/Sa.Outbox/Readme.md +++ b/src/Sa.Outbox/Readme.md @@ -1,18 +1,20 @@ # Sa.Outbox -Базовая инфраструктурная библиотека для реализации паттерна **Transactional Outbox** в распределённых .NET-системах. Гарантирует атомарную запись сообщения вместе с бизнес-операцией внутри одной транзакции БД и надёжную доставку с поддержкой повторных попыток, блокировок, многопоточности и мультитенантности. +Base infrastructure library for implementing the **Transactional Outbox** pattern in distributed .NET systems. Guarantees atomic message recording alongside business operations within a single database transaction, with reliable delivery, retries, locking, multi-threading, and multi-tenancy support. -Библиотека определяет абстракции и логику — конкретную работу с БД (PostgreSQL, SQL Server и т.д.) реализуют провайдеры-наследники (`Sa.Outbox.PostgreSql`, `Sa.Outbox.SqlServer` и др.). +Defines abstractions and core logic — concrete database work (PostgreSQL, SQL Server, etc.) is implemented by provider packages (`Sa.Outbox.PostgreSql`, `Sa.Outbox.SqlServer`, etc.). + +--- ## Quick Start -### 1. Установите пакет провайдера +### 1. Install a provider package ```bash dotnet add package Sa.Outbox.PostgreSql ``` -### 2. Настройте DI +### 2. Configure DI ```csharp builder.Services @@ -20,13 +22,15 @@ builder.Services .WithTenants((_, ts) => ts.WithTenantIds(1, 2, 3)) .WithDeliveries(b => b.AddDelivery()) ) - // провайдер (пример — PostgreSQL) + // Provider registration (example — PostgreSQL) .AddSaOutboxUsingPostgreSql(cfg => cfg .WithDataSource(ds => ds.WithConnectionString("Host=localhost;Database=outbox")) ); ``` -## Архитектура +--- + +## Architecture ``` ┌──────────────┐ Publish ┌─────────────┐ @@ -41,54 +45,81 @@ builder.Services └─────────────┘ ``` -### Два этапа жизненного цикла - -| Этап | Описание | -|------|----------| -| **Publication** | Сообщения записываются в таблицу outbox внутри транзакции бизнес-операции через `IOutboxBulkWriter.InsertBulk()` | -| **Delivery** | Фоновые задачи (`Sa.Schedule`) захватывают заблокированные сообщения, вызывают потребителей, обновляют статус | - -## Основные типы - -| Тип | Назначение | -|-----|------------| -| `IOutboxBuilder` | Fluent-билдер для конфигурации outbox-системы | -| `IOutboxMessagePublisher` | Публикация сообщений в outbox | -| `IConsumer\` | Интерфейс потребителя сообщений | -| `IOutboxContextOperations\` | Операции изменения статуса доставки | -| `OutboxConsumerSettings` | Единый immutable-снимок настроек consumer group (интервал, батчи, параллельность, повторы и т.д.) | -| `OutboxConsumerSettingsBuilder` | Fluent-билдер для создания и частичного обновления `OutboxConsumerSettings` | -| `IOutboxConsumerManager` | Runtime-менеджер настроек: atomic swap, pause/resume, подписки на изменения | -| `DeliverySnapshot` | Считывает настройки из статического регистра Schedule после билда DI | -| `DeliveryStatus` / `DeliveryStatusCode` | HTTP-подобные статусы доставки | -| `ExponentialBackoffRetryStrategy` | Экспоненциальный бэкофф с джиттером | -| `OutboxPartInfo` | Информация о части: TenantId, PartName | - -## Статусы доставки - -Полный набор HTTP-подобных кодов состояния: - -| Код | Статус | Значение | -|-----|--------|----------| -| 200 | `Ok()` | Успешно обработано | -| 201 | `Created()` | Создан побочный ресурс | -| 202 | `Accepted()` | Принято в обработку | -| 204 | `NoContent()` | Обработано, нет данных | -| 299 | `Aborted()` | Пропущено | -| 400 | `Warn()` | Временная ошибка → повтор | -| 500–508 | `Error()` | Постоянная ошибка | -| 508 | `ErrorMaxAttempts()` | Исчерпан максимум попыток | -| 103 | `Postpone()` | Отложенная обработка | -| 104 | `Retry()` | Повторить сейчас | - -## Конфигурация - -### Настройка потребителей +### Two lifecycle stages + +| Stage | Description | +|-------|-------------| +| **Publication** | Messages are written to the outbox table inside the business operation's transaction via `IOutboxBulkWriter.InsertBulk()` | +| **Delivery** | Background jobs (`Sa.Schedule`) acquire locked messages, invoke consumers, update status | + +--- + +## Key Types + +| Type | Purpose | +|------|---------| +| `IOutboxBuilder` | Fluent configuration builder | +| `IOutboxMessagePublisher` | Publish messages to outbox | +| `IConsumer\` | Message consumer interface | +| `IOutboxContextOperations\` | Delivery status change operations (`Ok`, `Error`, `Warn`, `Postpone`, etc.) | +| `OutboxConsumerSettings` | Immutable snapshot of consumer group settings (interval, batches, concurrency, retries…) | +| `OutboxConsumerSettingsBuilder` | Fluent builder for creating and partially updating `OutboxConsumerSettings` | +| `IOutboxConsumerManager` | Runtime manager: atomic swap, pause/resume, change subscriptions | +| `IDeliverySnapshot` | Read-only view of registered deliveries for diagnostics | +| `DeliveryStatus` / `DeliveryStatusCode` | HTTP-like delivery status codes | +| `ExponentialBackoffRetryStrategy` | Exponential backoff with jitter | +| `OutboxPartInfo` | Part info: TenantId, PartName | + +--- + +## Delivery Status Codes + +Full set of HTTP-like status codes: + +| Code | Status | Meaning | +|------|--------|---------| +| 200 | `Ok()` | Successfully processed | +| 201 | `Created()` | Side-effect resource created | +| 202 | `Accepted()` | Accepted for async processing | +| 203 | `Ok203()` | Non-Authoritative Information | +| 204 | `NoContent()` | Processed, no data | +| 299 | `Aborted()` | Intentionally skipped | +| 301 | `MovedPermanently()` | Moved to another queue | +| 400 | `Warn()` | Transient error → retry | +| 500–507 | `ErrorXXX()` | Permanent error | +| 508 | `ErrorMaxAttempts()` | Max attempts exhausted | +| 103 | `Postpone()` | Deferred processing | +| 104 | `Retry()` | Retry now | + +--- + +## Status Methods + +After processing each message, call exactly one method from `IOutboxContextOperations`: + +| Method | Description | +|--------|-------------| +| `msg.Ok(message?)` | Successfully processed (200 OK) | +| `msg.Created(message?)` | Side-effect resource created (201 Created) | +| `msg.Accepted(message?)` | Accepted for async processing (202 Accepted) | +| `msg.NoContent(message?)` | Processed, no data (204 No Content) | +| `msg.Aborted(message?)` | Intentionally skipped (299 Aborted) | +| `msg.Warn(exception, message?, postpone?)` | Transient error → retry (400 Warn) | +| `msg.Error(exception, message?)` | Permanent error (500 Error) | +| `msg.ErrorMaxAttempts()` | Max attempts exhausted (508) | +| `msg.Postpone(delay, message?)` | Defer processing (103 Postpone) | +| `msg.Retry(delay, message?)` | Retry with metadata (104 Retry) | + +--- + +## Configuration + +### Consumer Registration ```csharp builder.Services.AddSaOutbox(builder => builder .WithDeliveries(d => d - // Singleton delivery (один экземпляр на всё приложение) + // Singleton delivery (one instance for the whole app) .AddDelivery("orders", (sp, cs) => { cs .WithMaxBatchSize(32) @@ -97,95 +128,123 @@ builder.Services.AddSaOutbox(builder => builder .WithInterval(TimeSpan.FromSeconds(30)) .WithInitialDelay(TimeSpan.FromSeconds(5)); }) - // Scoped delivery (DI-скон на каждую доставку) + // Scoped delivery (DI-scoped per delivery) .AddDeliveryScoped("events") ) ); ``` -> **Self-bootstrapping:** `DeliveryJob` автоматически регистрирует настройки в `IOutboxConsumerManager` при первом запуске. Отдельный bootstrap-сервис не нужен — каждый job читает актуальные снимки из менеджера, включая runtime-изменения через `Apply()`. +> **Self-bootstrapping:** `DeliveryJob` automatically registers settings into `IOutboxConsumerManager` on first execution. No separate bootstrap service needed — each job reads live snapshots from the manager, including runtime changes via `Apply()`. -### Runtime-управление настройками +### Runtime Settings Management -`IOutboxConsumerManager` позволяет изменять настройки без перезапуска: +`IOutboxConsumerManager` allows changing settings without restarting: ```csharp -// Atomic swap — новый снимок применяется атомарно +// Atomic swap — new snapshot applied atomically manager.Apply("orders", s => s with { MaxBatchSize = 64 }); // Pause / Resume manager.Pause("orders"); manager.Resume("orders"); -// Подписка на изменения +// Subscribe to changes using var sub = manager.Subscribe("orders", updated => { - // реакция на изменение настроек + // react to settings change }); -``` -### Настройки потребления +// Check state +bool paused = manager.IsPaused("orders"); +bool registered = manager.IsRegistered("orders"); -`OutboxConsumerSettings` — единый immutable record. Все параметры задаются через `OutboxConsumerSettingsBuilder`: +// Unregister (removes settings AND detaches external control) +manager.Unregister("orders"); -| Параметр | По умолчанию | Описание | -|----------|-------------|----------| -| `MaxBatchSize` | 16 | Макс. размер батча | -| `LockDuration` | 10 сек | Время блокировки сообщения | -| `LockRenewal` | 3 сек | Период продления блокировки | -| `MaxDeliveryAttempts` | 3 | Максимум попыток доставки | -| `LookbackInterval` | 7 дней | История обработки | -| `ConcurrencyLimit` | 1 | Одновременных задач | -| `MaxConcurrency` | 1 | Макс. параллельных процессоров | -| `PerTenantMaxDegreeOfParallelism` | 1 | Параллельность по тенантам | -| `RetryCountOnError` | 0 | Повторы при ошибке (-1 = бесконечно) | -| `MaxProcessingIterations` | -1 | Итераций за цикл (-1 = безлимитно) | -| `BatchingWindow` | 0 сек | Окно агрегации сообщений | -| `Paused` | false | Флаг паузы consumer group | +// List all registered groups +var allGroups = manager.GetAllConsumerGroupIds(); +``` -### Мультитенантность +### Consumption Settings + +`OutboxConsumerSettings` is a single immutable record. All parameters are set through `OutboxConsumerSettingsBuilder`: + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `ConsumerGroupId` | — | Unique group identifier | +| `AsSingleton` | true | Singleton (one per cluster) vs Scoped | +| `Interval` | 1 min | Execution period between iterations | +| `InitialDelay` | 10 sec | Delay before first execution | +| `ConcurrencyLimit` | 1 | Number of parallel workers | +| `MaxConcurrency` | 48 | Absolute ceiling of processors | +| `IterationDelay` | 0 sec | Delay between iterations within a cycle | +| `MaxProcessingIterations` | 10 | Iterations per cycle (-1 = unlimited) | +| `LockDuration` | 10 sec | Record lock TTL | +| `LockRenewal` | 3 sec | Lock renewal interval | +| `LookbackInterval` | 7 days | History search window for unprocessed messages | +| `MaxDeliveryAttempts` | 3 | Max delivery attempts before DLQ | +| `MaxBatchSize` | 16 | Max messages per batch | +| `BatchingWindow` | 3 sec | Message aggregation window | +| `PerTenantTimeout` | 0 | Timeout per tenant processing | +| `PerTenantMaxDegreeOfParallelism` | 1 | Tenant parallelism (1 = sequential, -1 = all cores) | +| `RetryCountOnError` | 0 | Retries on error (-1 = infinite) | +| `Paused` | false | Pause flag | + +--- + +### Multi-Tenancy ```csharp .WithTenants((_, ts) => ts - .WithTenantIds(1, 2, 3) // Явный список - .WithAutoDetect() // Автоопределение из БД - .WithTenantDetector() // Кастомный детектор - .WithTenantParallelProcessing(3) // Параллельная обработка + .WithTenantIds(1, 2, 3) // Explicit list + .WithAutoDetect() // Auto-detect from messages at runtime + .WithTenantDetector() // Custom detector + .WithTenantParallelProcessing(3) // Parallel processing per tenant ) ``` -### Метаданные сообщений +--- + +### Message Metadata ```csharp -// Вариант 1: явное указание partName и PayloadId +// Option 1: explicit partName and PayloadId resolver options.AddMetadata(partName: "orders", getPayloadId: m => m.Id); -// Вариант 2: из IOutboxPublishable +// Option 2: derive from IOutboxPublishable options.AddMetadata(); ``` -## Доступные провайдеры +--- -| Провайдер | Пакет | Статус | -|-----------|-------|--------| +## Available Providers + +| Provider | Package | Status | +|----------|---------|--------| | PostgreSQL | `Sa.Outbox.PostgreSql` | ✅ production-ready | -| SQL Server | `Sa.Outbox.SqlServer` | 🔧 в разработке | -| Redis | `Sa.Outbox.Redis` | 🔧 в разработке | +| SQL Server | `Sa.Outbox.SqlServer` | 🔧 in development | +| Redis | `Sa.Outbox.Redis` | 🔧 in development | + +--- + +## Provider Requirements + +A provider must implement three key interfaces: -## Требования к провайдеру +| Interface | Purpose | +|-----------|---------| +| `IOutboxBulkWriter` | Bulk message insertion into DB | +| `IOutboxDeliveryManager` | Locking and message dispatch | +| `ITenantSource` | Tenant ID source | -Провайдер должен реализовать три ключевых интерфейса: +--- -| Интерфейс | Назначение | -|-----------|------------| -| `IOutboxBulkWriter` | Массовая вставка сообщений в БД | -| `IOutboxDeliveryManager` | Управление блокировкой и выдачей сообщений | -| `ITenantSource` | Источник идентификаторов тенантов | +## Dependencies -## Зависимости +- **Sa.Schedule** — background job scheduler +- Reference classes from **Sa** (LockRenewer, MurmurHash3, Retry, extensions) -- **Sa.Schedule** — планировщик фоновых задач -- Ссылочные классы из **Sa** (LockRenewer, MurmurHash3, Retry, расширения) +--- ## License diff --git a/src/Sa.Partitional.PostgreSql/Readme-ru.md b/src/Sa.Partitional.PostgreSql/Readme-ru.md new file mode 100644 index 00000000..5ccacfa6 --- /dev/null +++ b/src/Sa.Partitional.PostgreSql/Readme-ru.md @@ -0,0 +1,267 @@ +# Sa.Partitional.PostgreSql + +Библиотека декларативного партиционирования таблиц PostgreSQL для .NET 10 — поддерживает **range** (день / месяц / год) и **list** партиционирование с автоматической миграцией, планировщиком очистки и in-memory кэшированием. + +--- + +## Обзор + +Большие таблицы PostgreSQL теряют производительность по мере роста. Эта библиотека автоматизирует полный жизненный цикл партиционирования: + +1. **Объявление** партиционируемых таблиц через fluent builder. +2. **Миграция** — автоматическое создание отсутствующих партиций перед поступлением данных. +3. **Кэширование** — хранение метаданных партиций в памяти для избежания повторных запросов к каталогу. +4. **Очистка** — удаление старых партиций за пределами настраиваемого окна удержания. + +Всё подключается в ASP.NET Core `IServiceCollection` через единственный метод-расширение. + +--- + +## Быстрый старт + +```csharp +builder.Services.AddSaPartitional((sp, builder) => +{ + builder.AddSchema("public", schema => + { + // Таблица с range-партиционированием (по умолчанию ежедневно) + schema.CreateTable("events") + .PartByRange(PgPartBy.Day) + .WithFillFactor(90); + }); +}) +// Предварительное создание будущих партиций как фоновая задача +.AddPartMigrationSchedule((sp, opts) => opts.AsBackgroundJob = true) +// Удаление партиций старше 30 дней +.AddPartCleanupSchedule((sp, opts) => opts.AsBackgroundJob = true); +``` + +--- + +## Поддерживаемые стратегии + +| Стратегия | Описание | Пример | +|-----------|----------|--------| +| **Range** | Партиционирование по временным интервалам — день, месяц или год | `events_y2026m06d26`, `events_y2026m07` | +| **List** | Партиционирование по дискретным значениям ключей (строки или числа) | `orders_RU`, `orders_USA` | + +Обе стратегии можно комбинировать иерархически: root-таблица с list-партиционированием может иметь range-разделённых детей. + +--- + +## Документация + +| Документ | Содержимое | +|----------|-----------| +| [Guide](Guide.md) | Конфигурация, fluent builder, StrOrNum, соглашения об именовании, примеры DDL | +| [API Reference](ApiReference.md) | Сигнатуры интерфейсов, диаграмма архитектуры, ключевые типы | + +--- + +## Ключевые типы + +### IPartitionManager + +Главная точка входа для программного управления партициями: + +```csharp +public interface IPartitionManager +{ + Task Migrate(CancellationToken ct = default); + Task Migrate(DateTimeOffset[] dates, CancellationToken ct = default); + Task EnsureParts(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken ct = default); +} +``` + +- `Migrate()` — предварительное создание всех отсутствующих партиций на сегодня + окно вперёд +- `Migrate(dates[])` — предварительное создание только для конкретных дат +- `EnsureParts()` — гарантировать существование конкретной партиции (создаёт при необходимости) + +### PgPartBy + +Enum стратегии партиционирования с тремя предопределёнными значениями: + +| Значение | Формат имени | Пример | Диапазон | +|----------|-------------|--------|---------| +| `PgPartBy.Day` | `yYYYYmmDD` | `events_y2026m06d26` | StartOfDay → +1 день | +| `PgPartBy.Month` | `yYYYYmm` | `events_y2026m07` | StartOfMonth → +1 месяц | +| `PgPartBy.Year` | `yYYYY` | `events_y2026` | StartOfYear → +1 год | + +Дополнительные фабричные методы: +```csharp +PgPartBy.FromRange(PartByRange.Day); // из PartByRange enum +PgPartBy.FromPartName("root"); // из строки имени партиции +``` + +### StrOrNum + +Discriminated union для ключей list-партиционирования — поддерживает и строковые, и числовые значения: + +```csharp +// Implicit conversions +StrOrNum s = "tenant_a"; // → ChoiceStr +StrOrNum n = 42L; // → ChoiceNum + +// Pattern matching +result.Match( + onChoiceStr: v => Console.WriteLine($"String: {v}"), + onChoiceNum: v => Console.WriteLine($"Number: {v}") +); + +// Форматирование +s.ToFmtString(); // "s:tenant_a" +StrOrNum.FromFmtStr("n:42"); // → ChoiceNum(42) +``` + +Поддерживаемые implicit conversions: `string`, `int`, `long`, `short`. + +--- + +## Fluent Builder API + +Регистрация: `Setup.AddSaPartitional()` возвращает `IPartConfiguration`. + +```csharp +services.AddSaPartitional((sp, builder) => +{ + builder.AddSchema("outbox", schema => + { + schema.CreateTable("messages") + .AddFields("tenant_id varchar(50) NOT NULL") + .PartByList("tenant_id") + .WithFillFactor(80) + .AddMigration("tenant_a", new StrOrNum[] { "tenant_a_1", "tenant_a_2" }) + .AddMigration(new[] { "tenant_b" }); + }); +}); +``` + +### Методы ITableBuilder + +| Метод | Описание | +|-------|----------| +| `AddFields(params string[])` | Определения колонок (например, `"tenant_id varchar(50) NOT NULL"`) | +| `PartByRange(PgPartBy, fieldName?)` | Range-стратегия (Day/Month/Year) | +| `PartByList(params string[])` | List-партиционирование по колонке(ам) | +| `TimestampAs(fieldName)` | Переопределение имени timestamp-колонки (по умолч.: `created_at`) | +| `WithPartSeparator(string)` | Разделитель между частями в именах (по умолч.: `"__"`) | +| `WithFillFactor(int)` | Параметр хранения PostgreSQL fill factor | +| `WithPartTablePostfix(string)` | Суффикс для кэш-/партиционных таблиц (по умолч.: `"__part"`) | +| `AddPostSql(Func)` | Дополнительный SQL после CREATE TABLE | +| `AddConstraintPkSql(Func)` | Пользовательский CHECK / PK constraint SQL | +| `AddMigration(IPartTableMigrationSupport)` | Пользовательские значения миграции | +| `AddMigration(Func>)` | Асинхронная фабрика значений миграции | +| `AddMigration(params StrOrNum[])` | Inline значения list-партиций | +| `AddMigration(StrOrNum parent, StrOrNum[] childs)` | Иерархическая миграция (parent + дети) | +| `Build()` | Финализация настроек таблицы | + +--- + +## Настройки планировщика + +### MigrationScheduleSettings + +Управляет автоматическим предварительным созданием будущих партиций: + +| Свойство | По умолчанию | Описание | +|----------|-------------|----------| +| `ForwardDays` | `2` | Дней вперёд для пре-создания партиций | +| `AsBackgroundJob` | `false` | Запуск как hosted service | +| `MigrationJobName` | `"Migration job"` | Идентификатор задачи | +| `ExecutionInterval` | `~4h + jitter` | Интервал между миграциями | +| `WaitMigrationTimeout` | `3 сек` | Таймаут ожидания semaphore | + +### PartCleanupScheduleSettings + +Управляет автоматическим удалением старых партиций: + +| Свойство | По умолчанию | Описание | +|----------|-------------|----------| +| `DropPartsAfterRetention` | `30 дней` | Порог возраста для удаления | +| `AsBackgroundJob` | `false` | Запуск как hosted service | +| `ExecutionInterval` | `~4h + jitter` | Интервал между очистками | +| `InitialDelay` | `1 мин` | Задержка перед первым запуском | + +### PartCacheSettings + +In-memory кэш метаданных партиций: + +| Свойство | По умолчанию | Описание | +|----------|-------------|----------| +| `CachedFromDate` | `1 день` | Насколько далеко вперёд загружать партиции | + +--- + +## Соглашения об именовании + +Партиции следуют предсказуемым паттернам именования (разделитель по умолчанию `"__"`): + +| Компонент | Паттерн | Пример | +|-----------|---------|--------| +| Range (день) | `{table}{sep}y{YYYY}m{MM}d{DD}` | `events__part__y2026m06d26` | +| Range (месяц) | `{table}{sep}y{YYYY}m{MM}` | `events__part__y2026m07` | +| Range (год) | `{table}{sep}y{YYYY}` | `events__part__y2026` | +| List (вложенный) | `{table}{sep}{val1}_{val2}...` | `orders__part__EU_EU_1` | +| Кэш-таблица | `{table}{postfix}` | `events__part` | + +**Ограничения:** +- Идентификаторы не должны превышать 63 символа (лимит PostgreSQL). +- Схемы автоматически создаются через `CREATE SCHEMA IF NOT EXISTS`. +- Детские партиции используют `PARTITION OF parent FOR VALUES FROM (...) TO (...)` (range) или `FOR VALUES IN (...)` (list). +- После создания каждой range-партиции кэш-таблица отслеживает границы через `INSERT ... ON CONFLICT (id) DO NOTHING`. + +--- + +## Архитектура + +``` +┌─────────────────────┐ +│ IPartitionManager │ ← Публичная точка входа +├─────────────────────┤ +│ IMigrationService │ ← Пре-создание будущих партиций +│ IPartCleanupService│ ← Удаление старых партиций +├─────────────────────┤ +│ IPartRepository │ ← Выполнение DDL (CREATE/DROP PARTITION) +│ ISqlBuilder │ ← Генерация SQL-шаблонов +│ IPartCache │ ← In-memory кэш метаданных +├─────────────────────┤ +│ MigrationJob │ ← IJob обёртка для Sa.Schedule +│ PartCleanupJob │ ← IJob обёртка для Sa.Schedule +└─────────────────────┘ +``` + +--- + +## Зависимости + +- `Sa.Data.PostgreSql` — обёртка Npgsql со стратегией повторов +- `Sa.Schedule` — инфраструктура фоновых задач + +--- + +## Структура проекта + +``` +src/Sa.Partitional.PostgreSql/ +├── Setup.cs # Главная DI-точка AddSaPartitional() +├── IPartitionManager.cs # Публичный API управления партициями +├── PgPartBy.cs # Enum стратегии партиционирования +├── Classes/ +│ ├── StrOrNum.cs # Discriminated union (string | long) +│ └── Enumeration.cs # Шаблон type-safe base enum +├── Configuration/ # Fluent builder API +│ ├── IPartConfiguration.cs +│ └── Builder/ # ISettingsBuilder, ISchemaBuilder, ITableBuilder +├── Settings/ # ITableSettings, ITableSettingsStorage +├── Cache/ # In-memory кэш: PartCache, PartCacheSettings +├── Migration/ # Пре-создание: IMigrationService, MigrationJob +├── Cleaning/ # Удаление старых: IPartCleanupService, PartCleanupJob +├── Partitional/ # DDL repo: IPartRepository, PartByRangeInfo +└── SqlBuilder/ # SQL-шаблоны: ISqlBuilder, SqlTemplate.cs +``` + +--- + +## Лицензия + +MIT diff --git a/src/Sa.Partitional.PostgreSql/Readme.md b/src/Sa.Partitional.PostgreSql/Readme.md index 3a461702..e2349fd3 100644 --- a/src/Sa.Partitional.PostgreSql/Readme.md +++ b/src/Sa.Partitional.PostgreSql/Readme.md @@ -2,6 +2,8 @@ Declarative PostgreSQL table partitioning library for .NET 10 — supports **range** (day / month / year) and **list** partitioning with automated migration, cleanup scheduling, and in-memory caching. +--- + ## Overview Large PostgreSQL tables lose performance as they grow. This library automates the full partition lifecycle: @@ -13,6 +15,8 @@ Large PostgreSQL tables lose performance as they grow. This library automates th Everything wires into ASP.NET Core `IServiceCollection` through a single extension method. +--- + ## Quick Start ```csharp @@ -32,6 +36,8 @@ builder.Services.AddSaPartitional((sp, builder) => .AddPartCleanupSchedule((sp, opts) => opts.AsBackgroundJob = true); ``` +--- + ## Supported Strategies | Strategy | Description | Example | @@ -41,16 +47,217 @@ builder.Services.AddSaPartitional((sp, builder) => Both strategies can be combined hierarchically: a list-partitioned root can have range-partitioned children. -## Documentation +--- + +## Fluent Builder API + +### Schema + Table Declaration + +```csharp +services.AddSaPartitional((sp, builder) => +{ + builder.AddSchema("outbox", schema => + { + // Range-partitioned by day + schema.CreateTable("messages") + .AddFields("tenant_id varchar(50) NOT NULL") + .PartByRange(PgPartBy.Day, "created_at") + .WithFillFactor(80); + + // List-partitioned by tenant + schema.CreateTable("orders") + .AddFields("region varchar(10) NOT NULL") + .PartByList("region") + .AddMigration("EU", "US", "APAC"); + }); +}); +``` + +### ITableBuilder Methods + +| Method | Description | +|--------|-------------| +| `AddFields(params string[])` | Column definitions (e.g., `"tenant_id varchar(50) NOT NULL"`) | +| `PartByRange(PgPartBy, fieldName?)` | Range partitioning strategy (Day/Month/Year) | +| `PartByList(params string[])` | List partitioning on column(s) | +| `TimestampAs(fieldName)` | Override timestamp column name (default: `created_at`) | +| `WithPartSeparator(string)` | Separator between parts in names (default: `"__"`) | +| `WithFillFactor(int)` | PostgreSQL fill factor storage parameter | +| `WithPartTablePostfix(string)` | Suffix for cache/partition tables (default: `"__part"`) | +| `AddPostSql(Func)` | Extra SQL after CREATE TABLE | +| `AddConstraintPkSql(Func)` | Custom CHECK / PK constraint SQL | +| `AddMigration(IPartTableMigrationSupport)` | Provide custom migration values | +| `AddMigration(Func>)` | Async factory for migration values | +| `AddMigration(params StrOrNum[])` | Inline list partition values | +| `AddMigration(StrOrNum parent, StrOrNum[] childs)` | Hierarchical migration (parent + children) | +| `Build()` | Finalize table settings | + +--- + +## Key Types + +### IPartitionManager + +Main entry point for programmatic partition management: + +```csharp +public interface IPartitionManager +{ + Task Migrate(CancellationToken ct = default); + Task Migrate(DateTimeOffset[] dates, CancellationToken ct = default); + Task EnsureParts(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken ct = default); +} +``` + +- `Migrate()` — pre-create all missing partitions for today + forward window +- `Migrate(dates[])` — pre-create for specific dates only +- `EnsureParts()` — guarantee a specific partition exists (creates it if missing) + +### PgPartBy + +Partitioning strategy enum with three predefined values: + +| Value | Format Pattern | Example Name | Range | +|-------|---------------|--------------|-------| +| `PgPartBy.Day` | `yYYYYmmDD` | `events_y2026m06d26` | StartOfDay → +1 day | +| `PgPartBy.Month` | `yYYYYmm` | `events_y2026m07` | StartOfMonth → +1 month | +| `PgPartBy.Year` | `yYYYY` | `events_y2026` | StartOfYear → +1 year | + +Additional factory methods: +```csharp +PgPartBy.FromRange(PartByRange.Day); // from PartByRange enum +PgPartBy.FromPartName("root"); // from partition name string +``` + +### StrOrNum + +Discriminated union for list partition keys — supports both string and numeric values: + +```csharp +// Implicit conversions +StrOrNum s = "tenant_a"; // → ChoiceStr +StrOrNum n = 42L; // → ChoiceNum + +// Pattern matching +result.Match( + onChoiceStr: v => Console.WriteLine($"String: {v}"), + onChoiceNum: v => Console.WriteLine($"Number: {v}") +); + +// Formatting +s.ToFmtString(); // "s:tenant_a" +StrOrNum.FromFmtStr("n:42"); // → ChoiceNum(42) +``` + +Supported implicit conversions: `string`, `int`, `long`, `short`. + +--- + +## Schedule Settings + +### MigrationScheduleSettings + +Controls automatic pre-creation of future partitions: + +| Property | Default | Description | +|----------|---------|-------------| +| `ForwardDays` | `2` | Days ahead to pre-create partitions | +| `AsBackgroundJob` | `false` | Run as hosted service | +| `MigrationJobName` | `"Migration job"` | Job name identifier | +| `ExecutionInterval` | `~4h + jitter` | Interval between migrations | +| `WaitMigrationTimeout` | `3 sec` | Semaphore wait timeout | + +### PartCleanupScheduleSettings + +Controls automatic dropping of old partitions: + +| Property | Default | Description | +|----------|---------|-------------| +| `DropPartsAfterRetention` | `30 days` | Age threshold for deletion | +| `AsBackgroundJob` | `false` | Run as hosted service | +| `ExecutionInterval` | `~4h + jitter` | Interval between cleanups | +| `InitialDelay` | `1 min` | Delay before first run | + +### PartCacheSettings + +In-memory partition metadata cache: + +| Property | Default | Description | +|----------|---------|-------------| +| `CachedFromDate` | `1 day` | How far ahead to preload partitions | + +--- + +## Naming Conventions + +Partitions follow predictable naming patterns (separator defaults to `"__"`): + +| Component | Pattern | Example | +|-----------|---------|---------| +| Range (day) | `{table}{sep}y{YYYY}m{MM}d{DD}` | `events__part__y2026m06d26` | +| Range (month) | `{table}{sep}y{YYYY}m{MM}` | `events__part__y2026m07` | +| Range (year) | `{table}{sep}y{YYYY}` | `events__part__y2026` | +| List (nested) | `{table}{sep}{val1}_{val2}...` | `orders__part__EU_EU_1` | +| Cache table | `{table}{postfix}` | `events__part` | + +**Constraints:** +- Identifiers must not exceed 63 characters (PostgreSQL limit). +- Schemas are auto-created via `CREATE SCHEMA IF NOT EXISTS`. +- Child partitions use `PARTITION OF parent FOR VALUES FROM (...) TO (...)` (range) or `FOR VALUES IN (...)` (list). +- After each range partition is created, a cache table tracks boundaries via `INSERT ... ON CONFLICT (id) DO NOTHING`. + +--- + +## Architecture + +``` +┌─────────────────────┐ +│ IPartitionManager │ ← Public entry point +├─────────────────────┤ +│ IMigrationService │ ← Pre-create future partitions +│ IPartCleanupService│ ← Drop old partitions +├─────────────────────┤ +│ IPartRepository │ ← DDL execution (CREATE/DROP PARTITION) +│ ISqlBuilder │ ← SQL template generation +│ IPartCache │ ← In-memory metadata cache +├─────────────────────┤ +│ MigrationJob │ ← IJob wrapper for Sa.Schedule +│ PartCleanupJob │ ← IJob wrapper for Sa.Schedule +└─────────────────────┘ +``` + +--- + +## Dependencies + +- `Sa.Data.PostgreSql` — Npgsql client wrapper with retry strategy +- `Sa.Schedule` — Background job scheduling infrastructure + +--- + +## Project Layout + +``` +src/Sa.Partitional.PostgreSql/ +├── Setup.cs # Main DI entrypoint AddSaPartitional() +├── IPartitionManager.cs # Public partition management API +├── PgPartBy.cs # Partitioning strategy enum +├── Classes/ +│ ├── StrOrNum.cs # Discriminated union (string | long) +│ └── Enumeration.cs # Type-safe base enum pattern +├── Configuration/ # Fluent builder API +│ ├── IPartConfiguration.cs +│ └── Builder/ # ISettingsBuilder, ISchemaBuilder, ITableBuilder +├── Settings/ # ITableSettings, ITableSettingsStorage +├── Cache/ # In-memory cache: PartCache, PartCacheSettings +├── Migration/ # Pre-creation: IMigrationService, MigrationJob +├── Cleaning/ # Old-partition removal: IPartCleanupService, PartCleanupJob +├── Partitional/ # DDL repo: IPartRepository, PartByRangeInfo +└── SqlBuilder/ # SQL templates: ISqlBuilder, SqlTemplate.cs +``` -| Document | Contents | -|---|---| -| [Guide](Guide.md) | Configuration, fluent builder, StrOrNum, naming conventions, DDL examples | -| [API Reference](ApiReference.md) | Interface signatures, architecture diagram, key types | +--- -## Project Details +## License -- **Target framework:** `.NET 10.0` -- **Native AOT compatible:** Yes -- **Dependencies:** `Sa.Data.PostgreSql`, `Sa.Schedule` -- **License:** MIT +MIT diff --git a/src/Sa.Schedule/Readme-ru.md b/src/Sa.Schedule/Readme-ru.md new file mode 100644 index 00000000..19098bfc --- /dev/null +++ b/src/Sa.Schedule/Readme-ru.md @@ -0,0 +1,360 @@ +# Sa.Schedule + +Библиотека **Sa.Schedule** — надёжная, готовая к продакшену платформа для настройки и выполнения запланированных задач в .NET приложениях. Поддерживает периодические задачи, однократные выполнения, динамический контроль параллелизма, стратегии восстановления после ошибок, перехватчики и корректное завершение работы. + +--- + +## Быстрый старт + +```csharp +var builder = Host.CreateEmptyApplicationBuilder(args); + +builder.Services.AddSaSchedule(b => +{ + b.UseHostedService() + .AddJob((sp, job) => + { + job.EveryMinutes(5) + .WithName("Database cleanup") + .WithConcurrencyLimit(2) + .ConfigureErrorHandling(err => err + .IfErrorRetry(3) + .ThenAbortJob()); + }) + .AddJob(id: Guid.Parse("xxxx-xxxx")) + .EveryHours(1) + .StartImmediate(); +}); + +var app = builder.Build(); +await app.RunAsync(); +``` + +--- + +## Определение задач (Jobs) + +Задачи реализуют интерфейс `IJob`: + +```csharp +public class CleanupJob : IJob +{ + private readonly ILogger _logger; + private readonly IDbConnection _db; + + public CleanupJob(ILogger logger, IDbConnection db) + { + _logger = logger; + _db = db; + } + + public async Task Execute(IJobContext context, CancellationToken cancellationToken) + { + _logger.LogInformation("Running cleanup — iteration #{Num}", context.NumIterations); + await _db.ExecuteAsync("DELETE FROM temp_table WHERE created_at < @now", + new { now = DateTimeOffset.UtcNow }, cancellationToken); + } +} +``` + +Scoped-сервисы (DbContext, IDbConnection и т.д.) автоматически разрешаются в рамках DI-scope на каждое выполнение. + +### Lambda-задачи + +Для быстрых одноразовых задач без выделенного класса: + +```csharp +b.AddJob((context, ct) => +{ + Console.WriteLine($"Hello at {context.ExecuteAt}"); + return Task.CompletedTask; +}, jobId: Guid.NewGuid()) + .EverySeconds(10); +``` + +--- + +## Конфигурация задач (Builder API) + +| Метод | Описание | +|-------|----------| +| `.WithName(string)` | Человекочитаемое имя задачи | +| `.StartImmediate()` | Выполнить при старте без ожидания интервала | +| `.RunOnce()` | Выполнить ровно один раз, затем навсегда остановиться | +| `.WithInitialDelay(TimeSpan)` | Задержка перед первым выполнением | +| `.EveryTime(TimeSpan, string?)` | Периодический интервал с опциональным именем тайминга | +| `.EverySeconds(int)` | Утилита для секунд | +| `.EveryMinutes(int)` | Утилита для минут | +| `.EveryHours(int)` | Утилита для часов | +| `.EveryDays(int)` | Утилита для дней | +| `.OnceIn(TimeSpan)` | Выполнить один раз после задержки | +| `.Cron(string, string?)` | Расписание через cron-выражение (минута час деньМесяца месяц деньНедели) | +| `.WithContextStackSize(int)` | Хранить N предыдущих контекстов в стеке для отладки | +| `.WithTag(object)` | Прикрепить произвольные метаданные | +| `.WithConcurrencyLimit(int)` | Количество одновременных выполнений | +| `.WithMaxConcurrency(int)` | Максимальное количество зарезервированных слотов | +| `.Disabled()` | Зарегистрировать, но не запускать | +| `.Merge(IJobProperties)` | Объединить с другой конфигурацией | +| `.ConfigureErrorHandling(Action)` | Политика восстановления после ошибок | + +--- + +## Cron-расписание + +Используйте cron-выражения для точного контроля расписания. Формат следует стандартному 5-полевому cron: + +``` +минута час деньМесяца месяц деньНедели +``` + +**Поддерживаемые возможности:** +- `*` — wildcard (любое значение) +- `,` — список через запятую (например, `1,15,30`) +- `-` — диапазон (например, `1-5`) +- `/` — шаг (например, `*/5`, `1-20/3`) + +**Примеры:** + +```csharp +// Каждый день в 9:00 AM +b.AddJob() + .Cron("0 9 * * *") + .WithName("Daily report"); + +// Каждые 2 часа в минуту 0 +b.AddJob() + .Cron("0 */2 * * *") + .WithName("Hourly sync"); + +// Будни (Пн-Пт) в 14:30 +b.AddJob() + .Cron("30 14 * * 1-5") + .WithName("Weekday cleanup"); + +// Первое число каждого месяца в полночь +b.AddJob[MonthlyBackup]() + .Cron("0 0 1 * *") + .WithName("Monthly backup"); + +// Понедельник, Среда, Пятница в 6:00 AM +b.AddJob[TriWeeklyTask]() + .Cron("0 6 * * 1,3,5") + .WithName("Tri-weekly task"); + +// Каждые 15 минут +b.AddJob[HealthCheck]() + .Cron("*/15 * * * *") + .WithName("Health check"); + +// Комбинация диапазона и шага: каждый 3-й час с 9 до 17 +b.AddJob[BusinessMetrics]() + .Cron("0 9-17/3 * * 1-5") + .WithName("Business metrics"); +``` + +**Продвинутые примеры:** + +```csharp +// Последний день месяца (приблизительно — используйте 28-31 и позвольте cron отфильтровать) +b.AddJob[EndOfMonthReport]() + .Cron("0 0 28-31 * *") + .WithName("End of month report"); + +// Только високосный год (29 февраля) +b.AddJob[LeapYearTask]() + .Cron("0 0 29 2 *") + .WithName("Leap year task"); + +// Несколько дней недели (Пн, Ср, Пт в 9:00 и 17:00) +b.AddJob[PeakMonitor]() + .Cron("0 9,17 * * 1,3,5") + .WithName("Peak monitoring"); +``` + +--- + +## Модель параллелизма + +- **`ConcurrencyLimit`** — сколько слотов активно работают в любой момент (изначально). Можно менять динамически через `IJobScheduler.ConcurrencyLimit`. +- **`MaxConcurrency`** — общее количество предзарезервированных слотов. `ConcurrencyLimit ≤ MaxConcurrency`. +- Динамическая корректировка приостанавливает/возобновляет отдельные слоты без их пересоздания. + +--- + +## Обработка ошибок + +Каждая задача определяет свою собственную политику ошибок: + +```csharp +.ConfigureErrorHandling(err => err + .IfErrorRetry(count: 3) // Повторить до 3 раз + .DoSuppressError(ex => ex is TimeoutException) // Подавить таймауты молча + .ThenAbortJob()) // После исчерпания повторов остановить только эту задачу +``` + +### Действия обработки ошибок + +| Действие | Поведение | +|----------|----------| +| `CloseApplication` | Остановить всё приложение через `IHostApplicationLifetime.StopApplication()` (**по умолчанию**) | +| `AbortJob` | Остановить только текущую задачу; другие продолжат работу | +| `StopAllJobs` | Остановить все зарегистрированные задачи | + +### Глобальный обработчик ошибок + +Зарегистрируйте глобальный обработчик, который выполняется *перед* обработкой на уровне задачи: + +```csharp +b.AddErrorHandler((context, exception) => +{ + // Верните true для поглощения (подавления) ошибки + // Верните false, чтобы передать обработку на уровень задачи + if (exception is InvalidOperationException) + { + context.Logger.LogWarning("Known issue: {Msg}", exception.Message); + return true; + } + return false; +}); +``` + +### JobException + +Когда задача выбрасывает исключение, оно оборачивается в `JobException`, содержащий: +- `JobContext` — полный контекст в момент сбоя +- `ContextSnapshot` — лёгкий снимок (скалярные свойства + глубина стека), избегает дорогого глубокого клонирования +- `InnerException` — оригинальное исключение + +--- + +## Перехватчики (Interceptors) + +Перехватчики оборачивают каждое выполнение задачи, реализуя chain-of-responsibility: + +```csharp +public class LoggingInterceptor : IJobInterceptor +{ + private readonly ILogger _logger; + + public LoggingInterceptor(ILogger logger) + => _logger = logger; + + public async Task OnHandle(IJobContext context, Func next, object? key, CancellationToken ct) + { + _logger.LogInformation("[{Job}] Starting", context.JobName); + var sw = Stopwatch.StartNew(); + try + { + await next(); + sw.Stop(); + _logger.LogInformation("[{Job}] Completed in {Ms}ms", context.JobName, sw.ElapsedMilliseconds); + } + catch (Exception ex) + { + sw.Stop(); + _logger.LogError(ex, "[{Job}] Failed after {Ms}ms", context.JobName, sw.ElapsedMilliseconds); + throw; + } + } +} + +// Регистрация глобально +b.AddInterceptor(); +``` + +Можно зарегистрировать несколько перехватчиков — они применяются в порядке LIFO (последний добавленный = внешняя обёртка). + +--- + +## Управление во время выполнения + +Получите доступ к планировщику через DI: + +```csharp +public class Controller +{ + private readonly IScheduler _scheduler; + + public Controller(IScheduler scheduler) + => _scheduler = scheduler; + + public async Task RestartAll() + { + var count = await _scheduler.Restart(TestContext.Current.CancellationToken); + Console.WriteLine($"Restarted {count} jobs"); + } + + public async Task StopAll() + => await _scheduler.Stop(); + + public void ChangeConcurrency(Guid jobId, int newLimit) + { + var schedule = _scheduler.GetSchedule(jobId); + schedule?.ConcurrencyLimit = newLimit; + } +} +``` + +### IScheduler + +| Член | Описание | +|------|----------| +| `Settings` | Настройки на весь план | +| `Schedules` | Коллекция `IJobScheduler` | +| `Start(ct)` | Запустить все незаблокированные задачи | +| `Restart(ct)` | Остановить + перезапустить все запущенные задачи | +| `Stop()` | Корректная остановка с таймаутом 30 сек | +| `GetSchedule(id)` | Найти конкретный планировщик задач | + +### IJobScheduler + +| Член | Описание | +|------|----------| +| `JobId` | Уникальный идентификатор | +| `IsStarted` | Запущена ли задача в данный момент | +| `ActiveTasks` | Ожидающие задачи в очереди | +| `ConcurrencyLimit` | Получить/установить активный параллелизм | +| `StartChangeToken()` | Отслеживать изменения состояния start/stop | +| `Start(ct)` | Запустить эту задачу | +| `Stop()` | Остановить с таймаутом | + +--- + +## Архитектура + +``` +DI Setup (Setup.cs + ScheduleBuilder.cs) + ↓ +Configuration (JobSettings, JobProperties, JobErrorHandling) + ↓ +Factory (JobFactory → создаёт IJobScheduler) + ↓ +Scheduler (IScheduler → управляет IReadOnlyCollection) + ↓ +JobScheduler (один на каждую IJob, работает на базе SaWorkQueue) + ↓ +JobController (предзарезервированные слоты, пауза/возобновление через SemaphoreSlim) + ↓ +JobExecutor (DI-scoped + цепочка перехватчиков) + ↓ +IJob.Execute(...) +``` + +--- + +## Лучшие практики + +1. **Всегда используйте `UseHostedService()`** — интеграция с жизненным циклом Generic Host +2. **Предпочитайте типизированные задачи lambda-задачам** — лучшая тестируемость и разрешение DI +3. **Устанавливайте `ConcurrencyLimit` appropriately** — не перегружайте downstream-системы +4. **Используйте `DoSuppressError` для транзитных сбоев** — не падайте на восстанавливаемых ошибках +5. **Добавляйте перехватчики для сквозных задач** — логирование, метрики, распределённый трейсинг +6. **Мониторьте через `IJobScheduler.IsStarted` и `ActiveTasks`** — интегрируйте с health checks +7. **Используйте `OnceIn(TimeSpan)` для миграционных задач** — выполнить один раз после задержки деплоя +8. **Отключайте задачи вместо удаления** — полезно для feature flags и постепенного rollout + +--- + +## Лицензия + +MIT diff --git a/src/Sa.Schedule/Readme.md b/src/Sa.Schedule/Readme.md index 7fb08b22..0ff5ad7f 100644 --- a/src/Sa.Schedule/Readme.md +++ b/src/Sa.Schedule/Readme.md @@ -97,7 +97,9 @@ b.AddJob((context, ct) => | `.Merge(IJobProperties)` | Merge another configuration | | `.ConfigureErrorHandling(Action)` | Error recovery policy | -### Cron Scheduling +--- + +## Cron Scheduling Use cron expressions for precise scheduling control. The format follows standard 5-field cron: @@ -169,7 +171,9 @@ b.AddJob[PeakMonitor]() .WithName("Peak monitoring"); ``` -### Concurrency Model +--- + +## Concurrency Model - **`ConcurrencyLimit`** — how many slots are actively running at any time (initially). Can be changed dynamically via `IJobScheduler.ConcurrencyLimit`. - **`MaxConcurrency`** — total number of slot pre-allocated. `ConcurrencyLimit ≤ MaxConcurrency`. @@ -348,3 +352,9 @@ IJob.Execute(...) 6. **Monitor via `IJobScheduler.IsStarted` and `ActiveTasks`** — integrate with health checks 7. **Use `OnceIn(TimeSpan)` for migration jobs** — run once after deployment delay 8. **Disable jobs instead of removing** — useful for feature flags and gradual rollout + +--- + +## License + +MIT diff --git a/src/Sa.Utils.WorkQueue/Readme-ru.md b/src/Sa.Utils.WorkQueue/Readme-ru.md new file mode 100644 index 00000000..16846902 --- /dev/null +++ b/src/Sa.Utils.WorkQueue/Readme-ru.md @@ -0,0 +1,174 @@ +# Sa.Utils.WorkQueue + +Высокопроизводительная асинхронная очередь задач для .NET с ограниченной ёмкостью, динамическим контролем параллелизма и несколькими стратегиями масштабирования читателей. Построена на базе `System.Threading.Channels`. + +--- + +## Возможности + +| Возможность | Описание | +|-------------|----------| +| **Ограниченная очередь** | Back-pressure через `BoundedChannel` — переполнение обрабатывается по `Wait`, `DropWrite` или `DropOldest` | +| **Динамический параллелизм** | Изменяйте `ConcurrencyLimit` на лету — читатели адаптируются автоматически | +| **Стратегии масштабирования** | `Lifo` • `Fifo` • `RoundRobin` • `Random` — выберите подход к замене читателей при ресайзе | +| **DI-интеграция** | `AddSaWorkQueue` с полной поддержкой конфигурации | +| **Логирование без аллокаций** | `[LoggerMessage]` source generator для `ILogger` | +| **Корректное завершение** | `ShutdownAsync`, `DisposeAsync` — идемпотентно и потокобезопасно | +| **Стратегии ошибок** | `Continue` (по умолчанию на элемент), `StopReader` или `ShutdownQueue` | +| **Обратные вызовы статусов** | Отслеживайте жизненный цикл: `Running` → `Completed` / `Faulted` / `Cancelled` / `Aborted` | + +--- + +## 🚀 Быстрый старт + +### 1️⃣ Реализуйте процессор + +```csharp +public sealed class OrderWork(ILogger logger) : ISaWork +{ + public async Task Execute(OrderInput input, CancellationToken ct) + { + logger.LogInformation("Processing order {OrderId}", input.OrderId); + await ProcessOrderAsync(input, ct); // Ваша бизнес-логика + } +} +``` + +### 2️⃣ Зарегистрируйте в DI + +```csharp +builder.Services.AddSaWorkQueue((sp, opts) => + opts + .WithConcurrencyLimit(4) + .WithQueueCapacity(100) + .WithMaxConcurrency(16) + .WithReaderScalingStrategy(SaReaderScalingStrategy.RoundRobin) + .WithFullMode(BoundedChannelFullMode.DropOldestWhenFull) + .WithStatusCallback((input, status, ex) => + { + // logger.LogDebug("Заказ {Id} → {Status}", input.OrderId, status); + })); +``` + +### 3️⃣ Используйте через внедрение + +```csharp +public class OrderService(ISaWorkQueue queue) +{ + public async Task SubmitAsync(OrderInput order, CancellationToken ct) + { + await queue.Enqueue(order, ct); + } + + public async Task WaitForCompletionAsync(CancellationToken ct) + => await queue.WaitForIdleAsync(ct); + + public bool IsIdle() => queue.IsIdle(); + public int Pending => queue.QueueTasks; +} +``` + +--- + +## ⚙️ Настройка `SaWorkQueueOptions` + +Все параметры — immutable record поля с fluent-методами `With*`: + +```csharp +SaWorkQueueOptions.Create(processor) + .WithConcurrencyLimit(int) // Параллельных читателей (по умолч.: кол-во ядер) + .WithQueueCapacity(int) // Ёмкость канала (по умолч.: равно лимиту) + .WithMaxConcurrency(int) // Абсолютный потолок читателей (по умолч.: кол-во ядер) + .WithSingleWriter(bool) // Оптимизация для однопользовательских сценариев + .WithFullMode(BoundedChannelFullMode) // Wait | DropOldest | DropNewest | DropWrite + .WithReaderScalingStrategy(enum) // Lifo | Fifo | RoundRobin | Random + .WithStatusCallback(Action) + .WithHandleItemFaulted(Func) + .WithItemDisplayName(Func) // Пользовательское имя элемента для логирования +``` + +### Создание опций + +```csharp +// Через реализацию ISaWork +var opts = SaWorkQueueOptions.Create(new OrderWork(logger)); + +// Через делегат (без класса) +var opts = SaWorkQueueOptions.Create(async (input, ct) => { + await ProcessAsync(input, ct); +}); +``` + +--- + +## Стратегии масштабирования читателей + +Применяются при уменьшении `ConcurrencyLimit` на лету — определяют, каких читателей отменять: + +| Стратегия | Поведение | Лучше всего для | +|-----------|----------|-----------------| +| `Lifo` | Отменяет наиболее недавних читателей | CPU-bound задачи, локальность кэша | +| `Fifo` | Отменяет самых старых читателей | Ресурсная ротация, равномерное время жизни | +| `RoundRobin` | Циклический обход читателей | Стабильные воркеры, сбалансированная нагрузка | +| `Random` | Случайный выбор читателей | Тестирование, избегание паттернов | + +--- + +## 🔑 API `ISaWorkQueue` + +| Член | Тип | Описание | +|------|-----|----------| +| `Enqueue(input, ct)` | Метод | Добавить задачу (не блокирует, если есть место) | +| `WaitForIdleAsync(ct)` | Метод | Дождаться завершения всех задач | +| `ShutdownAsync()` | Метод | Корректное завершение (финиш активных + очистка) | +| `Shutdown()` | Метод | Синхронное завершение | +| `ForceCancelReaders()` | Метод | Аварийная остановка всех читателей | +| `ForceCancelReadersAsync()` | Метод | Асинхронная аварийная остановка | +| `IsIdle()` | Свойство | `true`, если нет ожидающих/активных задач | +| `IsEnabled` | Свойство | `true`, пока очередь активна | +| `QueueTasks` | Свойство | Всего задач в обработке + в очереди | +| `ConcurrencyLimit` | Свойство | Текущий лимит параллелизма (изменяемый) | +| `MaxConcurrency` | Свойство | Абсолютный потолок | +| `QueueCapacity` | Свойство | Ёмкость ограниченного канала | +| `ShutdownError` | Свойство | Исключение, вызвавшее shutdown, если было | + +--- + +## Жизненный цикл статусов + +Каждый элемент проходит через статусы, которые сообщаются через callback `StatusChanged`: + +| Статус | Значение | +|--------|----------| +| `Running` | Элемент обрабатывается | +| `Completed` | Успешно завершён | +| `Faulted` | Произошла необработанная ошибка | +| `Cancelled` | Отменён системой (shutdown, таймаут) | +| `Aborted` | Отменён явно токеном вызывающего | + +--- + +## Стратегии обработки ошибок + +Настраиваются через `.WithHandleItemFaulted(...)`: + +| Стратегия | Поведение | +|-----------|----------| +| `Continue` | Пометить элемент как Faulted, продолжить обработку остальных | +| `StopReader` | Пометить элемент как Faulted, остановить текущего читателя (автоматически заменится) | +| `ShutdownQueue` | Пометить элемент как Faulted, инициировать полное завершение очереди | + +По умолчанию: `ShutdownQueue` — ошибка элемента запускает shutdown. Для отказоустойчивых пайплайнов переопределите на `Continue` или `StopReader`. + +--- + +## ⚠️ Важные заметки + +1. **Жизненный цикл**: регистрируется как `Singleton`. Не используйте `Scoped`/`Transient`. +2. **Callback `StatusChanged`**: вызывается синхронно на thread-pool потоке. Избегайте длительных операций внутри. Исключения обработчика логируются, но не распространяются. +3. **Отмена**: каждый `Enqueue` принимает `CancellationToken`. Элементы различают отмену вызывающей стороной (`Aborted`) и системную отмену (`Cancelled`). +4. **Потокобезопасность**: все публичные члены потокобезопасны. Изменение `ConcurrencyLimit` на лету корректирует число читателей без потери ожидающих элементов. +5. **Идемпотентное завершение**: `ShutdownAsync`, `Shutdown`, `Dispose`, `DisposeAsync` безопасны для многократного вызова. +6. **`ConcurrencyLimit = 0`**: приостанавливает всю обработку (убивает всех читателей). Верните положительное значение для возобновления. +7. **`ForceCancelReaders` / `ForceCancelReadersAsync`**: аварийная остановка — мгновенно отменяет все reader-задачи. После вызова восстановите параллелизм установкой `ConcurrencyLimit = X` для запуска новых читателей. +8. **Back-pressure**: при заполненной очереди поведение зависит от `FullMode` — `Wait` блокирует вызывающего, `DropOldest` удаляет самый старый элемент, `DropNewest` отбрасывает входящий, `DropWrite` завершает вызов enqueue ошибкой. diff --git a/src/Sa.Utils.WorkQueue/Readme.md b/src/Sa.Utils.WorkQueue/Readme.md index b74a5b74..fc59361d 100644 --- a/src/Sa.Utils.WorkQueue/Readme.md +++ b/src/Sa.Utils.WorkQueue/Readme.md @@ -1,6 +1,6 @@ -# SaWorkQueue — Async Queue with Concurrency Limiting +# Sa.Utils.WorkQueue -> High-performance task queue for .NET 10 with dynamic scaling, DI integration, and a type-safe API. +High-performance async task queue for .NET with bounded capacity, dynamic concurrency scaling, and a type-safe API. --- @@ -11,10 +11,12 @@ | **Concurrency limiting** | Control the number of simultaneously executing tasks via `ConcurrencyLimit` | | **Dynamic scaling** | Change the limit at runtime: `queue.ConcurrencyLimit = newLimit` | | **Scaling strategies** | `Lifo` • `Fifo` • `RoundRobin` • `Random` — choose the one that fits your scenario | -| **DI integration** | Registration via `AddSaWorkQueue` with configuration support | -| **Zero-allocation logging** | `[LoggerMessage]` generation for `ILogger` | -| **Safe shutdown** | `DisposeAsync`, `ShutdownAsync`, `WaitForIdleAsync` — idempotent and thread‑safe | -| **Fault tolerance** | Configurable error strategy: `ShutdownQueue` (default), `StopReader`, or `Continue` | +| **DI integration** | Registration via `AddSaWorkQueue` or delegate-based `AddSaWorkQueue` | +| **Zero-allocation logging** | `[LoggerMessage]` source generator for `ILogger` | +| **Safe shutdown** | `ShutdownAsync`, `DisposeAsync` — idempotent and thread-safe | +| **Error strategies** | Per-item fault handling: `Continue`, `StopReader`, or `ShutdownQueue` | +| **Status callbacks** | Track item lifecycle: `Running` → `Completed` / `Faulted` / `Cancelled` / `Aborted` | +| **Back-pressure** | Configurable full-channel behavior: `Wait`, `DropOldest`, `DropNewest`, `DropWrite` | --- @@ -28,7 +30,7 @@ public sealed class OrderWork(ILogger logger) : ISaWork public async Task Execute(OrderInput input, CancellationToken ct) { logger.LogInformation("Processing order {OrderId}", input.OrderId); - await ProcessAsync(input, ct); // Your business logic + await ProcessOrderAsync(input, ct); // Your business logic } } ``` @@ -40,8 +42,10 @@ builder.Services.AddSaWorkQueue((sp, opts) => opts .WithConcurrencyLimit(4) .WithQueueCapacity(100) + .WithMaxConcurrency(16) .WithReaderScalingStrategy(SaReaderScalingStrategy.RoundRobin) - .WithStatusChanged((input, status, ex) => + .WithFullMode(BoundedChannelFullMode.DropOldestWhenFull) + .WithStatusCallback((input, status, ex) => { // logger.LogDebug("Order {Id} → {Status}", input.OrderId, status); })); @@ -53,9 +57,10 @@ builder.Services.AddSaWorkQueue((sp, opts) => public class OrderService(ISaWorkQueue queue) { public async Task SubmitAsync(OrderInput order, CancellationToken ct) - { - await queue.Enqueue(order, ct); // Does not block the caller - } + => await queue.Enqueue(order, ct); + + public async Task WaitForCompletionAsync(CancellationToken ct) + => await queue.WaitForIdleAsync(ct); public bool IsIdle() => queue.IsIdle(); public int Pending => queue.QueueTasks; @@ -66,63 +71,116 @@ public class OrderService(ISaWorkQueue queue) ## ⚙️ `SaWorkQueueOptions` Configuration +All parameters are immutable record fields with fluent `With*` methods: + ```csharp SaWorkQueueOptions.Create(processor) - .WithConcurrencyLimit(int) // Concurrency limit (default: CPU count) - .WithQueueCapacity(int) // Queue capacity (default: equals the limit) - .WithMaxConcurrency(int) // Absolute maximum number of readers - .WithReaderScalingStrategy(enum) // Lifo | Fifo | RoundRobin | Random - .WithSingleWriter(bool) // Optimisation for a single write source - .WithFullMode(enum) // Wait | DropWrite | DropOldest - .WithStatusCallback(Action<...>) // Callback on task status change - .WithHandleItemFaulted(Func<...>) // Continue | StopReader | ShutdownQueue - .WithItemDisplayName(Func<...>) // Display name for each work item (e.g., for logging) + .WithConcurrencyLimit(int) // Concurrency limit (default: CPU count) + .WithQueueCapacity(int) // Channel capacity (default: equals limit) + .WithMaxConcurrency(int) // Absolute ceiling of readers (default: CPU count) + .WithSingleWriter(bool) // Optimisation for single-writer scenarios + .WithFullMode(BoundedChannelFullMode) // Wait | DropOldest | DropNewest | DropWrite + .WithReaderScalingStrategy(enum) // Lifo | Fifo | RoundRobin | Random + .WithStatusCallback(Action) + .WithHandleItemFaulted(Func) + .WithItemDisplayName(Func) // Custom display name for logging +``` + +### Creating options + +```csharp +// Via ISaWork implementation +var opts = SaWorkQueueOptions.Create(new OrderWork(logger)); + +// Via delegate (no class needed) +var opts = SaWorkQueueOptions.Create(async (input, ct) => { + await ProcessAsync(input, ct); +}); ``` --- ## Reader Scaling Strategies +Applied when decreasing `ConcurrencyLimit` at runtime — determines which readers to cancel: + | Strategy | Behaviour | Best for | |----------|-----------|----------| -| `Lifo` (default) | Cancels the most recently created | CPU‑bound tasks, cache locality | -| `Fifo` | Cancels the earliest created | Connection pools, databases, rotation | -| `RoundRobin` 🔄 | Cycles through the queue | Load balancing, stable workers | -| `Random` 🎲 | Random selection | Testing, avoiding patterns | +| `Lifo` | Cancels the most recently created readers | CPU-bound tasks, cache locality | +| `Fifo` | Cancels the oldest readers | Resource rotation, even lifetime distribution | +| `RoundRobin` | Cyclic reader cancellation | Stable workers, balanced load | +| `Random` | Random reader cancellation | Testing, avoiding patterns | --- -## 🔑 Key Methods of `ISaWorkQueue` +## 🔑 `ISaWorkQueue` API + +| Member | Kind | Description | +|--------|------|-------------| +| `Enqueue(input, ct)` | Method | Add a task (non-blocking if there is room) | +| `WaitForIdleAsync(ct)` | Method | Wait until all tasks complete | +| `ShutdownAsync()` | Method | Graceful shutdown (finish active + drain) | +| `Shutdown()` | Method | Synchronous shutdown | +| `ForceCancelReaders()` | Method | Emergency stop of all readers | +| `ForceCancelReadersAsync()` | Method | Async emergency stop of all readers | +| `IsIdle()` | Property | `true` if no pending/active tasks | +| `IsEnabled` | Property | `true` while queue is active | +| `QueueTasks` | Property | Total tasks in progress + queued | +| `ConcurrencyLimit` | Property | Current parallelism limit (mutable) | +| `MaxConcurrency` | Property | Absolute ceiling | +| `QueueCapacity` | Property | Bounded channel capacity | +| `ShutdownError` | Property | Exception that triggered shutdown, if any | -```csharp -// Enqueue a task (does not block if there is room in the queue) -await queue.Enqueue(input, cancellationToken); +--- -// Wait until all tasks have been fully processed -await queue.WaitForIdleAsync(ct); +## Status Lifecycle -// Graceful shutdown: finish active tasks + clear the queue -await queue.ShutdownAsync(); +Each item flows through statuses communicated via the `StatusChanged` callback: -// Emergency stop of readers (without waiting for completion) -queue.ForceCancelReaders(); +| Status | Meaning | +|--------|---------| +| `Running` | Item is being processed | +| `Completed` | Finished successfully | +| `Faulted` | Unhandled error occurred | +| `Cancelled` | Cancelled by system (shutdown, timeout) | +| `Aborted` | Cancelled explicitly by caller's token | -// Monitoring -bool idle = queue.IsIdle(); // true if no active or pending tasks -int pending = queue.QueueTasks; // total tasks in progress + in the queue -int limit = queue.ConcurrencyLimit; // current concurrency limit -``` +--- + +## Error Strategies + +Configured via `.WithHandleItemFaulted(...)`: + +| Strategy | Behaviour | +|----------|----------| +| `Continue` | Mark item as Faulted, continue processing remaining items | +| `StopReader` | Mark item as Faulted, stop current reader (auto-replaced) | +| `ShutdownQueue` | Mark item as Faulted, trigger full queue shutdown | + +Default: `ShutdownQueue` — an item fault triggers a shutdown. For fault-tolerant pipelines, override to `Continue` or `StopReader`. --- -## ⚠️ Important Notes +## Back-Pressure Modes + +Configured via `.WithFullMode(...)`: + +| Mode | Behaviour | +|------|----------| +| `Wait` | Block the caller until space is available | +| `DropOldest` | Remove the oldest queued item, accept the new one | +| `DropNewest` | Discard the incoming item | +| `DropWrite` | Fail the enqueue call immediately | -1. **Lifecycle**: the queue is registered as `Singleton`. Do not use `Scoped`/`Transient`. -2. **`StatusChanged`**: invoked on a thread‑pool thread. Avoid long synchronous operations inside. -3. **Cancellation**: tasks receive a `CancellationToken`. Handle `OperationCanceledException` properly. -4. **Default error strategy**: `Shutdown` — a faulted item shut down the queue. Override with `.WithHandleItemFaulted()` if you need different behavior. -5. **`ForceCancelReaders` / `ForceCancelReadersAsync`**: emergency stop — kills reader tasks immediately. After calling, restore concurrency by setting `ConcurrencyLimit = X` to spawn replacement readers. -6. **Thread safety**: all public methods are thread-safe. Changing `ConcurrencyLimit` at runtime adjusts reader count without losing queued items. -7. **Reusability**: all shutdown/cleanup methods (`Shutdown`, `ShutdownAsync`, `Dispose`, `DisposeAsync`) are idempotent — safe to call multiple times. -8. **`ConcurrencyLimit = 0`**: pauses all processing (kills all readers). Set back to a positive value to resume. +--- + +## ⚠️ Important Notes +1. **Lifecycle**: registered as `Singleton`. Do not use `Scoped`/`Transient`. +2. **`StatusChanged` callback**: invoked synchronously on a thread-pool thread. Avoid long-running operations inside. Handler exceptions are logged but not propagated. +3. **Cancellation**: each `Enqueue` accepts a `CancellationToken`. Items distinguish caller-initiated cancellation (`Aborted`) from system cancellation (`Cancelled`). +4. **Thread safety**: all public members are thread-safe. Changing `ConcurrencyLimit` at runtime adjusts reader count without losing queued items. +5. **Idempotent shutdown**: `ShutdownAsync`, `Shutdown`, `Dispose`, `DisposeAsync` are safe to call multiple times. +6. **`ConcurrencyLimit = 0`**: pauses all processing (kills all readers). Restore a positive value to resume. +7. **`ForceCancelReaders` / `ForceCancelReadersAsync`**: emergency stop — immediately cancels all reader tasks. After calling, restore concurrency by setting `ConcurrencyLimit = X` to spawn replacement readers. +8. **Delegate-based registration**: `AddSaWorkQueue(configureOptions)` accepts a factory returning `SaWorkQueueOptions`, allowing registration without an `ISaWork` class. diff --git a/src/Samples/Configuration.Web/Configuration.Web.csproj b/src/Samples/Configuration.Web/Configuration.Web.csproj index 2ea8d818..57d3ba1f 100644 --- a/src/Samples/Configuration.Web/Configuration.Web.csproj +++ b/src/Samples/Configuration.Web/Configuration.Web.csproj @@ -1,4 +1,4 @@ - + net10.0 diff --git a/src/Samples/Configuration.Web/README.md b/src/Samples/Configuration.Web/README.md new file mode 100644 index 00000000..37c0786e --- /dev/null +++ b/src/Samples/Configuration.Web/README.md @@ -0,0 +1,157 @@ +# Configuration.Web + +ASP.NET Core Minimal API sample demonstrating **Sa.Configuration** — CLI argument parsing, secure secrets management from files/environment variables with `{{placeholder}}` substitution, and dynamic PostgreSQL-backed configuration. + +--- + +## Quick Start + +```bash +# 1. Start PostgreSQL (or use the shared Samples docker-compose) +cd src/Samples +docker compose up db -d + +# 2. Run the sample +dotnet run --project Samples/Configuration.Web +``` + +Open `http://localhost:5245/settings` in your browser to see all configuration values loaded from three sources: CLI arguments, secrets file, and PostgreSQL database. + +--- + +## What This Sample Demonstrates + +1. **Secrets Management** — `{{sa_secret}}` placeholders in `appsettings.json` are resolved from a chain of stores: environment variables → CLI arguments → `secrets.txt` file. +2. **Dynamic Configuration from PostgreSQL** — Application settings (`theme`, `language`, `notifications`) are stored in a DB table and reflected in-app without restart. +3. **CLI Argument Parsing** — The `Arguments` class is wired through `AddSaConfiguration()` for command-line secret overrides. +4. **Zero ORM** — No EF Core or migrations. Just raw Npgsql with `CREATE TABLE IF NOT EXISTS`. + +--- + +## Architecture + +``` +AddSaConfiguration() + ├── AddSaCommandLine(args) → CommandLineArgsSecretStore + ├── AddSaPostSecretProcessing() → ChainedSecrets( + │ │ ├── EnvironmentVariableSecretStore + │ │ ├── CommandLineArgsSecretStore + │ │ └── FileSecretStore (secrets.txt) + │ ) + └── AddSaPostgreSqlConfiguration → Dynamic settings from PostgreSQL +``` + +--- + +## Configuration Chain + +The placeholder format `{{key}}` resolves values from the chained secret stores in order: + +1. **Environment Variables** — e.g., `SA_SECRET="My Secret"` +2. **CLI Arguments** — e.g., `/sa_secret:"Override from CLI"` +3. **secrets.txt** — Plain text file with `key=value` pairs (comments with `#`, auto-trimmed quotes) + +Optional variant `{{?key}}` returns `null` instead of throwing if the key is missing. + +### Example: secrets.txt + +``` +sa_pg_host=localhost +sa_pg_user=postgres +sa_pg_password=postgres +sa_pg_port=5432 +sa_pg_database=postgres +sa_pg_schema=public + +sa_secret= "TOP SECRET!" +``` + +### Example: appsettings.json with Placeholders + +```json +{ + "secret": "{{sa_secret}}", + "sa": { + "pg": { + "connection": "User ID={{sa_pg_user}};Password={{sa_pg_password}};Host={{sa_pg_host}}" + } + } +} +``` + +--- + +## Key Code + +### Program.cs + +```csharp +var builder = WebApplication.CreateSlimBuilder(args); + +// Step 1: Wire up secrets + CLI args + env vars +builder.Configuration.AddSaConfiguration(); + +const string PG_KEY = "sa:pg:connection"; +string connectionString = builder.Configuration[PG_KEY] + ?? throw new ArgumentException(PG_KEY); + +// Step 2: Create PostgreSQL data source +using var ds = IPgDataSource.Create(connectionString); + +// Step 3: Create settings table (if not exists) +ds.ExecuteScalar(""" + CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + INSERT INTO settings (key, value) + VALUES ('theme', 'dark'), ('language', 'en'), ('notifications', 'enabled') + ON CONFLICT (key) DO NOTHING; +""", null).Wait(); + +// Step 4: Add PostgreSQL as dynamic config source +builder.Configuration.AddSaPostgreSqlConfiguration(new PostgreSqlConfigurationOptions +( + ConnectionString: connectionString, + SelectSql: "select * from settings" +)); + +// Step 5: Register endpoints +var todosApi = app.MapGroup("/settings"); +todosApi.MapGet("/", (IConfiguration configuration) => new Settings[] { + new (Key: PG_KEY, Value: configuration[PG_KEY]), + new (Key: "theme", Value: configuration["theme"]), + new (Key: "language", Value: configuration["language"]), + new (Key: "notifications", Value: configuration["notifications"]), + new (Key: "secret", Value: configuration["secret"]) +}).WithName("GetSettings"); +``` + +### Expected Response + +```json +[ + { "key": "sa:pg:connection", "value": "User ID=postgres;Password=postgres;..." }, + { "key": "theme", "value": "dark" }, + { "key": "language", "value": "en" }, + { "key": "notifications", "value": "enabled" }, + { "key": "secret", "value": "TOP SECRET!" } +] +``` + +--- + +## Dependencies + +| Package | Purpose | +|---------|---------| +| `Sa.Configuration` | Secrets management, CLI argument parsing | +| `Sa.Configuration.PostgreSql` | Dynamic config from PostgreSQL | +| `Sa.Data.PostgreSql` | Npgsql client wrapper | +| `Microsoft.AspNetCore.OpenApi` | OpenAPI support (dev only) | + +--- + +## License + +MIT diff --git a/src/Samples/Configuration.Web/Readme-ru.md b/src/Samples/Configuration.Web/Readme-ru.md new file mode 100644 index 00000000..39f45e1d --- /dev/null +++ b/src/Samples/Configuration.Web/Readme-ru.md @@ -0,0 +1,157 @@ +# Configuration.Web + +ASP.NET Core Minimal API sample, демонстрирующий работу **Sa.Configuration** — разбор аргументов командной строки, управление секретами из файлов и переменных окружения с подстановкой `{{placeholder}}`, а также динамическая конфигурация из PostgreSQL. + +--- + +## Быстрый старт + +```bash +# 1. Запустите PostgreSQL (или используйте общий docker-compose из Samples) +cd src/Samples +docker compose up db -d + +# 2. Запустите пример +dotnet run --project Samples/Configuration.Web +``` + +Откройте `http://localhost:5245/settings` в браузере, чтобы увидеть все значения конфигурации, загруженные из трёх источников: аргументы CLI, файл секретов и база данных PostgreSQL. + +--- + +## Что Демонстрирует Этот Пример + +1. **Управление Секретами** — плейсхолдеры `{{sa_secret}}` в `appsettings.json` разрешаются из цепочки хранилищ: переменные окружения → аргументы CLI → файл `secrets.txt`. +2. **Динамическая Конфигурация из PostgreSQL** — настройки приложения (`theme`, `language`, `notifications`) хранятся в таблице БД и отражаются внутри приложения без перезапуска. +3. **Разбор Аргументов Командной Строки** — класс `Arguments` подключается через `AddSaConfiguration()` для переопределения секретов из CLI. +4. **Без ORM** — ни EF Core, ни миграций. Только чистый Npgsql с `CREATE TABLE IF NOT EXISTS`. + +--- + +## Архитектура + +``` +AddSaConfiguration() + ├── AddSaCommandLine(args) → CommandLineArgsSecretStore + ├── AddSaPostSecretProcessing() → ChainedSecrets( + │ │ ├── EnvironmentVariableSecretStore + │ │ ├── CommandLineArgsSecretStore + │ │ └── FileSecretStore (secrets.txt) + │ ) + └── AddSaPostgreSqlConfiguration → Динамические настройки из PostgreSQL +``` + +--- + +## Цепочка Разрешения Секретов + +Формат плейсхолдера `{{key}}` разрешает значения из цепочки хранилищ по порядку: + +1. **Переменные Окружения** — например, `SA_SECRET="Мой Секрет"` +2. **Аргументы CLI** — например, `/sa_secret:"Переопределение из CLI"` +3. **secrets.txt** — текстовый файл с парами `ключ=значение` (комментарии с `#`, автоматическая обрезка кавычек) + +Опциональный вариант `{{?key}}` возвращает `null` вместо исключения, если ключ отсутствует. + +### Пример: secrets.txt + +``` +sa_pg_host=localhost +sa_pg_user=postgres +sa_pg_password=postgres +sa_pg_port=5432 +sa_pg_database=postgres +sa_pg_schema=public + +sa_secret= "ТОП СЕКРЕТ!" +``` + +### Пример: appsettings.json с Плейсхолдерами + +```json +{ + "secret": "{{sa_secret}}", + "sa": { + "pg": { + "connection": "User ID={{sa_pg_user}};Password={{sa_pg_password}};Host={{sa_pg_host}}" + } + } +} +``` + +--- + +## Ключевой Код + +### Program.cs + +```csharp +var builder = WebApplication.CreateSlimBuilder(args); + +// Шаг 1: Подключаем секреты + CLI аргументы + переменные окружения +builder.Configuration.AddSaConfiguration(); + +const string PG_KEY = "sa:pg:connection"; +string connectionString = builder.Configuration[PG_KEY] + ?? throw new ArgumentException(PG_KEY); + +// Шаг 2: Создаём источник данных PostgreSQL +using var ds = IPgDataSource.Create(connectionString); + +// Шаг 3: Создаём таблицу настроек (если не существует) +ds.ExecuteScalar(""" + CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + INSERT INTO settings (key, value) + VALUES ('theme', 'dark'), ('language', 'en'), ('notifications', 'enabled') + ON CONFLICT (key) DO NOTHING; +""", null).Wait(); + +// Шаг 4: Добавляем PostgreSQL как источник динамической конфигурации +builder.Configuration.AddSaPostgreSqlConfiguration(new PostgreSqlConfigurationOptions +( + ConnectionString: connectionString, + SelectSql: "select * from settings" +)); + +// Шаг 5: Регистрируем эндпоинты +var todosApi = app.MapGroup("/settings"); +todosApi.MapGet("/", (IConfiguration configuration) => new Settings[] { + new (Key: PG_KEY, Value: configuration[PG_KEY]), + new (Key: "theme", Value: configuration["theme"]), + new (Key: "language", Value: configuration["language"]), + new (Key: "notifications", Value: configuration["notifications"]), + new (Key: "secret", Value: configuration["secret"]) +}).WithName("GetSettings"); +``` + +### Ожидаемый Ответ + +```json +[ + { "key": "sa:pg:connection", "value": "User ID=postgres;Password=postgres;..." }, + { "key": "theme", "value": "dark" }, + { "key": "language", "value": "en" }, + { "key": "notifications", "value": "enabled" }, + { "key": "secret", "value": "ТОП СЕКРЕТ!" } +] +``` + +--- + +## Зависимости + +| Пакет | Назначение | +|-------|-----------| +| `Sa.Configuration` | Управление секретами, разбор аргументов CLI | +| `Sa.Configuration.PostgreSql` | Динамическая конфигурация из PostgreSQL | +| `Sa.Data.PostgreSql` | Обёртка клиента Npgsql | +| `Microsoft.AspNetCore.OpenApi` | Поддержка OpenAPI (только для разработки) | + +--- + +## Лицензия + +MIT diff --git a/src/Samples/FFMpeg.Console/Readme-ru.md b/src/Samples/FFMpeg.Console/Readme-ru.md new file mode 100644 index 00000000..ce142077 --- /dev/null +++ b/src/Samples/FFMpeg.Console/Readme-ru.md @@ -0,0 +1,100 @@ +# FFMpeg.Console + +Минимальное консольное приложение, демонстрирующее работу **Sa.Media.FFmpeg** — обёртки над FFmpeg со встроенными бинарниками для Windows x64 и Linux, поддерживающей конвертацию аудио, проверку кодеков и извлечение метаданных. + +--- + +## Быстрый старт + +```bash +# Запустите пример напрямую +dotnet run --project Samples/FFMpeg.Console +``` + +Пример конвертирует `data/input.mp3` → `data/output.wav` (PCM S16 LE, моно, 16 кГц) и выводит информацию о версии + список кодеков. + +> **Зависимости Linux:** На Ubuntu/Debian установите `libmp3lame0 libopus0 libvorbis0a libvorbisenc2`. На Alpine: `lame-libs opus libvorbis`. + +--- + +## Что Демонстрирует Этот Пример + +1. **Доступ к FFmpeg без конфигурации** — `IFFMpegExecutor.Default` автоматически находит бинарники через PATH или встроенную папку `sa/native/`. +2. **Кроссплатформенность** — Работает на Windows x64 и Linux из коробки. +3. **Конвертация аудио** — Конвертирует MP3 в WAV (PCM S16 LE) с настраиваемым количеством каналов и частотой дискретизации. +4. **Native AOT совместимость** — Публикуется с `PublishAot=true` и `InvariantGlobalization=true`. + +--- + +## Ключевой Код + +```csharp +using Sa.Media.FFmpeg; + +Console.WriteLine("Hello, [Sa.Media.FFmpeg]!"); +var ffmpeg = IFFMpegExecutor.Default; + +// Проверить версию +var ver = await ffmpeg.GetVersion(); +Console.WriteLine(ver.AsSpan(0, 21)); + +// Список всех поддерживаемых кодеков +var codecs = await ffmpeg.GetCodecs(); +Console.WriteLine(codecs); + +// Конвертировать MP3 → WAV (PCM S16 LE, моно) +await ffmpeg.ConvertToPcmS16Le( + "data/input.mp3", + "data/output.wav", + outputChannelCount: 1); +``` + +--- + +## Полная Поверхность API + +| Метод / Интерфейс | Назначение | +|-------------------|-----------| +| `IFFMpegExecutor.Default` | Singleton-экземпляр без DI | +| `GetVersion()` | Строка версии FFmpeg | +| `GetFormats()` | Список поддерживаемых форматов | +| `GetCodecs()` | Список поддерживаемых кодеков | +| `ConvertToPcmS16Le(path)` | Конвертация файла → WAV PCM S16 LE | +| `ConvertToPcmS16Le(stream, format, callback)` | Поточная конвертация | +| `ConvertToPcmS16LePreservingFormat()` | То же, но сохраняет оригинальную частоту/каналы | +| `ConvertToMp3()` | Конвертация → MP3 | +| `ConvertToOgg(libopus?)` | Конвертация → OGG (Vorbis или Opus) | +| `IFFProbeExecutor` | Извлечение метаданных через ffprobe | +| `GetMetaInfo(filePath)` | Длительность, битрейт, размер файла | +| `GetChannelsAndSampleRate(filePath)` | Количество каналов и частота дискретизации | +| `Setup.AddSaFFMpeg()` | Регистрация в DI-контейнере (Generic Host) | +| `FFMpegOptions` | Опции: путь к бинарнику, таймаут, рабочая директория | + +--- + +## Использование с DI (Generic Host) + +```csharp +var builder = Host.CreateApplicationBuilder(args); +builder.Services.AddSaFFMpeg(); +var app = builder.Build(); + +var executor = app.Services.GetRequiredService(); +await executor.ConvertToMp3("input.wav", "output.mp3"); +``` + +--- + +## Файлы Проекта + +| Файл | Путь | +|------|------| +| Исходный код | `Samples/FFMpeg.Console/Program.cs` | +| Проектный файл | `Samples/FFMpeg.Console/FFMpeg.Console.csproj` | +| Входной тестовый файл | `Samples/FFMpeg.Console/data/input.mp3` | + +--- + +## Лицензия + +MIT diff --git a/src/Samples/README-ru.md b/src/Samples/README-ru.md new file mode 100644 index 00000000..3327c342 --- /dev/null +++ b/src/Samples/README-ru.md @@ -0,0 +1,236 @@ +# Быстрый старт — примеры + +Набор runnable-примеров, демонстрирующих каждую библиотеку из набора **Sa**. +Все примеры целеют **.NET 10.0**, используют **Native AOT** и следуют одному паттерну DI + Generic Host. + +> **Инфраструктура:** большинству примеров нужен PostgreSQL (иногда Minio). +> Запустите общую инфраструктуру: `docker-compose up -d` (см. [`docker-compose.yml`](./docker-compose.yml)). + +--- + +## Содержание + +| # | Пример | Библиотека | Тип | Описание | +|---|--------|------------|-----|----------| +| 1 | [Configuration.Web](#1-configurationweb) | `Sa.Configuration` + `Sa.Configuration.PostgreSql` | Web API | Динамическая конфигурация из CLI-аргументов и таблицы PostgreSQL | +| 2 | [FFMpeg.Console](#2-ffmpegconsole) | `Sa.Media.FFmpeg` | Console | Проверка версии, список кодеков, конвертация MP3→WAV | +| 3 | [HybridFileStorage.Console](#3-hybridfilestorageconsole) | `Sa.HybridFileStorage` | Console | Гибридное хранилище файлов с абстракцией провайдеров | +| 4 | [Partitional.ConsoleApp](#4-partitionalconsoleapp) | `Sa.Partitional.PostgreSql` | Console | Декларативное партиционирование таблиц с расписанием миграций | +| 5 | [PgOutbox.ConsoleApp](#5-pgoutboxconsoleapp) | `Sa.Outbox.PostgreSql` | Console | Надёжная публикация сообщений через паттерн Outbox | +| 6 | [Schedule.Console](#6-scheduleconsole) | `Sa.Schedule` | Console | Планировщик задач с стратегиями обработки ошибок | + +--- + +## 1. Configuration.Web + +Демонстрирует чтение конфигурации из аргументов командной строки и таблицы PostgreSQL, подаваемую как минимальный ASP.NET Core API. + +### Что делает + +1. Создаёт slim-ASP.NET приложение. +2. Парсит аргументы CLI через `Sa.Configuration.Arguments`. +3. Читает секреты (например, connection strings) из переменных окружения / файлов. +4. Инициализирует таблицу `settings` в PostgreSQL. +5. Hot-reload настроек из БД — изменения отражаются без перезапуска. +6. Выставляет `GET /settings` со всеми ключами конфигурации. + +### Запуск + +```powershell +# 1. Убедитесь, что PostgreSQL запущен +docker-compose up -d db + +# 2. Установите connection string (или передайте как аргумент CLI) +$env:sa__pg__connection = "Host=localhost;Username=postgres;Password=postgres;Database=postgres" + +# 3. Запуск +dotnet run --project Configuration.Web +``` + +### Тест + +```powershell +curl http://localhost:5000/settings +``` + +Ожидаемый ответ: + +```json +[ + { "key": "sa:pg:connection", "value": "Host=localhost;..." }, + { "key": "theme", "value": "dark" }, + { "key": "language", "value": "en" }, + { "key": "notifications","value": "enabled" }, + { "key": "secret", "value": null } +] +``` + +--- + +## 2. FFMpeg.Console + +Демонстрирует использование `Sa.Media.FFmpeg` для аудиообработки со встроенными бинарниками FFmpeg. + +### Что делает + +1. Получает версию FFmpeg. +2. Выводит список доступных кодеков. +3. Конвертирует `data/input.mp3` → `data/output.wav` (моно PCM_S16LE). + +### Запуск + +```powershell +dotnet run --project FFMpeg.Console +``` + +### Ожидаемый вывод + +``` +Hello, [Sa.Media.FFmpeg]! +ffmpeg version 6.x... +[aac, ac3, flac, ..., pcm_s16le, ...] +``` + +--- + +## 3. HybridFileStorage.Console + +Демонстрирует `Sa.HybridFileStorage` — абстрагированный слой хранения файлов с автоматическим failover провайдера. + +### Что делает + +1. Регистрирует `InMemoryFileStorage` как основной провайдер. +2. Загружает текстовый файл (`"Hello, HybridFileStorage!"`). +3. Скачивает обратно и верифицирует содержимое. + +### Запуск + +```powershell +dotnet run --project HybridFileStorage.Console +``` + +### Ожидаемый вывод + +``` +starting +completed:Hello, HybridFileStorage! +``` + +--- + +## 4. Partitional.ConsoleApp + +Демонстрирует декларативное партиционирование таблиц PostgreSQL с запланированными миграциями и очисткой. + +### Что делает + +1. Настраивает таблицу `customer`, партиционированную по списку (`country`, `city`). +2. Определяет расписание миграций для RU (Moscow, Samara), USA (Alabama, New York), FR (Paris, Lyon, Bordeaux). +3. Выполняет `partition.Migrate()` для создания физических партиций. +4. Выводит список созданных партиций на следующие 3 дня. + +### Запуск + +```powershell +# 1. Убедитесь, что PostgreSQL запущен +docker-compose up -d db + +# 2. Запуск (использует захардкоженный conn string) +dotnet run --project Partitional.ConsoleApp +``` + +### Ожидаемый вывод + +``` +Hello, Partitional.PostgreSql! +list of parts: +customer_20260701 +customer_RU_Moscow_20260701 +... +Successfully: True +``` + +--- + +## 5. PgOutbox.ConsoleApp + +Демонстрирует паттерн Outbox для надёжной публикации сообщений с PostgreSQL-backed хранением. + +### Что делает + +1. Регистрирует две consumer группы: `Group1Consumer` (каждые 5с, одна итерация) и `RndConsumer` (каждые 25с, макс 2 попытки). +2. Публикует 3 начальных сообщения для tenant 1. +3. Фоновый сервис непрерывно публикует случайные сообщения для tenants 1–3. +4. Консьюмеры обрабатывают сообщения с разными исходами: Ok, Retry, Postpone, Warn, Abort, Error. + +### Запуск + +```powershell +# 1. Убедитесь, что PostgreSQL запущен +docker-compose up -d db + +# 2. Запуск +dotnet run --project PgOutbox.ConsoleApp +``` + +### Ожидаемый вывод + +``` +Hello, Pg Outbox! +======= Group1Consumer : 1 ======= +2026-07-01T... #123: Hi 1 [Ok] +2026-07-01T... #124: Hi 2 [Ok] +2026-07-01T... #125: Hi 3 [Ok] +======= RndConsumer : 1 ======= +... +``` + +--- + +## 6. Schedule.Console + +Демонстрирует `Sa.Schedule` — планировщик задач с стратегиями обработки ошибок. + +### Что делает + +1. Спрашивает, запускать как hosted service (Y/n). +2. Регистрирует `SomeJob`, который выполняется каждые 2 секунды с логикой retry при ошибке. +3. Добавляет interceptor, который логирует `` / `` вокруг каждого выполнения. +4. Через 5с останавливает планировщик, ждёт 2с, затем перезапускает. +5. Через 30с отменяет всё. + +### Запуск + +```powershell +# Интерактивный: нажмите 'n' для standalone режима или 'y' для hosted service +dotnet run --project Schedule.Console +``` + +### Ожидаемый вывод (standalone режим) + +``` +Hello, Schedule! As host service (Y/n): n + + +2026-07-01T... 0: Some 2 + + +2026-07-01T... 1: Some 2 + +err 0 +err 1 +*** stopped & start after 2 sec + +2026-07-01T... 0: Some 2 + +*** cancelled on timeout +*** THE END *** +``` + +--- + +## Остановка инфраструктуры + +```powershell +docker-compose down +``` diff --git a/src/Samples/README.md b/src/Samples/README.md new file mode 100644 index 00000000..01cb1845 --- /dev/null +++ b/src/Samples/README.md @@ -0,0 +1,236 @@ +# Samples Quick Start + +Collection of runnable samples demonstrating each library in the **Sa** suite. +All samples target **.NET 10.0**, use **Native AOT**, and follow the same DI + Generic Host pattern. + +> **Infrastructure:** most samples require PostgreSQL (and sometimes Minio). +> Start shared infrastructure with: `docker-compose up -d` (see [`docker-compose.yml`](./docker-compose.yml)). + +--- + +## Table of Contents + +| # | Sample | Library | Type | Description | +|---|--------|---------|------|-------------| +| 1 | [Configuration.Web](#1-configurationweb) | `Sa.Configuration` + `Sa.Configuration.PostgreSql` | Web API | Dynamic config from CLI args + PostgreSQL table | +| 2 | [FFMpeg.Console](#2-ffmpegconsole) | `Sa.Media.FFmpeg` | Console | Version check, codec list, MP3→WAV conversion | +| 3 | [HybridFileStorage.Console](#3-hybridfilestorageconsole) | `Sa.HybridFileStorage` | Console | Hybrid file storage with provider abstraction | +| 4 | [Partitional.ConsoleApp](#4-partitionalconsoleapp) | `Sa.Partitional.PostgreSql` | Console | Declarative table partitioning with migration schedule | +| 5 | [PgOutbox.ConsoleApp](#5-pgoutboxconsoleapp) | `Sa.Outbox.PostgreSql` | Console | Reliable message publishing via Outbox pattern | +| 6 | [Schedule.Console](#6-scheduleconsole) | `Sa.Schedule` | Console | Scheduled job executor with failure strategies | + +--- + +## 1. Configuration.Web + +Demonstrates reading configuration from command-line arguments and a PostgreSQL `settings` table, served as a minimal ASP.NET Core API. + +### What it does + +1. Creates a slim ASP.NET app. +2. Parses CLI args via `Sa.Configuration.Arguments`. +3. Reads secrets (e.g. connection strings) from env vars / files. +4. Seeds a `settings` table in PostgreSQL. +5. Hot-reloads settings from DB — changes reflect without restart. +6. Exposes `GET /settings` returning all config keys. + +### Run + +```powershell +# 1. Ensure PostgreSQL is running +docker-compose up -d db + +# 2. Set the connection string (or pass as CLI arg) +$env:sa__pg__connection = "Host=localhost;Username=postgres;Password=postgres;Database=postgres" + +# 3. Run +dotnet run --project Configuration.Web +``` + +### Test + +```powershell +curl http://localhost:5000/settings +``` + +Expected response: + +```json +[ + { "key": "sa:pg:connection", "value": "Host=localhost;..." }, + { "key": "theme", "value": "dark" }, + { "key": "language", "value": "en" }, + { "key": "notifications","value": "enabled" }, + { "key": "secret", "value": null } +] +``` + +--- + +## 2. FFMpeg.Console + +Demonstrates using `Sa.Media.FFmpeg` for audio processing with built-in FFmpeg binaries. + +### What it does + +1. Gets FFmpeg version. +2. Lists available codecs. +3. Converts `data/input.mp3` → `data/output.wav` (mono PCM_S16LE). + +### Run + +```powershell +dotnet run --project FFMpeg.Console +``` + +### Expected output + +``` +Hello, [Sa.Media.FFmpeg]! +ffmpeg version 6.x... +[aac, ac3, flac, ..., pcm_s16le, ...] +``` + +--- + +## 3. HybridFileStorage.Console + +Demonstrates `Sa.HybridFileStorage` — an abstracted file storage layer with automatic provider failover. + +### What it does + +1. Registers `InMemoryFileStorage` as the primary provider. +2. Uploads a text file (`"Hello, HybridFileStorage!"`). +3. Downloads it back and verifies content. + +### Run + +```powershell +dotnet run --project HybridFileStorage.Console +``` + +### Expected output + +``` +starting +completed:Hello, HybridFileStorage! +``` + +--- + +## 4. Partitional.ConsoleApp + +Demonstrates declarative PostgreSQL table partitioning with scheduled migrations and cleanup. + +### What it does + +1. Configures a `customer` table partitioned by list (`country`, `city`). +2. Defines migration schedules for RU (Moscow, Samara), USA (Alabama, New York), FR (Paris, Lyon, Bordeaux). +3. Runs `partition.Migrate()` to create physical partitions. +4. Lists created partitions for the next 3 days. + +### Run + +```powershell +# 1. Ensure PostgreSQL is running +docker-compose up -d db + +# 2. Run (uses hardcoded conn string) +dotnet run --project Partitional.ConsoleApp +``` + +### Expected output + +``` +Hello, Partitional.PostgreSql! +list of parts: +customer_20260701 +customer_RU_Moscow_20260701 +... +Successfully: True +``` + +--- + +## 5. PgOutbox.ConsoleApp + +Demonstrates the Outbox pattern for reliable message publishing with PostgreSQL-backed outbox storage. + +### What it does + +1. Registers two consumer groups: `Group1Consumer` (every 5s, single iteration) and `RndConsumer` (every 25s, max 2 attempts). +2. Publishes 3 initial messages for tenant 1. +3. Background service continuously publishes random messages for tenants 1–3. +4. Consumers handle messages with various outcomes: Ok, Retry, Postpone, Warn, Abort, Error. + +### Run + +```powershell +# 1. Ensure PostgreSQL is running +docker-compose up -d db + +# 2. Run +dotnet run --project PgOutbox.ConsoleApp +``` + +### Expected output + +``` +Hello, Pg Outbox! +======= Group1Consumer : 1 ======= +2026-07-01T... #123: Hi 1 [Ok] +2026-07-01T... #124: Hi 2 [Ok] +2026-07-01T... #125: Hi 3 [Ok] +======= RndConsumer : 1 ======= +... +``` + +--- + +## 6. Schedule.Console + +Demonstrates `Sa.Schedule` — a scheduled job executor with failure handling strategies. + +### What it does + +1. Prompts whether to run as a hosted service (Y/n). +2. Registers `SomeJob` that runs every 2 seconds with retry-on-error logic. +3. Adds an interceptor that logs `` / `` around each execution. +4. After 5s stops the scheduler, waits 2s, then restarts. +5. After 30s cancels everything. + +### Run + +```powershell +# Interactive: press 'n' for standalone mode or 'y' for hosted service +dotnet run --project Schedule.Console +``` + +### Standalone mode expected output + +``` +Hello, Schedule! As host service (Y/n): n + + +2026-07-01T... 0: Some 2 + + +2026-07-01T... 1: Some 2 + +err 0 +err 1 +*** stopped & start after 2 sec + +2026-07-01T... 0: Some 2 + +*** cancelled on timeout +*** THE END *** +``` + +--- + +## Stopping Infrastructure + +```powershell +docker-compose down +``` diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryConsumerGroupManagerTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryConsumerGroupManagerTests.cs index 72b0fd7f..c3d7b6cb 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryConsumerGroupManagerTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryConsumerGroupManagerTests.cs @@ -119,7 +119,7 @@ private IOutboxConsumerManager GetInitializedManager() // Авто-регистрация групп при первом доступе (имитирует поведение DeliveryJob) if (!manager.IsRegistered(groupId)) - manager.Register(groupId, SettingsForTestGroup); + manager.TryRegister(groupId, SettingsForTestGroup); if (!manager.IsRegistered(blockingId)) { @@ -143,7 +143,7 @@ private IOutboxConsumerManager GetInitializedManager() PerTenantMaxDegreeOfParallelism: 1, Paused: false, Version: 0); - manager.Register(BlockingGroupId, blockingSettings); + manager.TryRegister(BlockingGroupId, blockingSettings); } return _cachedManager = manager; @@ -527,7 +527,7 @@ public void Manager_Unregister_RemovesFromManager() Assert.Null(manager.Get(group)); // Restore for other tests in the sequential collection - manager.Register(group, settings); + manager.TryRegister(group, settings); } [Fact] @@ -547,7 +547,7 @@ public void Manager_Unregister_GetAllExcludesRemoved() Assert.Equal(allBefore.Count - 1, allAfter.Count); // Restore for other tests in the sequential collection - manager.Register(removedGroup, settings); + manager.TryRegister(removedGroup, settings); } [Fact] @@ -577,7 +577,7 @@ public async Task Manager_Unregister_ProcessMessages_SkipsUnregistered() Assert.True(result >= 0); // Restore for other tests in the sequential collection - manager.Register(fixture.CountingGroupId, fixture.SettingsForTestGroup); + manager.TryRegister(fixture.CountingGroupId, fixture.SettingsForTestGroup); } #endregion diff --git a/src/Tests/Sa.Outbox.Tests/OutboxConsumerManagerTests.cs b/src/Tests/Sa.Outbox.Tests/OutboxConsumerManagerTests.cs index 7b8f8a56..c5d2419e 100644 --- a/src/Tests/Sa.Outbox.Tests/OutboxConsumerManagerTests.cs +++ b/src/Tests/Sa.Outbox.Tests/OutboxConsumerManagerTests.cs @@ -1,4 +1,4 @@ -using Sa.Outbox.Delivery; +using Sa.Outbox.Delivery; namespace Sa.Outbox.Tests; @@ -40,7 +40,7 @@ public void Pause_SetsPausedToTrue() var group = "pause-test"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); Assert.False(manager.IsPaused(group)); manager.Pause(group); @@ -54,7 +54,7 @@ public void Resume_SetsPausedToFalse() var group = "resume-test"; var settings = CreateSettings(group, paused: true); - manager.Register(group, settings); + manager.TryRegister(group, settings); Assert.True(manager.IsPaused(group)); manager.Resume(group); @@ -68,7 +68,7 @@ public void Pause_ThenResume_RestartsWithOriginalSettings() var group = "pause-resume-cycle"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); manager.Pause(group); Assert.True(manager.IsPaused(group)); @@ -127,7 +127,7 @@ public void Pause_PreservesAllSettingsExceptPaused() var group = "pause-preserve"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); manager.Pause(group); var updated = manager.Get(group); @@ -152,7 +152,7 @@ public void Subscribe_CallbackFiredOnApply() var group = "subscribe-test"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); OutboxConsumerSettings? captured = null; using var subscription = manager.Subscribe(group, s => captured = s); @@ -171,7 +171,7 @@ public void Subscribe_CallbackFiredOnPause() var group = "subscribe-pause"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); OutboxConsumerSettings? captured = null; using var subscription = manager.Subscribe(group, s => captured = s); @@ -189,7 +189,7 @@ public void Subscribe_CallbackFiredOnResume() var group = "subscribe-resume"; var settings = CreateSettings(group, paused: true); - manager.Register(group, settings); + manager.TryRegister(group, settings); OutboxConsumerSettings? captured = null; using var subscription = manager.Subscribe(group, s => captured = s); @@ -207,7 +207,7 @@ public void Subscribe_MultipleCallbacks_AllFired() var group = "subscribe-multiple"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); var callback1Invoked = false; var callback2Invoked = false; @@ -228,7 +228,7 @@ public void Unsubscribe_DisposedCallbackNotFired() var group = "subscribe-unsubscribe"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); var callbackInvoked = false; var subscription = manager.Subscribe(group, _ => callbackInvoked = true); @@ -246,7 +246,7 @@ public void Subscribe_ReceivesUpdatedVersion() var group = "subscribe-version"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); int versionReceived = -1; using var subscription = manager.Subscribe(group, s => versionReceived = s.Version); @@ -263,7 +263,7 @@ public void Subscribe_SubscriberErrorDoesNotBreakPipeline() var group = "subscribe-error-tolerance"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); // Subscriber that throws using var badSub = manager.Subscribe(group, _ => throw new InvalidOperationException("boom")); @@ -289,7 +289,7 @@ public void Subscribe_NonExistentGroup_CreatesListenerEntry() using var subscription = manager.Subscribe(group, _ => callbackInvoked = true); // Now register — subscriber should receive - manager.Register(group, CreateSettings(group)); + manager.TryRegister(group, CreateSettings(group)); Assert.True(callbackInvoked); } @@ -323,7 +323,7 @@ public void Apply_TransformsSettingsAtomically() var group = "apply-atomic"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); manager.Apply(group, s => s with { MaxBatchSize = 128, MaxDeliveryAttempts = 5 }); @@ -350,7 +350,7 @@ public void Apply_VersionIncrements() var group = "apply-version"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); var initialVersion = settings.Version; manager.Apply(group, s => s with { MaxBatchSize = 1, Version = s.Version + 1 }); @@ -367,7 +367,7 @@ public void Apply_ConsecutiveUpdates_AccumulateChanges() var group = "apply-chain"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); manager.Apply(group, s => s with { MaxBatchSize = 8 }); manager.Apply(group, s => s with { MaxDeliveryAttempts = 10 }); @@ -398,7 +398,7 @@ public void Get_ReturnsCurrentSnapshot() var group = "get-snapshot"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); var snapshot = manager.Get(group); Assert.NotNull(snapshot); @@ -412,7 +412,7 @@ public void Get_AfterApply_ReturnsUpdatedSnapshot() var group = "get-after-apply"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); manager.Apply(group, s => s with { MaxBatchSize = 256 }); var snapshot = manager.Get(group); @@ -428,7 +428,7 @@ public void IsRegistered_TrueAfterRegister() Assert.False(manager.IsRegistered(group)); - manager.Register(group, CreateSettings(group)); + manager.TryRegister(group, CreateSettings(group)); Assert.True(manager.IsRegistered(group)); } @@ -439,7 +439,7 @@ public void IsRegistered_FalseAfterUnregister() var group = "is-unregistered"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); Assert.True(manager.IsRegistered(group)); manager.Unregister(group); @@ -474,9 +474,9 @@ public void GetAllConsumerGroupIds_ReturnsAllRegisteredGroups() var group2 = "group-beta"; var group3 = "group-gamma"; - manager.Register(group1, CreateSettings(group1)); - manager.Register(group2, CreateSettings(group2)); - manager.Register(group3, CreateSettings(group3)); + manager.TryRegister(group1, CreateSettings(group1)); + manager.TryRegister(group2, CreateSettings(group2)); + manager.TryRegister(group3, CreateSettings(group3)); var ids = manager.GetAllConsumerGroupIds(); Assert.Equal(3, ids.Count); @@ -492,8 +492,8 @@ public void GetAllConsumerGroupIds_ExcludesUnregistered() var group1 = "keep-me"; var group2 = "remove-me"; - manager.Register(group1, CreateSettings(group1)); - manager.Register(group2, CreateSettings(group2)); + manager.TryRegister(group1, CreateSettings(group1)); + manager.TryRegister(group2, CreateSettings(group2)); manager.Unregister(group2); var ids = manager.GetAllConsumerGroupIds(); @@ -512,7 +512,7 @@ public async Task Concurrent_ApplyAndRead_NoDataRace() var group = "concurrent-test"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); var exceptions = new List(); var iterations = 100; @@ -568,7 +568,7 @@ public async Task Concurrent_PauseResumeAndSubscribe_NoCrash() var group = "stress-test"; var settings = CreateSettings(group); - manager.Register(group, settings); + manager.TryRegister(group, settings); var fired = 0; using var sub = manager.Subscribe(group, _ => Interlocked.Increment(ref fired)); From 56e79c8c3b8326b075e3a983521d523c984a8171 Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 1 Jul 2026 17:32:28 +0300 Subject: [PATCH 27/33] docs: add XML documentation to all public API in Sa.HybridFileStorage* projects --- .../FileSystemStorage.cs | 9 +- .../FileSystemStorageOptions.cs | 23 +++ .../FileSystemStorageSettings.cs | 26 ++- src/Sa.HybridFileStorage.FileSystem/Setup.cs | 16 +- .../IPostgresFileStorageConfiguration.cs | 37 ++++ .../PostgresFileStorage.cs | 29 +-- .../PostgresFileStorageOptions.cs | 52 ++++++ src/Sa.HybridFileStorage.Postgres/Setup.cs | 10 +- src/Sa.HybridFileStorage.S3/S3FileStorage.cs | 47 +++-- .../S3FileStorageOptions.cs | 31 +++- src/Sa.HybridFileStorage.S3/Setup.cs | 9 + .../Domain/UploadFileInput.cs | 13 ++ src/Sa.HybridFileStorage/Exceptions.cs | 30 ++- src/Sa.HybridFileStorage/FileMetadata.cs | 18 ++ src/Sa.HybridFileStorage/HybridFileStorage.cs | 60 +++--- .../HybridFileStorageExtensions.cs | 174 ++++++++++++------ .../IHybridFileStorage.cs | 8 +- .../IHybridFileStorageConfiguration.cs | 17 ++ .../IHybridFileStorageContainer.cs | 3 + ...HybridFileStorageContainerConfiguration.cs | 5 + .../InMemoryFileStorage.cs | 26 ++- .../InMemoryFileStorageOptions.cs | 5 + .../Interceptors/IDeleteInterceptor.cs | 24 +++ .../Interceptors/IDownloadInterceptor.cs | 25 +++ .../Interceptors/IInterceptorContainer.cs | 20 ++ .../Interceptors/IUploadInterceptor.cs | 25 +++ .../Interceptors/InterceptorContainer.cs | 18 +- src/Sa.HybridFileStorage/Setup.cs | 16 +- 28 files changed, 652 insertions(+), 124 deletions(-) diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs index eba11144..c46d56be 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs @@ -69,6 +69,13 @@ public async Task UploadAsync( if (!string.IsNullOrEmpty(directory)) EnsureDirectory(directory); + // Smart preallocation: use actual length when available, fall back to 0 for unknown sizes + long preallocationSize = 1024 * 1024; + if (fileStream.CanSeek && fileStream.Length > 0 && fileStream.Length <= int.MaxValue) + { + preallocationSize = fileStream.Length; + } + await using var fileStreamOutput = new FileStream(filePath, new FileStreamOptions { Mode = FileMode.Create, @@ -76,7 +83,7 @@ public async Task UploadAsync( Share = FileShare.None, BufferSize = _bufferSize, Options = FileOptions.Asynchronous | FileOptions.SequentialScan, - PreallocationSize = 5 * 1024 * 1024 + PreallocationSize = (int)preallocationSize }); await fileStream.CopyToAsync(fileStreamOutput, cancellationToken).ConfigureAwait(false); diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs index 649c756b..94100692 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs @@ -2,19 +2,42 @@ namespace Sa.HybridFileStorage.FileSystem; +/// +/// Mutable configuration options for the filesystem file storage provider, used with fluent builder pattern. +/// public sealed record FileSystemStorageOptions { + /// + /// Gets or sets the storage type identifier. Defaults to ("fs"). + /// [Required] [StringLength(10)] public string StorageType { get; set; } = FileSystemStorageSettings.DefaultStorageType; + + /// + /// Gets or sets the base directory path where files will be stored. + /// [Required] [StringLength(255)] public string BasePath { get; set; } = string.Empty; + + /// + /// Gets or sets a value indicating whether this storage is read-only. Defaults to false. + /// public bool IsReadOnly { get; set; } = false; + + /// + /// Gets or sets the basket (container) name. Must be 3–63 characters, start with a letter or underscore. + /// Defaults to ("share"). + /// [Required] [StringLength(63, MinimumLength = 3)] public string Basket { get; set; } = FileSystemStorageSettings.DefaultBasket; + /// + /// Validates the current configuration and throws a if any property is invalid. + /// + /// Thrown when , , or is invalid. public void Validate() { if (string.IsNullOrWhiteSpace(BasePath)) diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs index 3d9b3c30..4f0136c3 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs @@ -1,18 +1,42 @@ namespace Sa.HybridFileStorage.FileSystem; +/// +/// Immutable settings for the filesystem file storage provider. +/// public sealed record FileSystemStorageSettings { + /// + /// Gets the storage type identifier. Defaults to ("fs"). + /// public string StorageType { get; init; } = DefaultStorageType; + /// + /// Gets the basket (container) name. Defaults to ("share"). + /// public string Basket { get; init; } = DefaultBasket; + /// + /// Gets the base directory path where files will be stored. + /// public required string BasePath { get; init; } + /// + /// Gets a value indicating whether this storage is read-only. Defaults to false. + /// public bool IsReadOnly { get; init; } = false; + /// + /// Gets the buffer size used for file I/O operations. Defaults to 256 KB. + /// public int BufferSize { get; init; } = 256 * 1024; - + /// + /// Gets the default storage type identifier. + /// public const string DefaultStorageType = "fs"; + + /// + /// Gets the default basket name. + /// public const string DefaultBasket = "share"; } diff --git a/src/Sa.HybridFileStorage.FileSystem/Setup.cs b/src/Sa.HybridFileStorage.FileSystem/Setup.cs index 2f903c96..02316681 100644 --- a/src/Sa.HybridFileStorage.FileSystem/Setup.cs +++ b/src/Sa.HybridFileStorage.FileSystem/Setup.cs @@ -3,9 +3,17 @@ namespace Sa.HybridFileStorage.FileSystem; - +/// +/// Provides extension methods for registering the filesystem file storage provider with the .NET Generic Host. +/// public static class Setup { + /// + /// Registers the filesystem file storage provider using immutable . + /// + /// The service collection to add the services to. + /// Immutable settings for the filesystem storage provider. + /// The same instance with the service added. public static IServiceCollection AddSaFileSystemFileStorage( this IServiceCollection services, FileSystemStorageSettings options) @@ -15,6 +23,12 @@ public static IServiceCollection AddSaFileSystemFileStorage( return services; } + /// + /// Registers the filesystem file storage provider using a mutable options builder with fluent configuration. + /// + /// The service collection to add the services to. + /// An action that receives an and a instance for fluent configuration. + /// The same instance with the service added. public static IServiceCollection AddSaFileSystemFileStorage( this IServiceCollection services, Action configure) diff --git a/src/Sa.HybridFileStorage.Postgres/IPostgresFileStorageConfiguration.cs b/src/Sa.HybridFileStorage.Postgres/IPostgresFileStorageConfiguration.cs index bdf74c52..4b48b9b3 100644 --- a/src/Sa.HybridFileStorage.Postgres/IPostgresFileStorageConfiguration.cs +++ b/src/Sa.HybridFileStorage.Postgres/IPostgresFileStorageConfiguration.cs @@ -2,12 +2,49 @@ namespace Sa.HybridFileStorage.Postgres; +/// +/// Defines a fluent configuration pipeline for the PostgreSQL file storage provider. +/// public interface IPostgresFileStorageConfiguration { + /// + /// Configures a custom PostgreSQL data source for connecting to the database. + /// + /// An optional action to configure the PostgreSQL data source settings builder. + /// The same instance for fluent chaining. IPostgresFileStorageConfiguration AddDataSource(Action? configure = null); + + /// + /// Configures the after the storage provider is registered. + /// + /// An action that receives an and for customization. + /// The same instance for fluent chaining. IPostgresFileStorageConfiguration ConfigureOptions(Action configure); + + /// + /// Sets the storage type identifier used in file IDs. Defaults to "pg". + /// + /// The storage type identifier. + /// The same instance for fluent chaining. IPostgresFileStorageConfiguration WithStorageType(string storageType); + + /// + /// Sets the database schema name where the files table resides. Defaults to "public". + /// + /// The database schema name. + /// The same instance for fluent chaining. IPostgresFileStorageConfiguration WithSchemaName(string schemaName); + + /// + /// Sets the database table name for storing file metadata. Defaults to "files". + /// + /// The database table name. + /// The same instance for fluent chaining. IPostgresFileStorageConfiguration WithTableName(string tableName); + + /// + /// Marks the PostgreSQL storage as read-only, preventing write operations. + /// + /// The same instance for fluent chaining. IPostgresFileStorageConfiguration AsReadOnly(); } diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs index 4f26e7bb..fd9151fb 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs @@ -21,7 +21,7 @@ internal sealed class PostgresFileStorage( private const string InsertSql = """ - INSERT INTO {0} (id, name, file_ext, data, size, tenant_id, basket, created_at) + INSERT INTO {0} (id, name, file_ext, data, size, tenant_id, basket, created_at) VALUES (@id, @name, @file_ext, @data, @size, @tenant_id, @basket, @created_at) ON CONFLICT (id, tenant_id, basket, created_at) DO UPDATE SET data = EXCLUDED.data, @@ -105,22 +105,24 @@ await partManager.EnsureParts( _qualifiedTableName, createdAtDay, [metadata.TenantId, _partName], - cancellationToken); + cancellationToken).ConfigureAwait(false); string fileId = FileIdParser.FormatToFileId( StorageType, _partName, metadata.TenantId, createdAtDay, metadata.FileName); string fileExtension = FileIdParser.GetFileExtension(metadata.FileName); + // If stream isn't seekable, copy into a recyclable MemoryStream first Stream ms = fileStream; + bool ownsMs = false; - if (!fileStream.CanSeek) // fileStream is not MemoryStream ms) + if (!fileStream.CanSeek) { ms = streamManager.GetStream(); - await fileStream.CopyToAsync(ms, cancellationToken); + await fileStream.CopyToAsync(ms, cancellationToken).ConfigureAwait(false); + ownsMs = true; } - - if (fileStream.CanSeek) // fileStream is not MemoryStream ms) + else { ms.Position = 0; } @@ -139,13 +141,14 @@ await dataSource.ExecuteNonQuery(sql, , new NpgsqlParameter("tenant_id", metadata.TenantId) , new NpgsqlParameter("basket", _partName) , new NpgsqlParameter("created_at", createdAt) - ], cancellationToken); + ], cancellationToken).ConfigureAwait(false); } finally { - if (ms != fileStream) + // Only dispose the buffer we allocated — never dispose the caller's stream + if (ownsMs) { - await ms.DisposeAsync(); + await ms.DisposeAsync().ConfigureAwait(false); } } @@ -170,7 +173,7 @@ public async Task DeleteAsync(string fileId, CancellationToken cancellatio new NpgsqlParameter("basket", _partName), new NpgsqlParameter("timestamp", timestamp), new NpgsqlParameter("id", fileId) - ], cancellationToken); + ], cancellationToken).ConfigureAwait(false); return rowsAffected > 0; } @@ -189,15 +192,15 @@ public async Task DownloadAsync( int rowsAffected = await dataSource.ExecuteReader(sql, async (reader, i) => { - using var fs = await reader.GetStreamAsync(0, cancellationToken); - await loadStream(fs, cancellationToken); + using var fs = await reader.GetStreamAsync(0, cancellationToken).ConfigureAwait(false); + await loadStream(fs, cancellationToken).ConfigureAwait(false); }, [ new NpgsqlParameter("tenant_id", tenantId), new NpgsqlParameter("basket", _partName), new NpgsqlParameter("timestamp", timestamp), new NpgsqlParameter("id", fileId) - ], cancellationToken); + ], cancellationToken).ConfigureAwait(false); return rowsAffected > 0; } diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs index 34244c3a..053eecda 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs @@ -2,29 +2,81 @@ namespace Sa.HybridFileStorage.Postgres; +/// +/// Configuration options for the PostgreSQL storage engine (schema, table, read-only mode). +/// public sealed record StorageOptions { + /// + /// Gets or sets the database schema name. Defaults to "public". + /// public string SchemaName { get; set; } = "public"; + + /// + /// Gets or sets the database table name for storing file metadata. Defaults to "files". + /// public string TableName { get; set; } = "files"; + + /// + /// Gets or sets the storage type identifier. Defaults to "pg". + /// public string StorageType { get; set; } = "pg"; + + /// + /// Gets or sets a value indicating whether this storage is read-only. Defaults to false. + /// public bool IsReadOnly { get; set; } } +/// +/// Configuration options for automatic cleanup of expired file records. +/// public sealed class CleanupOptions { + /// + /// Gets or sets the number of days after which file records are considered expired and eligible for cleanup. Defaults to 1095 (3 years). + /// public int ExpireDays { get; set; } = 365 * 3; } +/// +/// Configuration options for PostgreSQL table partitioning. +/// public sealed class PartOptions { + /// + /// Gets or sets the number of days in advance to generate the migration schedule for new partitions. Defaults to 2. + /// public int MigrationScheduleForwardDays { get; set; } = 2; + + /// + /// Gets or sets the partitioning granularity (day, month, or year). Defaults to . + /// public PgPartBy PgPartBy { get; set; } = PgPartBy.Day; + + /// + /// Gets or sets the basket (container) name. Defaults to "share". + /// public string Basket { get; set; } = "share"; } +/// +/// Aggregates all configuration options for the PostgreSQL file storage provider. +/// public sealed class PostgresFileStorageOptions { + /// + /// Gets the storage-specific options (schema, table, type, read-only mode). + /// public StorageOptions StorageOptions { get; } = new(); + + /// + /// Gets the partitioning options. + /// public PartOptions PartOptions { get; } = new(); + + /// + /// Gets the cleanup options. + /// public CleanupOptions CleanupOptions { get; } = new(); } diff --git a/src/Sa.HybridFileStorage.Postgres/Setup.cs b/src/Sa.HybridFileStorage.Postgres/Setup.cs index e441bbde..732ae8b2 100644 --- a/src/Sa.HybridFileStorage.Postgres/Setup.cs +++ b/src/Sa.HybridFileStorage.Postgres/Setup.cs @@ -2,9 +2,17 @@ namespace Sa.HybridFileStorage.Postgres; - +/// +/// Provides extension methods for registering the PostgreSQL file storage provider with the .NET Generic Host. +/// public static class Setup { + /// + /// Registers the PostgreSQL file storage provider with the specified service collection. + /// + /// The service collection to add the services to. + /// An optional action to configure the PostgreSQL storage via . + /// The same instance with the services added. public static IServiceCollection AddSaPostgreSqlFileStorage( this IServiceCollection services, Action? configure = null) diff --git a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs index 3c748af9..25fa4873 100644 --- a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs +++ b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs @@ -21,7 +21,9 @@ internal sealed class S3FileStorage( private readonly string _storageType = options.StorageType; private readonly bool _isReadOnly = options.IsReadOnly; private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; - private int _bucketEnsured; // 0 = not ensured, 1 = ensured (thread-safe via Interlocked) + + // Async lazy initialization for bucket ensure — prevents concurrent CreateBucket calls + private volatile Task _ensureBucketTask; public string StorageType => _storageType; public bool IsReadOnly => _isReadOnly; @@ -87,20 +89,41 @@ public async Task UploadAsync( _timeProvider.GetUtcNow()); } - private async ValueTask EnsureBucketAsync(CancellationToken cancellationToken) + private async Task EnsureBucketAsync(CancellationToken cancellationToken) + { + // Fast path: if already ensured, skip entirely + var task = _ensureBucketTask; + if (task is not null) + { + await task.ConfigureAwait(false); + return; + } + + // Slow path: create the task that will ensure the bucket + var createdTask = EnsureBucketCoreAsync(cancellationToken); + + // CompareExchange ensures only one task survives — others will await the winner + var comparison = Interlocked.CompareExchange(ref _ensureBucketTask, createdTask, null); + if (comparison is not null) + { + // Another thread won the race — await its task and discard ours + await createdTask.ConfigureAwait(false); + await comparison.ConfigureAwait(false); + return; + } + + // We won — execute and let callers await the result + await createdTask.ConfigureAwait(false); + } + + private async Task EnsureBucketCoreAsync(CancellationToken cancellationToken) { - // Lock-free проверка: проверяем bucket только один раз за время жизни экземпляра - if (Volatile.Read(ref _bucketEnsured) == 0) + if (await client.IsBucketExists(cancellationToken).ConfigureAwait(false)) { - if (await client.IsBucketExists(cancellationToken).ConfigureAwait(false)) - { - Volatile.Write(ref _bucketEnsured, 1); - return; - } - - await client.CreateBucket(cancellationToken).ConfigureAwait(false); - Volatile.Write(ref _bucketEnsured, 1); + return; } + + await client.CreateBucket(cancellationToken).ConfigureAwait(false); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs b/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs index cf856220..0cb51a82 100644 --- a/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs +++ b/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs @@ -1,18 +1,47 @@ namespace Sa.HybridFileStorage.S3; +/// +/// Configuration options for the S3 (MinIO-compatible) file storage provider. +/// public sealed class S3FileStorageOptions { + /// + /// Gets or sets the storage type identifier. Defaults to "s3". + /// public string StorageType { get; init; } = "s3"; + + /// + /// Gets or sets the basket (container) name. Defaults to "share". + /// public string Basket { get; init; } = "share"; /// - /// http://localhost:9000 + /// Gets or sets the S3-compatible endpoint URL (e.g., http://localhost:9000). /// public required string Endpoint { get; init; } + + /// + /// Gets or sets the access key for authenticating with the S3 service. + /// public required string AccessKey { get; init; } + + /// + /// Gets or sets the secret key for authenticating with the S3 service. + /// public required string SecretKey { get; init; } + + /// + /// Gets or sets the name of the S3 bucket to use for file storage. + /// public required string Bucket { get; init; } + + /// + /// Gets or sets the AWS region. Defaults to "eu-central-1". + /// public string Region { get; init; } = "eu-central-1"; + /// + /// Gets or sets a value indicating whether this storage is read-only. Defaults to false. + /// public bool IsReadOnly { get; init; } = false; } diff --git a/src/Sa.HybridFileStorage.S3/Setup.cs b/src/Sa.HybridFileStorage.S3/Setup.cs index 9bdb8123..881cc51e 100644 --- a/src/Sa.HybridFileStorage.S3/Setup.cs +++ b/src/Sa.HybridFileStorage.S3/Setup.cs @@ -4,8 +4,17 @@ namespace Sa.HybridFileStorage.S3; +/// +/// Provides extension methods for registering the S3 file storage provider with the .NET Generic Host. +/// public static class Setup { + /// + /// Registers the S3 file storage provider with the specified service collection. + /// + /// The service collection to add the services to. + /// Configuration options for the S3 storage provider. + /// The same instance with the services added. public static IServiceCollection AddSaS3FileStorage(this IServiceCollection services, S3FileStorageOptions options) { var settings = new S3BucketClientSetupSettings diff --git a/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs b/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs index 3672b5ed..5372e4f8 100644 --- a/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs +++ b/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs @@ -1,9 +1,22 @@ namespace Sa.HybridFileStorage.Domain; +/// +/// Represents input metadata for a file upload operation. +/// public sealed record UploadFileInput { + /// + /// Gets or sets the tenant identifier associated with the file. Defaults to 0. + /// public int TenantId { get; init; } = 0; + + /// + /// Gets or sets the file name. Defaults to an empty string. + /// public string FileName { get; init; } = string.Empty; + /// + /// Gets a default (empty) instance. + /// public static UploadFileInput Empty { get; } = new(); } diff --git a/src/Sa.HybridFileStorage/Exceptions.cs b/src/Sa.HybridFileStorage/Exceptions.cs index e335ce37..49ed2530 100644 --- a/src/Sa.HybridFileStorage/Exceptions.cs +++ b/src/Sa.HybridFileStorage/Exceptions.cs @@ -1,30 +1,50 @@ -using System.Diagnostics.CodeAnalysis; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Security; namespace Sa.HybridFileStorage; +/// +/// Thrown when no storage provider is available to perform the requested operation. +/// public class HybridFileStorageNoAvailableException() : Exception("No storage available."); - +/// +/// Thrown when an operation fails across multiple storage providers. +/// public class HybridFileStorageAggregateException(IEnumerable innerExceptions) : AggregateException("Operation failed for some available storages.", innerExceptions); - +/// +/// Thrown when a write operation is attempted but all storage providers are read-only. +/// public class HybridFileStorageWritableException() : Exception("Cannot perform operation. All storage options are read-only."); - - +/// +/// Provides static helper methods for throwing hybrid file storage exceptions. +/// public static class HybridFileStorageThrowHelper { + /// + /// Throws a . + /// [DoesNotReturn] public static void ThrowWritableException() => throw new HybridFileStorageWritableException(); + /// + /// Throws a indicating an invalid file ID format. + /// [DoesNotReturn] public static void ThrowInvalidFileIdFormat() => throw new FormatException("Invalid file ID format."); + /// + /// Throws a indicating that access to the specified path outside the base directory was denied. + /// + /// The path that caused the security violation. + /// The allowed base directory. [DoesNotReturn] public static void ThrowSecurityException(string path, string basePath) => throw new SecurityException( diff --git a/src/Sa.HybridFileStorage/FileMetadata.cs b/src/Sa.HybridFileStorage/FileMetadata.cs index 65e3f4d5..12942a3c 100644 --- a/src/Sa.HybridFileStorage/FileMetadata.cs +++ b/src/Sa.HybridFileStorage/FileMetadata.cs @@ -1,9 +1,27 @@ namespace Sa.HybridFileStorage; +/// +/// Represents metadata for a file stored in the hybrid file storage system. +/// public sealed class FileMetadata { + /// + /// Gets or sets the basket (container) name where the file is stored. + /// public required string Basket { get; init; } + + /// + /// Gets or sets the file name. + /// public required string FileName { get; init; } + + /// + /// Gets or sets the tenant identifier associated with the file. + /// public int TenantId { get; init; } + + /// + /// Gets the storage type identifier (e.g., "fs", "s3", "pg"). + /// public required string StorageType { get; init; } } diff --git a/src/Sa.HybridFileStorage/HybridFileStorage.cs b/src/Sa.HybridFileStorage/HybridFileStorage.cs index 2449a82b..af352b64 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorage.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorage.cs @@ -13,13 +13,27 @@ internal sealed class HybridFileStorage( private void EnsureWritable(string basket) { - if (!container.Storages.Any(c => c.Basket == basket)) + bool hasWritable = false; + bool hasAny = false; + + foreach (var storage in container.Storages) { - throw new HybridFileStorageNoAvailableException(); + if (storage.Basket == basket) + { + hasAny = true; + if (!storage.IsReadOnly) + { + hasWritable = true; + } + } } + if (!hasAny) + { + throw new HybridFileStorageNoAvailableException(); + } - if (Storages.All(f => f.Basket == basket && f.IsReadOnly)) + if (!hasWritable) { throw new HybridFileStorageWritableException(); } @@ -38,12 +52,12 @@ public async Task UploadAsync( return await ExecuteStorageOperationAsync( container.Storages.Where(c => !c.IsReadOnly && c.Basket == basket), - async (storage, ct) => await interceptors.ExecuteBeforeUploadAsync(storage, input, fileStream, ct), - async (storage, ct) => await storage.UploadAsync(input, fileStream, ct), - interceptors.ExecuteAfterUploadAsync, - interceptors.ExecuteOnUploadErrorAsync, + async (storage, ct) => await interceptors.ExecuteBeforeUploadAsync(storage, input, fileStream, ct).ConfigureAwait(false), + async (storage, ct) => await storage.UploadAsync(input, fileStream, ct).ConfigureAwait(false), + async (storage, result, ct) => await interceptors.ExecuteAfterUploadAsync(storage, result, ct).ConfigureAwait(false), + async (storage, e, ct) => await interceptors.ExecuteOnUploadErrorAsync(storage, e, ct).ConfigureAwait(false), cancellationToken - ); + ).ConfigureAwait(false); } public async Task DownloadAsync( @@ -55,12 +69,12 @@ public async Task DownloadAsync( return await ExecuteStorageOperationAsync( CanProcess(fileId), - async (storage, ct) => await interceptors.ExecuteBeforeDownloadAsync(storage, fileId, loadStream, ct), - async (storage, ct) => await storage.DownloadAsync(fileId, loadStream, ct), - async (storage, result, ct) => await interceptors.ExecuteAfterDownloadAsync(storage, fileId, result, ct), - async (storage, e, ct) => await interceptors.ExecuteOnDownloadErrorAsync(storage, fileId, e, ct), + async (storage, ct) => await interceptors.ExecuteBeforeDownloadAsync(storage, fileId, loadStream, ct).ConfigureAwait(false), + async (storage, ct) => await storage.DownloadAsync(fileId, loadStream, ct).ConfigureAwait(false), + async (storage, result, ct) => await interceptors.ExecuteAfterDownloadAsync(storage, fileId, result, ct).ConfigureAwait(false), + async (storage, e, ct) => await interceptors.ExecuteOnDownloadErrorAsync(storage, fileId, e, ct).ConfigureAwait(false), cancellationToken - ); + ).ConfigureAwait(false); } public async Task DeleteAsync( @@ -71,12 +85,12 @@ public async Task DeleteAsync( return await ExecuteStorageOperationAsync( CanProcess(fileId).Where(c => !c.IsReadOnly), - async (storage, ct) => await interceptors.ExecuteBeforeDeleteAsync(storage, fileId, ct), - async (storage, ct) => await storage.DeleteAsync(fileId, ct), - async (storage, result, ct) => await interceptors.ExecuteAfterDeleteAsync(storage, fileId, result, ct), - async (storage, e, ct) => await interceptors.ExecuteOnDeleteErrorAsync(storage, fileId, e, ct), + async (storage, ct) => await interceptors.ExecuteBeforeDeleteAsync(storage, fileId, ct).ConfigureAwait(false), + async (storage, ct) => await storage.DeleteAsync(fileId, ct).ConfigureAwait(false), + async (storage, result, ct) => await interceptors.ExecuteAfterDeleteAsync(storage, fileId, result, ct).ConfigureAwait(false), + async (storage, e, ct) => await interceptors.ExecuteOnDeleteErrorAsync(storage, fileId, e, ct).ConfigureAwait(false), cancellationToken - ); + ).ConfigureAwait(false); } @@ -94,15 +108,15 @@ private static async Task ExecuteStorageOperationAsync( { try { - if (!await beforeOperation(storage, cancellationToken)) continue; - var result = await operation(storage, cancellationToken); - await afterOperation(storage, result, cancellationToken); + if (!await beforeOperation(storage, cancellationToken).ConfigureAwait(false)) continue; + var result = await operation(storage, cancellationToken).ConfigureAwait(false); + await afterOperation(storage, result, cancellationToken).ConfigureAwait(false); return result; } catch (Exception e) { exceptions.Add(e); - await onError(storage, e, cancellationToken); + await onError(storage, e, cancellationToken).ConfigureAwait(false); } } @@ -122,7 +136,7 @@ private static async Task ExecuteStorageOperationAsync( foreach (var fs in container.Storages) { - var meta = await fs.GetMetadataAsync(fileId, cancellationToken); + var meta = await fs.GetMetadataAsync(fileId, cancellationToken).ConfigureAwait(false); if (meta != null) return meta; } diff --git a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs index e4d782be..58ee5566 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs @@ -1,10 +1,29 @@ using Sa.HybridFileStorage.Domain; -using System.Diagnostics.CodeAnalysis; namespace Sa.HybridFileStorage; +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +/// +/// Provides extension methods for common file storage operations on . +/// public static class HybridFileStorageExtensions { + /// + /// Copies a file from the local filesystem into the hybrid file storage. + /// + /// The hybrid file storage instance. + /// The path to the local file to upload. + /// The target basket (container) name. + /// Metadata about the file being uploaded. + /// The size of the buffer used when reading the file. Defaults to 81920 bytes. + /// A cancellation token to cancel the operation if needed. + /// A containing the result of the upload operation. public static async Task CopyFromFileAsync( this IHybridFileStorage storage, string filePath, @@ -13,7 +32,11 @@ public static async Task CopyFromFileAsync( int bufferSize = 81920, CancellationToken ct = default) { - // копируем файл в хранилище + ArgumentNullException.ThrowIfNull(storage); + ArgumentNullException.ThrowIfNull(filePath); + ArgumentNullException.ThrowIfNull(basket); + ArgumentNullException.ThrowIfNull(input); + await using var fs = new FileStream(filePath, new FileStreamOptions { Mode = FileMode.Open, @@ -27,9 +50,21 @@ public static async Task CopyFromFileAsync( basket: basket, input: input, fileStream: fs, - cancellationToken: ct); + cancellationToken: ct) + .ConfigureAwait(false); } + /// + /// Copies a file from one storage scope (basket) to another within the hybrid file storage system. + /// + /// The hybrid file storage instance. + /// The unique identifier of the source file. + /// The target basket (container) name. + /// An optional callback to customize the upload metadata based on the source file's metadata. + /// A cancellation token to cancel the operation if needed. + /// A containing the result of the upload operation in the target basket. + /// Thrown when the source file does not exist. + /// Thrown when the file is already in the target scope. public static async Task CopyToBasketAsync( this IHybridFileStorage storage, string fileId, @@ -37,7 +72,12 @@ public static async Task CopyToBasketAsync( Func? configure = null, CancellationToken ct = default) { - var metadata = await storage.GetMetadataAsync(fileId, ct); + ArgumentNullException.ThrowIfNull(storage); + ArgumentNullException.ThrowIfNull(fileId); + ArgumentNullException.ThrowIfNull(basket); + + var metadata = await storage.GetMetadataAsync(fileId, ct) + .ConfigureAwait(false); if (metadata is null) { @@ -66,9 +106,11 @@ public static async Task CopyToBasketAsync( basket, uploadInput, sourceStream, - downloadCt); + downloadCt) + .ConfigureAwait(false); }, - ct); + ct) + .ConfigureAwait(false); if (!downloaded) { @@ -78,16 +120,16 @@ public static async Task CopyToBasketAsync( return result; } - - [DoesNotReturn] - static void ThrowFileAlreadyInTargetScope() => - throw new InvalidOperationException("File already in target scope"); - - [DoesNotReturn] - static void ThrowSourceFileNotFound(string fileId) => - throw new FileNotFoundException($"Source file not found: {fileId}"); - - + /// + /// Copies multiple files from various storage scopes to a target basket in parallel. + /// + /// The hybrid file storage instance. + /// The collection of file IDs to copy. + /// The target basket (container) name. + /// An optional callback to customize the upload metadata based on each source file's metadata. + /// Optional settings controlling parallelism, timeout, error handling, and progress reporting. + /// A cancellation token to cancel the entire batch operation. + /// A containing lists of succeeded results and failed errors. public static async Task> CopyToScopeBatchAsync( this IHybridFileStorage storage, IEnumerable fileIds, @@ -96,6 +138,9 @@ public static async Task> CopyToScopeBatchAsync( BatchOptions? options = default, CancellationToken cancellationToken = default) { + ArgumentNullException.ThrowIfNull(storage); + ArgumentNullException.ThrowIfNull(fileIds); + ArgumentNullException.ThrowIfNull(basket); var opts = options ?? new BatchOptions(); var fileList = fileIds as IList ?? [.. fileIds]; @@ -114,75 +159,92 @@ public static async Task> CopyToScopeBatchAsync( var progress = opts.Progress; int completed = 0; + // Объект для синхронизации доступа к общим коллекциям и счетчикам + Lock lockObj = new (); - async Task ProcessFileAsync(string fileId, int index) + async Task ProcessFileAsync(string fileId, int index, CancellationToken ct) { try { + // Используем ct от делегата, а не внешний cancellationToken using var cts = opts.OperationTimeout > TimeSpan.Zero - ? CancellationTokenSource.CreateLinkedTokenSource(cancellationToken) + ? CancellationTokenSource.CreateLinkedTokenSource(ct) : null; cts?.CancelAfter(opts.OperationTimeout); - var ct = cts?.Token ?? cancellationToken; + var operationCt = cts?.Token ?? ct; return await storage.CopyToBasketAsync( fileId, basket, configure, - ct: ct); + ct: operationCt) + .ConfigureAwait(false); } catch (Exception ex) { - failed.Add(new BatchError(fileId, ex, index)); - progress?.Report(new BatchOperationProgress( - fileList.Count, - Interlocked.Increment(ref completed), - succeeded.Count, - failed.Count, - fileId, - ex)); + // Блокировка для безопасного изменения списков и атомарного отчета о прогрессе + lock (lockObj) + { + failed.Add(new BatchError(fileId, ex, index)); + completed++; + + progress?.Report(new BatchOperationProgress( + fileList.Count, + completed, + succeeded.Count, + failed.Count, + fileId, + ex)); + } return null; } } - var parallelOptions = new ParallelOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = opts.MaxDegreeOfParallelism }; - await Parallel.ForEachAsync(fileList.Select((id, idx) => (id, idx)), parallelOptions, async (item, ct) => - { - var (fileId, index) = item; - - if (!opts.ContinueOnError && failed.Count > 0) + await Parallel.ForEachAsync( + fileList.Select((id, idx) => (id, idx)), + parallelOptions, + async (item, ct) => { - return; - } + var (fileId, index) = item; - var result = await ProcessFileAsync(fileId, index); + bool hasFailed; + lock (lockObj) + { + hasFailed = failed.Count > 0; + } - if (result is not null) - { - lock (succeeded) + if (!opts.ContinueOnError && hasFailed) { - succeeded.Add(result); + return; } - } - if (result is not null) - { - progress?.Report(new BatchOperationProgress( - fileList.Count, - Interlocked.Increment(ref completed), - succeeded.Count, - failed.Count, - fileId)); - } - }); + var result = await ProcessFileAsync(fileId, index, ct) + .ConfigureAwait(false); + + if (result is not null) + { + lock (lockObj) + { + succeeded.Add(result); + completed++; + + progress?.Report(new BatchOperationProgress( + fileList.Count, + completed, + succeeded.Count, + failed.Count, + fileId)); + } + } + }); return new BatchResult { @@ -190,4 +252,12 @@ await Parallel.ForEachAsync(fileList.Select((id, idx) => (id, idx)), parallelOpt Failed = failed.AsReadOnly() }; } + + [DoesNotReturn] + static void ThrowFileAlreadyInTargetScope() => + throw new InvalidOperationException("File already in target scope"); + + [DoesNotReturn] + static void ThrowSourceFileNotFound(string fileId) => + throw new FileNotFoundException($"Source file not found: {fileId}"); } diff --git a/src/Sa.HybridFileStorage/IHybridFileStorage.cs b/src/Sa.HybridFileStorage/IHybridFileStorage.cs index 54fb21da..3d6efba0 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorage.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorage.cs @@ -10,7 +10,7 @@ namespace Sa.HybridFileStorage; public interface IHybridFileStorage { /// - /// storages + /// Gets the collection of registered file storage providers. /// IEnumerable Storages { get; } @@ -48,6 +48,12 @@ Task DownloadAsync( /// True if the file was successfully deleted; otherwise, false. Task DeleteAsync(string fileId, CancellationToken cancellationToken = default); + /// + /// Retrieves the metadata for the file associated with the specified file ID. + /// + /// The unique identifier for the file. + /// A cancellation token to cancel the operation if needed. + /// The file metadata, or null if the file is not found. Task GetMetadataAsync( string fileId, CancellationToken cancellationToken = default); diff --git a/src/Sa.HybridFileStorage/IHybridFileStorageConfiguration.cs b/src/Sa.HybridFileStorage/IHybridFileStorageConfiguration.cs index ed5cceca..7115a356 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorageConfiguration.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorageConfiguration.cs @@ -2,13 +2,30 @@ namespace Sa.HybridFileStorage; +/// +/// Defines a configuration pipeline for hybrid file storage interceptors and storage providers. +/// public interface IHybridFileStorageConfiguration { + /// + /// Configures interceptors that can observe or modify upload/download/delete operations. + /// + /// An action that receives an for registering interceptor implementations. + /// The same instance for fluent chaining. IHybridFileStorageConfiguration ConfigureInterceptors( Action configure); + /// + /// Configures storage providers that will participate in the hybrid file storage system. + /// + /// An action that receives a for registering storage implementations. + /// The same instance for fluent chaining. IHybridFileStorageConfiguration ConfigureStorage( Action configure); + /// + /// Enables automatic logging of file storage operations through registered interceptors. + /// + /// The same instance for fluent chaining. IHybridFileStorageConfiguration AddLogging(); } diff --git a/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs b/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs index ddf5bcd5..a4da65ce 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs @@ -4,5 +4,8 @@ namespace Sa.HybridFileStorage; public interface IHybridFileStorageContainer : IHybridFileStorageContainerConfiguration { + /// + /// Gets the collection of registered file storage providers. + /// IEnumerable Storages { get; } } diff --git a/src/Sa.HybridFileStorage/IHybridFileStorageContainerConfiguration.cs b/src/Sa.HybridFileStorage/IHybridFileStorageContainerConfiguration.cs index d59c9c4a..f5634fcc 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorageContainerConfiguration.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorageContainerConfiguration.cs @@ -4,5 +4,10 @@ namespace Sa.HybridFileStorage; public interface IHybridFileStorageContainerConfiguration { + /// + /// Adds a file storage provider to the hybrid file storage container. + /// + /// The storage implementation to register. + /// The same instance for chaining. IHybridFileStorageContainerConfiguration AddStorage(IFileStorage storage); } diff --git a/src/Sa.HybridFileStorage/InMemoryFileStorage.cs b/src/Sa.HybridFileStorage/InMemoryFileStorage.cs index 0e116836..ae0d6a1e 100644 --- a/src/Sa.HybridFileStorage/InMemoryFileStorage.cs +++ b/src/Sa.HybridFileStorage/InMemoryFileStorage.cs @@ -4,7 +4,10 @@ namespace Sa.HybridFileStorage; - +/// +/// An in-memory implementation of that stores file data as byte arrays in a . +/// Suitable for testing, caching, or small-scale scenarios where persistence is not required. +/// public sealed class InMemoryFileStorage( InMemoryFileStorageOptions? options = null, TimeProvider? timeProvider = null) : IFileStorage @@ -13,17 +16,32 @@ public sealed class InMemoryFileStorage( private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; + /// + /// Gets the default storage type identifier for in-memory storage. + /// public const string DefaultStorageType = "mem"; private readonly ConcurrentDictionary _storage = []; + /// + /// Gets the basket (container) name used by this storage instance. + /// public string Basket => _options.Basket; + /// + /// Gets the storage type identifier ("mem"). + /// public string StorageType => DefaultStorageType; + /// + /// Gets a value indicating whether this storage instance is read-only. + /// public bool IsReadOnly => _options.IsReadOnly; + /// + /// Gets the scheme separator used to construct file IDs in the format "storageType://basket/tenant/filename". + /// public const string SchemeSeparator = "://"; private void EnsureWritable() @@ -42,7 +60,8 @@ public async Task UploadAsync( EnsureWritable(); using var memoryStream = new MemoryStream(); - await fileStream.CopyToAsync(memoryStream, cancellationToken); + await fileStream.CopyToAsync(memoryStream, cancellationToken) + .ConfigureAwait(false); byte[] fileData = memoryStream.ToArray(); string path = Path.Combine(Basket, metadata.TenantId.ToString(), metadata.FileName).Replace('\\', '/'); @@ -62,7 +81,8 @@ public async Task DownloadAsync( if (_storage.TryGetValue(fileId, out var fileData)) { using var memoryStream = new MemoryStream(fileData); - await loadStream(memoryStream, cancellationToken); + await loadStream(memoryStream, cancellationToken) + .ConfigureAwait(false); return true; } return false; diff --git a/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs b/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs index db45c76c..bf096fc3 100644 --- a/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs +++ b/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs @@ -1,3 +1,8 @@ namespace Sa.HybridFileStorage; +/// +/// Configuration options for the in-memory file storage provider. +/// +/// The default basket (container) name. Defaults to "share". +/// true if the storage should reject write operations; otherwise, false. public sealed record InMemoryFileStorageOptions(string Basket = "share", bool IsReadOnly = false); diff --git a/src/Sa.HybridFileStorage/Interceptors/IDeleteInterceptor.cs b/src/Sa.HybridFileStorage/Interceptors/IDeleteInterceptor.cs index b974e1a2..66ca4dfa 100644 --- a/src/Sa.HybridFileStorage/Interceptors/IDeleteInterceptor.cs +++ b/src/Sa.HybridFileStorage/Interceptors/IDeleteInterceptor.cs @@ -2,16 +2,40 @@ namespace Sa.HybridFileStorage.Interceptors; +/// +/// Defines an interceptor that can observe and react to file deletion operations. +/// public interface IDeleteInterceptor { + /// + /// Determines whether the specified file can be deleted by this storage provider. + /// + /// The storage provider attempting the deletion. + /// The unique identifier of the file to delete. + /// A cancellation token to cancel the operation if needed. + /// true if the file can be deleted; otherwise, false. ValueTask CanDeleteAsync(IFileStorage storage, string fileId, CancellationToken cancellationToken); + /// + /// Called after a delete operation completes, regardless of success or failure. + /// + /// The storage provider that performed the delete. + /// The unique identifier of the file that was deleted. + /// true if the delete operation succeeded; otherwise, false. + /// A cancellation token to cancel the operation if needed. ValueTask AfterDeleteAsync( IFileStorage storage, string fileId, bool success, CancellationToken cancellationToken); + /// + /// Called when a delete operation throws an exception. + /// + /// The storage provider that encountered the error. + /// The unique identifier of the file that caused the error. + /// The exception thrown by the delete operation. + /// A cancellation token to cancel the operation if needed. ValueTask OnDeleteErrorAsync(IFileStorage storage, string fileId, Exception exception, diff --git a/src/Sa.HybridFileStorage/Interceptors/IDownloadInterceptor.cs b/src/Sa.HybridFileStorage/Interceptors/IDownloadInterceptor.cs index 3e06422f..36963313 100644 --- a/src/Sa.HybridFileStorage/Interceptors/IDownloadInterceptor.cs +++ b/src/Sa.HybridFileStorage/Interceptors/IDownloadInterceptor.cs @@ -2,20 +2,45 @@ namespace Sa.HybridFileStorage.Interceptors; +/// +/// Defines a contract for intercepting download operations in the hybrid file storage system. +/// public interface IDownloadInterceptor { + /// + /// Determines whether a download operation should proceed. + /// + /// The storage provider initiating the download. + /// The unique identifier of the file to download. + /// A function that processes the downloaded file stream. + /// A cancellation token to cancel the operation if needed. + /// true to allow the download; otherwise, false. ValueTask CanDownloadAsync( IFileStorage storage, string fileId, Func loadStream, CancellationToken cancellationToken); + /// + /// Called after a download operation completes, regardless of success or failure. + /// + /// The storage provider that performed the download. + /// The unique identifier of the downloaded file. + /// true if the download succeeded; otherwise, false. + /// A cancellation token to cancel the operation if needed. ValueTask AfterDownloadAsync( IFileStorage storage, string fileId, bool success, CancellationToken cancellationToken); + /// + /// Called when a download operation throws an exception. + /// + /// The storage provider that encountered the error. + /// The unique identifier of the file that caused the error. + /// The exception thrown by the download operation. + /// A cancellation token to cancel the operation if needed. ValueTask OnDownloadErrorAsync( IFileStorage storage, string fileId, diff --git a/src/Sa.HybridFileStorage/Interceptors/IInterceptorContainer.cs b/src/Sa.HybridFileStorage/Interceptors/IInterceptorContainer.cs index c099c015..82d4c475 100644 --- a/src/Sa.HybridFileStorage/Interceptors/IInterceptorContainer.cs +++ b/src/Sa.HybridFileStorage/Interceptors/IInterceptorContainer.cs @@ -1,8 +1,28 @@ namespace Sa.HybridFileStorage.Interceptors; +/// +/// Provides a container for registering interceptor implementations that hook into upload, download, and delete operations. +/// public interface IInterceptorContainer { + /// + /// Registers a delete interceptor. + /// + /// The delete interceptor to register. + /// The same instance for chaining. IInterceptorContainer AddDeleteInterceptor(IDeleteInterceptor interceptor); + + /// + /// Registers a download interceptor. + /// + /// The download interceptor to register. + /// The same instance for chaining. IInterceptorContainer AddDownloadInterceptor(IDownloadInterceptor interceptor); + + /// + /// Registers an upload interceptor. + /// + /// The upload interceptor to register. + /// The same instance for chaining. IInterceptorContainer AddUploadInterceptor(IUploadInterceptor interceptor); } diff --git a/src/Sa.HybridFileStorage/Interceptors/IUploadInterceptor.cs b/src/Sa.HybridFileStorage/Interceptors/IUploadInterceptor.cs index 68765669..4a582444 100644 --- a/src/Sa.HybridFileStorage/Interceptors/IUploadInterceptor.cs +++ b/src/Sa.HybridFileStorage/Interceptors/IUploadInterceptor.cs @@ -2,13 +2,38 @@ namespace Sa.HybridFileStorage.Interceptors; +/// +/// Defines a contract for intercepting upload operations in the hybrid file storage system. +/// public interface IUploadInterceptor { + /// + /// Determines whether an upload operation should proceed. + /// + /// The storage provider initiating the upload. + /// Metadata about the file being uploaded. + /// A stream containing the file data to be uploaded. + /// A cancellation token to cancel the operation if needed. + /// true to allow the upload; otherwise, false. ValueTask CanUploadAsync( IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken cancellationToken); + + /// + /// Called after an upload operation completes, regardless of success or failure. + /// + /// The storage provider that performed the upload. + /// The result of the upload operation. + /// A cancellation token to cancel the operation if needed. ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken cancellationToken); + + /// + /// Called when an upload operation throws an exception. + /// + /// The storage provider that encountered the error. + /// The exception thrown by the upload operation. + /// A cancellation token to cancel the operation if needed. ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken cancellationToken); } diff --git a/src/Sa.HybridFileStorage/Interceptors/InterceptorContainer.cs b/src/Sa.HybridFileStorage/Interceptors/InterceptorContainer.cs index 6737d4e6..ffae3775 100644 --- a/src/Sa.HybridFileStorage/Interceptors/InterceptorContainer.cs +++ b/src/Sa.HybridFileStorage/Interceptors/InterceptorContainer.cs @@ -35,7 +35,7 @@ public async Task ExecuteBeforeUploadAsync( { foreach (var interceptor in _uploadInterceptors) { - if (!await interceptor.CanUploadAsync(storage, input, fileStream, cancellationToken)) + if (!await interceptor.CanUploadAsync(storage, input, fileStream, cancellationToken).ConfigureAwait(false)) { return false; } @@ -50,7 +50,7 @@ public async Task ExecuteAfterUploadAsync( { foreach (var interceptor in _uploadInterceptors) { - await interceptor.AfterUploadAsync(storage, result, cancellationToken); + await interceptor.AfterUploadAsync(storage, result, cancellationToken).ConfigureAwait(false); } } @@ -61,7 +61,7 @@ public async Task ExecuteOnUploadErrorAsync( { foreach (var interceptor in _uploadInterceptors) { - await interceptor.OnUploadErrorAsync(storage, exception, cancellationToken); + await interceptor.OnUploadErrorAsync(storage, exception, cancellationToken).ConfigureAwait(false); } } @@ -73,7 +73,7 @@ public async Task ExecuteBeforeDownloadAsync( { foreach (var interceptor in _downloadInterceptors) { - if (!await interceptor.CanDownloadAsync(storage, fileId, loadStream, cancellationToken)) + if (!await interceptor.CanDownloadAsync(storage, fileId, loadStream, cancellationToken).ConfigureAwait(false)) { return false; } @@ -89,7 +89,7 @@ public async Task ExecuteAfterDownloadAsync( { foreach (var interceptor in _downloadInterceptors) { - await interceptor.AfterDownloadAsync(storage, fileId, success, cancellationToken); + await interceptor.AfterDownloadAsync(storage, fileId, success, cancellationToken).ConfigureAwait(false); } } @@ -101,7 +101,7 @@ public async Task ExecuteOnDownloadErrorAsync( { foreach (var interceptor in _downloadInterceptors) { - await interceptor.OnDownloadErrorAsync(storage, fileId, exception, cancellationToken); + await interceptor.OnDownloadErrorAsync(storage, fileId, exception, cancellationToken).ConfigureAwait(false); } } @@ -112,7 +112,7 @@ public async Task ExecuteBeforeDeleteAsync( { foreach (var interceptor in _deleteInterceptors) { - if (!await interceptor.CanDeleteAsync(storage, fileId, cancellationToken)) + if (!await interceptor.CanDeleteAsync(storage, fileId, cancellationToken).ConfigureAwait(false)) { return false; } @@ -128,7 +128,7 @@ public async Task ExecuteAfterDeleteAsync( { foreach (var interceptor in _deleteInterceptors) { - await interceptor.AfterDeleteAsync(storage, fileId, success, cancellationToken); + await interceptor.AfterDeleteAsync(storage, fileId, success, cancellationToken).ConfigureAwait(false); } } @@ -140,7 +140,7 @@ public async Task ExecuteOnDeleteErrorAsync( { foreach (var interceptor in _deleteInterceptors) { - await interceptor.OnDeleteErrorAsync(storage, fileId, exception, cancellationToken); + await interceptor.OnDeleteErrorAsync(storage, fileId, exception, cancellationToken).ConfigureAwait(false); } } } diff --git a/src/Sa.HybridFileStorage/Setup.cs b/src/Sa.HybridFileStorage/Setup.cs index 31c3b961..8aca7d86 100644 --- a/src/Sa.HybridFileStorage/Setup.cs +++ b/src/Sa.HybridFileStorage/Setup.cs @@ -3,8 +3,17 @@ namespace Sa.HybridFileStorage; +/// +/// Provides extension methods for registering hybrid file storage services with the .NET Generic Host. +/// public static class Setup { + /// + /// Registers the hybrid file storage infrastructure with the specified service collection. + /// + /// The service collection to add the services to. + /// An optional action to configure the storage container and interceptors. + /// The same instance with the services added. public static IServiceCollection AddSaHybridFileStorage( this IServiceCollection services, Action? configure = null) @@ -15,7 +24,12 @@ public static IServiceCollection AddSaHybridFileStorage( return services; } - + /// + /// Registers the in-memory file storage provider with the specified service collection. + /// + /// The service collection to add the services to. + /// Optional configuration options for the in-memory storage. If null, a default instance is used. + /// The same instance with the service added. public static IServiceCollection AddSaInMemoryFileStorage( this IServiceCollection services, InMemoryFileStorageOptions? options = null) From ef4c0ea93fa4328a314787ce601d3ad28ba866c7 Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 1 Jul 2026 19:05:18 +0300 Subject: [PATCH 28/33] Hybrid storage full refactoring --- .../FileSystemStorage.cs | 30 +- src/Sa.HybridFileStorage.FileSystem/README.md | 242 +++++++++ .../Readme-ru.md | 249 +++++++++ .../FileIdParser.cs | 69 --- .../PostgresFileStorage.cs | 6 +- src/Sa.HybridFileStorage.Postgres/README.md | 316 ++++++++++++ .../Readme-ru.md | 311 +++++++++++ src/Sa.HybridFileStorage.S3/README.md | 231 +++++++++ src/Sa.HybridFileStorage.S3/Readme-ru.md | 218 ++++++++ src/Sa.HybridFileStorage.S3/S3FileStorage.cs | 21 +- src/Sa.HybridFileStorage/FileIdParser.cs | 104 ++++ .../HybridFileStorageContainer.cs | 41 +- ...HybridFileStorageContainerConfiguration.cs | 26 + .../HybridFileStorageExtensions.cs | 42 +- .../HybridStorageBuilder.cs | 16 +- .../IHybridFileStorageConfiguration.cs | 4 +- .../IHybridFileStorageContainer.cs | 12 +- .../InMemoryFileStorage.cs | 52 +- .../InMemoryFileStorageOptions.cs | 10 +- .../Interceptors/DeleteLoggingInterceptor.cs | 71 +++ .../DownloadLoggingInterceptor.cs | 75 +++ .../Interceptors/LoggingInterceptor.cs | 176 ------- .../Interceptors/Setup.cs | 24 +- .../Interceptors/UploadLoggingInterceptor.cs | 59 +++ src/Sa.HybridFileStorage/Readme-ru.md | 486 +++++++++++++----- src/Sa.HybridFileStorage/Readme.md | 486 +++++++++++++----- .../FileIdParserTests.cs | 2 +- .../PostgresFileStorageTests.cs | 2 +- 28 files changed, 2777 insertions(+), 604 deletions(-) create mode 100644 src/Sa.HybridFileStorage.FileSystem/README.md create mode 100644 src/Sa.HybridFileStorage.FileSystem/Readme-ru.md delete mode 100644 src/Sa.HybridFileStorage.Postgres/FileIdParser.cs create mode 100644 src/Sa.HybridFileStorage.Postgres/README.md create mode 100644 src/Sa.HybridFileStorage.Postgres/Readme-ru.md create mode 100644 src/Sa.HybridFileStorage.S3/README.md create mode 100644 src/Sa.HybridFileStorage.S3/Readme-ru.md create mode 100644 src/Sa.HybridFileStorage/FileIdParser.cs create mode 100644 src/Sa.HybridFileStorage/HybridFileStorageContainerConfiguration.cs create mode 100644 src/Sa.HybridFileStorage/Interceptors/DeleteLoggingInterceptor.cs create mode 100644 src/Sa.HybridFileStorage/Interceptors/DownloadLoggingInterceptor.cs delete mode 100644 src/Sa.HybridFileStorage/Interceptors/LoggingInterceptor.cs create mode 100644 src/Sa.HybridFileStorage/Interceptors/UploadLoggingInterceptor.cs diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs index c46d56be..2b2c3bb0 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs @@ -1,5 +1,4 @@ using Sa.HybridFileStorage.Domain; -using System.Globalization; using System.Runtime.CompilerServices; @@ -117,8 +116,6 @@ public async Task DownloadAsync( Options = FileOptions.Asynchronous | FileOptions.SequentialScan, }); - if (fs == null || fs == Stream.Null) return false; - await loadStream(fs, cancellationToken).ConfigureAwait(false); return true; } @@ -220,35 +217,14 @@ private void EnsurePathWithinBase(string path) if (!CanProcess(fileId)) return Task.FromResult(null); - //parse: "storageType://basket/tenant/filename" - ReadOnlySpan span = fileId.AsSpan(); - int schemeEnd = span.IndexOf(SchemeSeparator.AsSpan()); - if (schemeEnd == -1) - return Task.FromResult(null); - - var pathPart = span[(schemeEnd + SchemeSeparator.Length)..]; - - // "tenantId/filename" - int slashIndex = pathPart.IndexOf('/'); - if (slashIndex == -1) - return Task.FromResult(null); - - var scopeSpan = pathPart[..slashIndex]; - - var nextSpan = pathPart[(slashIndex + 1)..]; - slashIndex = nextSpan.IndexOf('/'); - - var tenantSpan = nextSpan[..slashIndex]; - var fileNameSpan = nextSpan[(slashIndex + 1)..]; - - if (!int.TryParse(tenantSpan, NumberStyles.None, CultureInfo.InvariantCulture, out int tenantId)) + if (!FileIdParser.TryParse(fileId, out var basket, out var tenantId, out _, out var fileName)) return Task.FromResult(null); var metadata = new FileMetadata { StorageType = StorageType, - Basket = scopeSpan.ToString(), - FileName = fileNameSpan.ToString(), + Basket = basket, + FileName = fileName, TenantId = tenantId }; diff --git a/src/Sa.HybridFileStorage.FileSystem/README.md b/src/Sa.HybridFileStorage.FileSystem/README.md new file mode 100644 index 00000000..9742e3e5 --- /dev/null +++ b/src/Sa.HybridFileStorage.FileSystem/README.md @@ -0,0 +1,242 @@ +# Sa.HybridFileStorage.FileSystem + +Local filesystem provider for `Sa.HybridFileStorage`. Stores files as physical files on disk with path sanitisation, security checks, and retry logic for transient I/O errors. + +--- + +## Table of Contents + +- [Overview](#overview) +- [File ID Format](#file-id-format) +- [Installation](#installation) +- [Quick Start](#quick-start) + - [Without DI](#without-di) + - [With DI](#with-di) +- [CRUD Examples](#crud-examples) +- [Settings Reference](#settings-reference) +- [Security](#security) +- [Error Handling](#error-handling) + +--- + +## Overview + +`FileSystemStorage` implements `IFileStorage` backed by the local file system. Files are stored under a configurable base directory using the structure: + +``` +{BasePath}/{Basket}/{TenantId}/{FileName} +``` + +Key characteristics: +- **Path sanitisation** — prevents directory traversal attacks +- **Smart preallocation** — uses `FileStreamOptions.PreallocationSize` when stream length is known +- **Retry helper** — retries `IOException` on delete operations +- **Streaming reads/writes** — configurable buffer size for memory efficiency + +--- + +## File ID Format + +``` +fs://{basket}/{tenantId}/{fileName} +``` + +**Examples:** +- `fs://documents/42/report.pdf` +- `fs://uploads/7/avatar.png` +- `fs://share/100/data.bin` + +> Note: slashes and backslashes in `FileName` are sanitized to forward slashes and stripped of leading separators. + +--- + +## Installation + +```powershell +dotnet add package Sa.HybridFileStorage.FileSystem +``` + +--- + +## Quick Start + +### Without DI + +```csharp +using Sa.HybridFileStorage.FileSystem; +using Sa.HybridFileStorage.Domain; + +var settings = new FileSystemStorageSettings +{ + BasePath = @"C:\data\files", + Basket = "documents" +}; + +using var storage = new FileSystemStorage(settings); + +// Upload +using var stream = File.OpenRead(@"C:\temp\document.pdf"); +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "document.pdf", TenantId = 42 }, + stream, ct); + +Console.WriteLine(result.FileId); // fs://documents/42/document.pdf + +// Download +bool found = await storage.DownloadAsync(result.FileId, async (fs, token) => +{ + using var reader = new StreamReader(fs, Encoding.UTF8); + var content = await reader.ReadToEndAsync(token); + Console.WriteLine(content); +}, ct); + +// Delete +bool deleted = await storage.DeleteAsync(result.FileId, ct); +``` + +### With DI + +```csharp +using Sa.HybridFileStorage.FileSystem; + +// Option 1: Immutable settings (recommended) +builder.Services.AddSaFileSystemFileStorage(new FileSystemStorageSettings +{ + BasePath = @"C:\data\files", + Basket = "documents" +}); + +// Option 2: Mutable options with fluent builder +builder.Services.AddSaFileSystemFileStorage((sp, options) => +{ + options.BasePath = @"C:\data\files"; + options.Basket = "documents"; + options.IsReadOnly = false; + options.StorageType = "fs"; +}); +``` + +--- + +## CRUD Examples + +### Upload from Stream + +```csharp +using var stream = new MemoryStream(Encoding.UTF8.GetBytes("Hello, world!")); +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "hello.txt", TenantId = 1 }, + stream, ct); + +// File created at: {BasePath}/documents/1/hello.txt +// File ID: fs://documents/1/hello.txt +``` + +### Upload from File + +Use `CopyFromFileAsync` from `Sa.HybridFileStorage` core: + +```csharp +var result = await hybridStorage.CopyFromFileAsync( + filePath: @"C:\temp\large-video.mp4", + basket: "media", + input: new UploadFileInput { FileName = "video.mp4", TenantId = 5 }, + bufferSize: 1024 * 1024, // 1 MB buffer for large files + ct: ct); +``` + +### Download to Memory + +```csharp +byte[]? downloaded = default; +await storage.DownloadAsync(result.FileId, async (stream, token) => +{ + downloaded = await stream.ReadAllBytesAsync(token); +}, ct); +``` + +### Download to Disk + +```csharp +using var destination = new FileStream(@"C:\output\downloaded.pdf", FileMode.Create); +await storage.DownloadAsync(result.FileId, async (source, token) => + await source.CopyToAsync(destination, 81920, token), + ct); +``` + +### Get Metadata + +```csharp +var metadata = await storage.GetMetadataAsync(result.FileId, ct); +if (metadata != null) +{ + Console.WriteLine($"Basket: {metadata.Basket}"); + Console.WriteLine($"Tenant: {metadata.TenantId}"); + Console.WriteLine($"Name: {metadata.FileName}"); + Console.WriteLine($"Type: {metadata.StorageType}"); +} +``` + +--- + +## Settings Reference + +### FileSystemStorageSettings (immutable) + +| Property | Description | Default | +|----------|-------------|---------| +| `BasePath` | Root directory for all files | *(required)* | +| `Basket` | Container name appended to BasePath | `"share"` | +| `StorageType` | Scheme prefix in File ID | `"fs"` | +| `IsReadOnly` | Prevent write/delete operations | `false` | +| `BufferSize` | Read/write buffer size in bytes | `262144` (256 KB) | + +### FileSystemStorageOptions (mutable, fluent builder) + +Used with the `Action` overload: + +| Property | Description | Default | +|----------|-------------|---------| +| `BasePath` | Root directory for all files | *(required)* | +| `Basket` | Container name | `"share"` | +| `StorageType` | Scheme prefix in File ID | `"fs"` | +| `IsReadOnly` | Prevent write/delete operations | `false` | + +Call `options.Validate()` after configuration to enforce required fields. + +--- + +## Security + +`FileSystemStorage` protects against directory traversal attacks: + +1. **Path sanitisation** — leading `/` or `\` characters in `FileName` are stripped; all backslashes are converted to forward slashes +2. **Base path containment** — every resolved file path is checked for belonging to `{BasePath}/{Basket}`. Attempts to escape via `../` are rejected with `SecurityException` +3. **Deterministic paths** — File IDs map to relative paths without resolution, preventing symlink-based attacks + +```csharp +// Safe — normalised to "report.pdf" +new UploadFileInput { FileName = "/api/files/download/file/var/www/report.pdf" } +// Creates: {BasePath}/documents/1/report.pdf + +// Blocked — directory traversal detected +// fileName = "../../../etc/passwd" → throws SecurityException +``` + +--- + +## Error Handling + +| Scenario | Behavior | +|----------|----------| +| `IsReadOnly = true` + upload/delete | Throws `HybridFileStorageWritableException` | +| File not found during download/delete | Returns `false` (no exception) | +| IOException on delete | Retries internally; returns `false` if all retries fail | +| Path escape attempt | Throws `SecurityException` | +| Invalid File ID format | Throws `ArgumentException` | + +--- + +## License + +MIT diff --git a/src/Sa.HybridFileStorage.FileSystem/Readme-ru.md b/src/Sa.HybridFileStorage.FileSystem/Readme-ru.md new file mode 100644 index 00000000..6fd2ddc3 --- /dev/null +++ b/src/Sa.HybridFileStorage.FileSystem/Readme-ru.md @@ -0,0 +1,249 @@ +# Sa.HybridFileStorage.FileSystem + +Провайдер локальной файловой системы для `Sa.HybridFileStorage`. Хранит файлы как физические файлы на диске с санитизацией путей, проверками безопасности и логикой повтора при преходящих ошибках ввода-вывода. + +--- + +## Содержание + +- [Обзор](#обзор) +- [Формат File ID](#формат-file-id) +- [Установка](#установка) +- [Быстрый старт](#быстрый-старт) + - [Без DI](#без-di) + - [С DI](#с-di) +- [Примеры CRUD](#примеры-crud) +- [Справочник настроек](#справочник-настроек) +- [Безопасность](#безопасность) +- [Обработка ошибок](#обработка-ошибок) + +--- + +## Обзор + +`FileSystemStorage` реализует `IFileStorage` поверх локальной файловой системы. Файлы хранятся в настраиваемой корневой директории по структуре: + +``` +{BasePath}/{Basket}/{TenantId}/{FileName} +``` + +Ключевые особенности: +- **Санитизация путей** — предотвращает атаки через обход директорий +- **Умная преаллокация** — использует `FileStreamOptions.PreallocationSize`, когда длина потока известна +- **Повтор (retry)** — повторяет `IOException` при операциях удаления +- **Потоковое чтение/запись** — настраиваемый размер буфера для эффективности памяти + +--- + +## Формат File ID + +``` +fs://{basket}/{tenantId}/{fileName} +``` + +**Примеры:** +- `fs://documents/42/report.pdf` +- `fs://uploads/7/avatar.png` +- `fs://share/100/data.bin` + +> Примечание: слеши и обратные слеши в `FileName` санитизируются в прямые слеши и очищаются от ведущих разделителей. + +--- + +## Установка + +```powershell +dotnet add package Sa.HybridFileStorage.FileSystem +``` + +--- + +## Быстрый старт + +### Без DI + +```csharp +using Sa.HybridFileStorage.FileSystem; +using Sa.HybridFileStorage.Domain; + +var settings = new FileSystemStorageSettings +{ + BasePath = @"C:\data\files", + Basket = "documents" +}; + +using var storage = new FileSystemStorage(settings); + +// Загрузка +using var stream = File.OpenRead(@"C:\temp\document.pdf"); +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "document.pdf", TenantId = 42 }, + stream, ct); + +Console.WriteLine(result.FileId); // fs://documents/42/document.pdf + +// Скачивание +bool found = await storage.DownloadAsync(result.FileId, async (fs, token) => +{ + using var reader = new StreamReader(fs, Encoding.UTF8); + var content = await reader.ReadToEndAsync(token); + Console.WriteLine(content); +}, ct); + +// Удаление +bool deleted = await storage.DeleteAsync(result.FileId, ct); +``` + +### С DI + +```csharp +using Sa.HybridFileStorage.FileSystem; + +// Вариант 1: неизменяемые настройки (рекомендуется) +builder.Services.AddSaFileSystemFileStorage(new FileSystemStorageSettings +{ + BasePath = @"C:\data\files", + Basket = "documents" +}); + +// Вариант 2: изменяемые опции с fluent builder +builder.Services.AddSaFileSystemFileStorage((sp, options) => +{ + options.BasePath = @"C:\data\files"; + options.Basket = "documents"; + options.IsReadOnly = false; + options.StorageType = "fs"; +}); +``` + +--- + +## Примеры CRUD + +### Загрузка из Stream + +```csharp +using var stream = new MemoryStream(Encoding.UTF8.GetBytes("Hello, world!")); +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "hello.txt", TenantId = 1 }, + stream, ct); + +// Файл создан: {BasePath}/documents/1/hello.txt +// File ID: fs://documents/1/hello.txt +``` + +### Загрузка из файла + +Используйте `CopyFromFileAsync` из ядра `Sa.HybridFileStorage`: + +```csharp +var result = await hybridStorage.CopyFromFileAsync( + filePath: @"C:\temp\large-video.mp4", + basket: "media", + input: new UploadFileInput { FileName = "video.mp4", TenantId = 5 }, + bufferSize: 1024 * 1024, // 1 MB буфер для больших файлов + ct: ct); +``` + +### Скачивание в память + +```csharp +byte[]? downloaded = default; +await storage.DownloadAsync(result.FileId, async (stream, token) => +{ + downloaded = await stream.ReadAllBytesAsync(token); +}, ct); +``` + +### Скачивание на диск + +```csharp +using var destination = new FileStream(@"C:\output\downloaded.pdf", FileMode.Create); +await storage.DownloadAsync(result.FileId, async (source, token) => + await source.CopyToAsync(destination, 81920, token), + ct); +``` + +### Получение метаданных + +```csharp +var metadata = await storage.GetMetadataAsync(result.FileId, ct); +if (metadata != null) +{ + Console.WriteLine($"Корзина: {metadata.Basket}"); // documents + Console.WriteLine($"Тенант: {metadata.TenantId}"); // 42 + Console.WriteLine($"Имя: {metadata.FileName}"); // report.pdf + Console.WriteLine($"Тип: {metadata.StorageType}"); // fs +} +``` + +### Удаление + +```csharp +bool deleted = await storage.DeleteAsync(result.FileId, ct); +// Возвращает false, если файл не существует +``` + +--- + +## Справочник настроек + +### FileSystemStorageSettings (неизменяемые) + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `BasePath` | Корневая директория для всех файлов | *(обязательно)* | +| `Basket` | Имя контейнера, добавляемое к BasePath | `"share"` | +| `StorageType` | Префикс схемы в File ID | `"fs"` | +| `IsReadOnly` | Запрет операций записи/удаления | `false` | +| `BufferSize` | Размер буфера чтения/записи в байтах | `262144` (256 КБ) | + +### FileSystemStorageOptions (изменяемые, fluent builder) + +Используется с перегрузкой `Action`: + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `BasePath` | Корневая директория для всех файлов | *(обязательно)* | +| `Basket` | Имя контейнера | `"share"` | +| `StorageType` | Префикс схемы в File ID | `"fs"` | +| `IsReadOnly` | Запрет операций записи/удаления | `false` | + +Вызовите `options.Validate()` после конфигурации для проверки обязательных полей. + +--- + +## Безопасность + +`FileSystemStorage` защищает от атак через обход директорий: + +1. **Санитизация путей** — ведущие символы `/` или `\` в `FileName` удаляются; все обратные слеши конвертируются в прямые +2. **Контейнирование базового пути** — каждый разрешённый путь файла проверяется на принадлежность `{BasePath}/{Basket}`. Попытки побега через `../` отклоняются с `SecurityException` +3. **Детерминированные пути** — File ID отображаются в относительные пути без вычисления, предотвращая атаки через симлинки + +```csharp +// Безопасно — нормализуется до "report.pdf" +new UploadFileInput { FileName = "/api/files/download/file/var/www/report.pdf" } +// Создаёт: {BasePath}/documents/1/report.pdf + +// Заблокировано — обнаружен обход пути +// fileName = "../../../etc/passwd" → выбрасывается SecurityException +``` + +--- + +## Обработка ошибок + +| Сценарий | Поведение | +|----------|----------| +| `IsReadOnly = true` + загрузка/удаление | Выбрасывает `HybridFileStorageWritableException` | +| Файл не найден при скачивании/удалении | Возвращает `false` (без исключения) | +| IOException при удалении | Повторяется внутренне; возвращает `false`, если все повторы неудачны | +| Попытка обхода пути | Выбрасывает `SecurityException` | +| Неверный формат File ID | Выбрасывает `ArgumentException` | + +--- + +## Лицензия + +MIT diff --git a/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs b/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs deleted file mode 100644 index 662cf157..00000000 --- a/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System.Globalization; - -namespace Sa.HybridFileStorage.Postgres; - -internal static class FileIdParser -{ - public const string SchemeSeparator = "://"; - - public static bool TryParse( - string fileId, - out string basket, - out int tenantId, - out long timestamp, - out string fileName) - { - tenantId = default; - timestamp = default; - fileName = string.Empty; - basket = string.Empty; - - if (string.IsNullOrEmpty(fileId)) return false; - - ReadOnlySpan span = fileId.AsSpan(); - int schemeEnd = span.IndexOf(SchemeSeparator.AsSpan()); - if (schemeEnd == -1) return false; - - var afterSpan = span[(schemeEnd + SchemeSeparator.Length)..]; - int scopeEnd = afterSpan.IndexOf('/'); - if (scopeEnd == -1) return false; - - basket = afterSpan[..scopeEnd].ToString(); - - afterSpan = afterSpan[(scopeEnd + 1)..]; - - - int tenantEnd = afterSpan.IndexOf('/'); - if (tenantEnd == -1) return false; - - var tenantSpan = afterSpan[..tenantEnd]; - if (!int.TryParse(tenantSpan, NumberStyles.None, CultureInfo.InvariantCulture, out tenantId)) - return false; - - var afterTenant = afterSpan[(tenantEnd + 1)..]; - int timestampEnd = afterTenant.IndexOf('/'); - if (timestampEnd == -1) return false; - - var timestampSpan = afterTenant[..timestampEnd]; - if (!long.TryParse(timestampSpan, NumberStyles.None, CultureInfo.InvariantCulture, out timestamp)) - return false; - - fileName = afterTenant[(timestampEnd + 1)..].ToString(); - return !string.IsNullOrEmpty(fileName); - } - - - public static string FormatToFileId( - string storageType, - string basket, - int tenantId, - DateTimeOffset date, - string fileName) - => $"{storageType}://{basket}/{tenantId}/{date.ToUnixTimeSeconds()}/{NormalizeFileName(fileName)}"; - - public static string NormalizeFileName(string fileName) - => fileName.TrimStart('\\', '/').Replace('\\', '/'); - - public static string GetFileExtension(string fileName) - => Path.GetExtension(fileName ?? string.Empty).ToLower().TrimStart('.'); -} diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs index fd9151fb..a75a6ff7 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs @@ -50,7 +50,7 @@ private readonly string _qualifiedTableName = $"{options.SchemaName}.\"{Sanitize(options.TableName)}\""; private readonly string _schemePrefix - = $"{options.StorageType}{FileIdParser.SchemeSeparator}{options.TableName}/"; + = $"{options.StorageType}{FileIdParser.SchemeSeparator}{Sanitize(basket)}/"; private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; @@ -75,8 +75,8 @@ public bool CanProcess(string? fileId) { var fileSpan = fileId.AsSpan(); - if (!string.IsNullOrWhiteSpace(fileId) - && fileSpan.StartsWith(_schemePrefix.AsSpan(), StringComparison.Ordinal)) return false; + if (string.IsNullOrWhiteSpace(fileId) + || !fileSpan.StartsWith(_schemePrefix.AsSpan(), StringComparison.Ordinal)) return false; int schemeEnd = fileSpan.IndexOf(FileIdParser.SchemeSeparator.AsSpan()); if (schemeEnd == -1) return false; diff --git a/src/Sa.HybridFileStorage.Postgres/README.md b/src/Sa.HybridFileStorage.Postgres/README.md new file mode 100644 index 00000000..2382e1be --- /dev/null +++ b/src/Sa.HybridFileStorage.Postgres/README.md @@ -0,0 +1,316 @@ +# Sa.HybridFileStorage.Postgres + +PostgreSQL-backed file storage provider for `Sa.HybridFileStorage`. Stores files as `BYTEA` in a partitioned database table with automatic partition management, scheduled migration, and background cleanup. + +--- + +## Table of Contents + +- [Overview](#overview) +- [File ID Format](#file-id-format) +- [Installation](#installation) +- [Quick Start](#quick-start) + - [Without DI](#without-di) + - [With DI](#with-di) +- [CRUD Examples](#crud-examples) +- [Partitioning](#partitioning) +- [Scheduled Maintenance](#scheduled-maintenance) +- [Settings Reference](#settings-reference) +- [Dependencies](#dependencies) + +--- + +## Overview + +`PostgresFileStorage` implements `IFileStorage` backed by PostgreSQL. File binary data is stored as `BYTEA` columns in a partitioned table. Key characteristics: + +- **Automatic partitioning** — declarative list + range partitioning via `Sa.Partitional.PostgreSql` +- **Upsert semantics** — `ON CONFLICT DO UPDATE` handles re-uploads transparently +- **Scheduled migrations** — background job pre-creates future partitions +- **Background cleanup** — drops old partitions beyond retention period +- **Non-seekable stream handling** — buffers unseekable streams into `RecyclableMemoryStreamManager` +- **Timestamp in File ID** — includes Unix seconds for range partition resolution + +--- + +## File ID Format + +``` +pg://{basket}/{tenantId}/{unixTimestamp}/{fileName} +``` + +**Examples:** +- `pg://files/42/1751347200/report.pdf` +- `pg://docs/7/1751347200/invoice.csv` +- `pg://share/100/1751347200/data.bin` + +> The timestamp is the Unix epoch seconds of the upload date (UTC midnight). It determines which partition the row belongs to. + +--- + +## Installation + +```powershell +dotnet add package Sa.HybridFileStorage.Postgres +``` + +This package depends on `Sa.Data.PostgreSql` and `Sa.Partitional.PostgreSql`. + +--- + +## Quick Start + +### Without DI + +```csharp +using Sa.HybridFileStorage.Postgres; +using Sa.HybridFileStorage.Domain; + +// Configure via fluent builder +var configurator = new PostgresFileStorageConfiguration(services); + +// Or register through DI extension +builder.Services.AddSaPostgreSqlFileStorage(cfg => cfg + .AddDataSource(ds => ds + .WithConnectionString("Host=localhost;Database=mydb;Username=postgres;Password=password") + .WithSearchPath("public")) + .WithSchemaName("public") + .WithTableName("files") + .WithStorageType("pg") + .ConfigureOptions((sp, options) => + { + // Customize partitioning + options.PartOptions.Basket = "files"; + options.PartOptions.PgPartBy = PgPartBy.Day; + options.PartOptions.MigrationScheduleForwardDays = 2; + + // Customize cleanup + options.CleanupOptions.ExpireDays = 365 * 3; // 3 years + })); +``` + +### With DI + +```csharp +using Sa.HybridFileStorage.Postgres; + +builder.Services.AddSaPostgreSqlFileStorage(cfg => cfg + .AddDataSource(ds => ds + .WithConnectionString("Host=db.example.com;Database=app;Username=app_user;Password=secret") + .WithSearchPath("storage")) + .WithTableName("binary_data") + .WithSchemaName("storage") + .ConfigureOptions((sp, opts) => + { + opts.PartOptions.Basket = "attachments"; + opts.PartOptions.PgPartBy = PgPartBy.Month; + opts.CleanupOptions.ExpireDays = 730; // 2 years + })); +``` + +--- + +## CRUD Examples + +### Upload from Stream + +```csharp +using var stream = new MemoryStream(Encoding.UTF8.GetBytes("Hello, Postgres!")); +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "hello.txt", TenantId = 42 }, + stream, ct); + +Console.WriteLine(result.FileId); +// pg://files/42/1751347200/hello.txt +// (timestamp = today's UTC midnight as Unix seconds) +``` + +### Upload Non-Seekable Stream + +```csharp +// Non-seekable streams are automatically buffered into RecyclableMemoryStreamManager +await using var nonSeekable = CreateNonSeekableStream(); + +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "blob.dat", TenantId = 7 }, + nonSeekable, ct); + +// Internally: copied → buffered → upserted → buffer recycled +``` + +### Download to Memory + +```csharp +byte[]? downloaded = default; +await storage.DownloadAsync(result.FileId, async (stream, token) => +{ + downloaded = await stream.ReadAllBytesAsync(token); +}, ct); +``` + +### Download Direct Processing + +```csharp +await storage.DownloadAsync(result.FileId, async (stream, token) => +{ + // Process stream directly — no intermediate buffering + using var reader = new BinaryReader(stream); + while (reader.ReadByte() is byte b) + { + // ... + } +}, ct); +``` + +### Get Metadata + +```csharp +var metadata = await storage.GetMetadataAsync(result.FileId, ct); +if (metadata != null) +{ + Console.WriteLine($"Корзина: {metadata.Basket}"); // files + Console.WriteLine($"Тенант: {metadata.TenantId}"); // 42 + Console.WriteLine($"Имя: {metadata.FileName}"); // hello.txt + Console.WriteLine($"Тип: {metadata.StorageType}"); // pg +} +``` + +### Delete + +```csharp +bool deleted = await storage.DeleteAsync(result.FileId, ct); +// Parses tenantId and timestamp from File ID for targeted DELETE +``` + +--- + +## Partitioning + +Files are stored in a partitioned table with dual partitioning strategy: + +1. **List partitioning** — by `(tenant_id, basket)` tuple +2. **Range partitioning** — by `created_at` (date) + +### Schema auto-creation + +The provider uses `Sa.Partitional.PostgreSql` to manage partitions: + +```sql +-- Auto-created table structure: +CREATE TABLE public.files ( + id TEXT NOT NULL, + name TEXT NOT NULL, + size INT NOT NULL, + file_ext TEXT NOT NULL, + tenant_id INT NOT NULL, + basket TEXT NOT NULL, + data BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL -- used for range partitioning +) PARTITION BY RANGE (created_at); + +-- Each (tenant_id, basket) pair gets its own list partition within each date range +``` + +### Partition strategies + +| Strategy | `PgPartBy` value | Use case | +|----------|------------------|----------| +| Day | `PgPartBy.Day` | High-volume systems, fine-grained cleanup | +| Month | `PgPartBy.Month` | Medium volume, balanced granularity | +| Year | `PgPartBy.Year` | Low volume, simple management | + +### Migration schedule + +New partitions are pre-created in advance (default: 2 days ahead) via a background job: + +```csharp +.ConfigureOptions((sp, opts) => +{ + opts.PartOptions.MigrationScheduleForwardDays = 2; +}) +``` + +### Cleanup schedule + +Old partitions beyond the retention period are dropped via a background job: + +```csharp +.ConfigureOptions((sp, opts) => +{ + opts.CleanupOptions.ExpireDays = 365 * 3; // drop partitions older than 3 years +}) +``` + +--- + +## Scheduled Maintenance + +Two background jobs are registered automatically: + +| Job | Purpose | Configuration | +|-----|---------|--------------| +| **Migration** | Pre-create upcoming partitions | `forwardDays`, `asBackgroundJob` | +| **Cleanup** | Drop old partitions after retention | `dropPartsAfterRetention` (TimeSpan) | + +Both run as background hosted services and use the same PostgreSQL connection pool. + +--- + +## Settings Reference + +### PostgresFileStorageOptions + +| Property | Description | Default | +|----------|-------------|---------| +| `StorageOptions.SchemaName` | PostgreSQL schema | `"public"` | +| `StorageOptions.TableName` | Table name for file data | `"files"` | +| `StorageOptions.StorageType` | Scheme prefix in File ID | `"pg"` | +| `StorageOptions.IsReadOnly` | Prevent write/delete operations | `false` | +| `PartOptions.Basket` | Scope/container name (used as list partition key) | `"share"` | +| `PartOptions.PgPartBy` | Range partitioning granularity | `PgPartBy.Day` | +| `PartOptions.MigrationScheduleForwardDays` | Days ahead to pre-create partitions | `2` | +| `CleanupOptions.ExpireDays` | Retention period before partition drop (days) | `365 * 3` | + +### IPostgresFileStorageConfiguration (fluent builder) + +| Method | Description | +|--------|-------------| +| `AddDataSource(Action?)` | Configure PostgreSQL connection | +| `WithSchemaName(string)` | Override schema name | +| `WithTableName(string)` | Override table name | +| `WithStorageType(string)` | Override storage type identifier | +| `AsReadOnly()` | Mark as read-only | +| `ConfigureOptions(Action)` | Late-stage customization | + +--- + +## Dependencies + +| Package | Purpose | +|---------|---------| +| `Sa.Data.PostgreSql` | Npgsql client (`IPgDataSource`) | +| `Sa.Partitional.PostgreSql` | Declarative partition management (`IPartitionManager`) | +| `Microsoft.IO.RecyclableMemoryStream` | Efficient memory buffering for non-seekable streams | + +--- + +## Data Model + +The underlying table structure: + +| Column | Type | Purpose | +|--------|------|---------| +| `id` | `TEXT` | Canonical File ID (primary key part) | +| `name` | `TEXT` | Original file name | +| `size` | `INT` | File size in bytes | +| `file_ext` | `TEXT` | File extension (e.g., "pdf", "png") | +| `tenant_id` | `INT` | Tenant identifier (list partition key) | +| `basket` | `TEXT` | Container/scope name (list partition key) | +| `data` | `BYTEA` | Raw file binary content | +| `created_at` | `TIMESTAMPTZ` | Upload date (UTC midnight, range partition key) | + +--- + +## License + +MIT diff --git a/src/Sa.HybridFileStorage.Postgres/Readme-ru.md b/src/Sa.HybridFileStorage.Postgres/Readme-ru.md new file mode 100644 index 00000000..13ae5ff2 --- /dev/null +++ b/src/Sa.HybridFileStorage.Postgres/Readme-ru.md @@ -0,0 +1,311 @@ +# Sa.HybridFileStorage.Postgres + +Провайдер файловых хранилищ на базе PostgreSQL для `Sa.HybridFileStorage`. Хранит файлы как `BYTEA` в партиционированной таблице с автоматическим управлением партициями, запланированной миграцией и фоновой очисткой. + +--- + +## Содержание + +- [Обзор](#обзор) +- [Формат File ID](#формат-file-id) +- [Установка](#установка) +- [Быстрый старт](#быстрый-старт) + - [Без DI](#без-di) + - [С DI](#с-di) +- [Примеры CRUD](#примеры-crud) +- [Партиционирование](#партиционирование) +- [Запланированное обслуживание](#запланированное-обслуживание) +- [Справочник настроек](#справочник-настроек) +- [Зависимости](#зависимости) + +--- + +## Обзор + +`PostgresFileStorage` реализует `IFileStorage` поверх PostgreSQL. Бинарные данные файлов хранятся в колонке `BYTEA` партиционированной таблицы. Ключевые особенности: + +- **Автоматическое партиционирование** — декларативное list + range партиционирование через `Sa.Partitional.PostgreSql` +- **Upsert-семантика** — `ON CONFLICT DO UPDATE` прозрачно обрабатывает повторные загрузки +- **Запланированная миграция** — фоновое задание заранее создаёт будущие партиции +- **Фоновая очистка** — удаляет старые партиции за пределами периода удержания +- **Обработка не-seekable потоков** — буферизует не-seekable потоки через `RecyclableMemoryStreamManager` +- **Таймстамп в File ID** — включает Unix-секунды для разрешения range-партиций + +--- + +## Формат File ID + +``` +pg://{basket}/{tenantId}/{unixTimestamp}/{fileName} +``` + +**Примеры:** +- `pg://files/42/1751347200/report.pdf` +- `pg://docs/7/1751347200/invoice.csv` +- `pg://share/100/1751347200/data.bin` + +> Таймстамп — это Unix-секунды полуночи UTC даты загрузки. Он определяет, к какой партиции относится строка. + +--- + +## Установка + +```powershell +dotnet add package Sa.HybridFileStorage.Postgres +``` + +--- + +## Быстрый старт + +### Без DI + +```csharp +using Sa.HybridFileStorage.Postgres; +using Sa.HybridFileStorage.Domain; + +// Регистрация через fluent builder +builder.Services.AddSaPostgreSqlFileStorage(cfg => cfg + .AddDataSource(ds => ds + .WithConnectionString("Host=localhost;Database=mydb;Username=postgres;Password=password") + .WithSearchPath("public")) + .WithSchemaName("public") + .WithTableName("files") + .WithStorageType("pg") + .ConfigureOptions((sp, options) => + { + // Настройка партиционирования + options.PartOptions.Basket = "files"; + options.PartOptions.PgPartBy = PgPartBy.Day; + options.PartOptions.MigrationScheduleForwardDays = 2; + + // Настройка очистки + options.CleanupOptions.ExpireDays = 365 * 3; // 3 года + })); +``` + +### С DI + +```csharp +using Sa.HybridFileStorage.Postgres; + +builder.Services.AddSaPostgreSqlFileStorage(cfg => cfg + .AddDataSource(ds => ds + .WithConnectionString("Host=db.example.com;Database=app;Username=app_user;Password=secret") + .WithSearchPath("storage")) + .WithTableName("binary_data") + .WithSchemaName("storage") + .ConfigureOptions((sp, opts) => + { + opts.PartOptions.Basket = "attachments"; + opts.PartOptions.PgPartBy = PgPartBy.Month; + opts.CleanupOptions.ExpireDays = 730; // 2 года + })); +``` + +--- + +## Примеры CRUD + +### Загрузка из Stream + +```csharp +using var stream = new MemoryStream(Encoding.UTF8.GetBytes("Hello, Postgres!")); +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "hello.txt", TenantId = 42 }, + stream, ct); + +Console.WriteLine(result.FileId); +// pg://files/42/1751347200/hello.txt +// (timestamp = полуночь UTC сегодняшнего дня в Unix-секундах) +``` + +### Загрузка не-seekable потока + +```csharp +// Не-seekable потоки автоматически буферизуются в RecyclableMemoryStreamManager +await using var nonSeekable = CreateNonSeekableStream(); + +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "blob.dat", TenantId = 7 }, + nonSeekable, ct); + +// Внутренне: скопирован → буферизован → upsert → буфер перезапущен +``` + +### Скачивание в память + +```csharp +byte[]? downloaded = default; +await storage.DownloadAsync(result.FileId, async (stream, token) => +{ + downloaded = await stream.ReadAllBytesAsync(token); +}, ct); +``` + +### Прямая обработка при скачивании + +```csharp +await storage.DownloadAsync(result.FileId, async (stream, token) => +{ + // Обрабатываем поток напрямую — без промежуточной буферизации + using var reader = new BinaryReader(stream); + while (reader.ReadByte() is byte b) + { + // ... + } +}, ct); +``` + +### Получение метаданных + +```csharp +var metadata = await storage.GetMetadataAsync(result.FileId, ct); +if (metadata != null) +{ + Console.WriteLine($"Корзина: {metadata.Basket}"); // files + Console.WriteLine($"Тенант: {metadata.TenantId}"); // 42 + Console.WriteLine($"Имя: {metadata.FileName}"); // hello.txt + Console.WriteLine($"Тип: {metadata.StorageType}"); // pg +} +``` + +### Удаление + +```csharp +bool deleted = await storage.DeleteAsync(result.FileId, ct); +// Парсит tenantId и timestamp из File ID для таргетированного DELETE +``` + +--- + +## Партиционирование + +Файлы хранятся в партиционированной таблице с двойной стратегией: + +1. **List-партиционирование** — по кортежу `(tenant_id, basket)` +2. **Range-партиционирование** — по `created_at` (дата) + +### Авто-создание схемы + +Провайдер использует `Sa.Partitional.PostgreSql` для управления партициями: + +```sql +-- Автоматически созданная структура таблицы: +CREATE TABLE public.files ( + id TEXT NOT NULL, + name TEXT NOT NULL, + size INT NOT NULL, + file_ext TEXT NOT NULL, + tenant_id INT NOT NULL, + basket TEXT NOT NULL, + data BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL -- используется для range-партиционирования +) PARTITION BY RANGE (created_at); + +-- Каждая пара (tenant_id, basket) получает собственную list-партию внутри каждого диапазона дат +``` + +### Стратегии партиционирования + +| Стратегия | Значение `PgPartBy` | Сценарий использования | +|-----------|---------------------|----------------------| +| День | `PgPartBy.Day` | Высоконагруженные системы, тонкая очистка | +| Месяц | `PgPartBy.Month` | Средняя нагрузка, сбалансированная гранулярность | +| Год | `PgPartBy.Year` | Низкая нагрузка, простое управление | + +### Расписание миграции + +Новые партиции создаются заранее (по умолчанию: за 2 дня) через фоновое задание: + +```csharp +.ConfigureOptions((sp, opts) => +{ + opts.PartOptions.MigrationScheduleForwardDays = 2; +}) +``` + +### Расписание очистки + +Старые партиции за пределами периода удержания удаляются через фоновое задание: + +```csharp +.ConfigureOptions((sp, opts) => +{ + opts.CleanupOptions.ExpireDays = 365 * 3; // удалять партиции старше 3 лет +}) +``` + +--- + +## Запланированное обслуживание + +Автоматически регистрируются два фоновых задания: + +| Задание | Назначение | Конфигурация | +|---------|-----------|-------------| +| **Миграция** | Заранее создавать будущие партиции | `forwardDays`, `asBackgroundJob` | +| **Очистка** | Удалять старые партиции после истечения срока | `dropPartsAfterRetention` (TimeSpan) | + +Оба работают как фоновые hosted-сервисы и используют общий пул подключений PostgreSQL. + +--- + +## Справочник настроек + +### PostgresFileStorageOptions + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `StorageOptions.SchemaName` | Схема PostgreSQL | `"public"` | +| `StorageOptions.TableName` | Имя таблицы для данных файлов | `"files"` | +| `StorageOptions.StorageType` | Префикс схемы в File ID | `"pg"` | +| `StorageOptions.IsReadOnly` | Запрет операций записи/удаления | `false` | +| `PartOptions.Basket` | Имя контейнера (ключ list-партиции) | `"share"` | +| `PartOptions.PgPartBy` | Гранулярность range-партиционирования | `PgPartBy.Day` | +| `PartOptions.MigrationScheduleForwardDays` | Дней заранее для предсоздания партиций | `2` | +| `CleanupOptions.ExpireDays` | Период удержания перед удалением партиции (дни) | `365 * 3` | + +### IPostgresFileStorageConfiguration (fluent builder) + +| Метод | Описание | +|-------|----------| +| `AddDataSource(Action?)` | Настроить подключение PostgreSQL | +| `WithSchemaName(string)` | Переопределить имя схемы | +| `WithTableName(string)` | Переопределить имя таблицы | +| `WithStorageType(string)` | Переопределить идентификатор типа хранилища | +| `AsReadOnly()` | Пометить как read-only | +| `ConfigureOptions(Action)` | Поздняя кастомизация | + +--- + +## Зависимости + +| Пакет | Назначение | +|-------|-----------| +| `Sa.Data.PostgreSql` | Npgsql клиент (`IPgDataSource`) | +| `Sa.Partitional.PostgreSql` | Управление партициями (`IPartitionManager`) | +| `Microsoft.IO.RecyclableMemoryStream` | Эффективная буферизация памяти для не-seekable потоков | + +--- + +## Модель данных + +Структура базовой таблицы: + +| Колонка | Тип | Назначение | +|---------|-----|-----------| +| `id` | `TEXT` | Канонический File ID (часть первичного ключа) | +| `name` | `TEXT` | Оригинальное имя файла | +| `size` | `INT` | Размер файла в байтах | +| `file_ext` | `TEXT` | Расширение файла (напр., "pdf", "png") | +| `tenant_id` | `INT` | Идентификатор тенанта (ключ list-партиции) | +| `basket` | `TEXT` | Имя контейнера (ключ list-партиции) | +| `data` | `BYTEA` | Сырые бинарные данные файла | +| `created_at` | `TIMESTAMPTZ` | Дата загрузки (полуночь UTC, ключ range-партиции) | + +--- + +## Лицензия + +MIT diff --git a/src/Sa.HybridFileStorage.S3/README.md b/src/Sa.HybridFileStorage.S3/README.md new file mode 100644 index 00000000..d1f2c4cb --- /dev/null +++ b/src/Sa.HybridFileStorage.S3/README.md @@ -0,0 +1,231 @@ +# Sa.HybridFileStorage.S3 + +S3-compatible cloud storage provider for `Sa.HybridFileStorage`. Wraps Minio/AWS S3 with automatic bucket creation, MIME type detection, and streaming file I/O. + +--- + +## Table of Contents + +- [Overview](#overview) +- [File ID Format](#file-id-format) +- [Installation](#installation) +- [Quick Start](#quick-start) + - [Without DI](#without-di) + - [With DI](#with-di) +- [CRUD Examples](#crud-examples) +- [Settings Reference](#settings-reference) +- [Dependencies](#dependencies) + +--- + +## Overview + +`S3FileStorage` implements `IFileStorage` backed by any S3-compatible service (AWS S3, MinIO, DigitalOcean Spaces, etc.). Key characteristics: + +- **Auto bucket creation** — creates the target bucket on first upload if it doesn't exist (thread-safe, single-flight) +- **MIME type detection** — resolves content type from file extension using `MimeTypeMap` +- **Streaming uploads/downloads** — uses `IS3BucketClient` for memory-efficient transfers +- **Thread-safe initialization** — concurrent `EnsureBucket` calls are deduplicated via `Interlocked.CompareExchange` + +--- + +## File ID Format + +``` +s3://{basket}/{tenantId}/{fileName} +``` + +**Examples:** +- `s3://uploads/42/document.pdf` +- `s3://avatars/7/profile.png` +- `s3://backups/100/database.sql` + +--- + +## Installation + +```powershell +dotnet add package Sa.HybridFileStorage.S3 +``` + +This package depends on `Sa.Data.S3` which provides the underlying `IS3BucketClient`. + +--- + +## Quick Start + +### Without DI + +```csharp +using Sa.HybridFileStorage.S3; +using Sa.HybridFileStorage.Domain; +using Sa.Data.S3; + +// Configure S3 client +var setup = new S3BucketClientSetupSettings +{ + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123", + Bucket = "mybucket", + Region = "us-east-1" +}; + +var client = new S3BucketClient(setup); + +var options = new S3FileStorageOptions +{ + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123", + Bucket = "mybucket", + Basket = "uploads", + Region = "us-east-1" +}; + +using var storage = new S3FileStorage(client, options); + +// Upload +using var stream = File.OpenRead(@"C:\temp\document.pdf"); +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "document.pdf", TenantId = 42 }, + stream, ct); + +Console.WriteLine(result.FileId); // s3://uploads/42/document.pdf +Console.WriteLine(result.AbsoluteUrl); // http://localhost:9000/mybucket/uploads/42/document.pdf + +// Download +bool found = await storage.DownloadAsync(result.FileId, async (fs, token) => +{ + using var reader = new StreamReader(fs, Encoding.UTF8); + var content = await reader.ReadToEndAsync(token); + Console.WriteLine(content); +}, ct); + +// Delete +bool deleted = await storage.DeleteAsync(result.FileId, ct); +``` + +### With DI + +```csharp +using Sa.HybridFileStorage.S3; + +builder.Services.AddSaS3FileStorage(new S3FileStorageOptions +{ + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123", + Bucket = "mybucket", + Basket = "uploads", + Region = "us-east-1" +}); + +// The DI container resolves IS3BucketClient automatically +``` + +--- + +## CRUD Examples + +### Upload from Stream + +```csharp +using var stream = new MemoryStream(Encoding.UTF8.GetBytes("Hello, S3!")); +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "hello.txt", TenantId = 1 }, + stream, ct); + +// File stored at: mybucket/uploads/1/hello.txt +// Content-Type: text/plain (auto-detected from extension) +``` + +### Upload Large Files + +```csharp +// Streaming large files without loading into memory +await using var fs = new FileStream(@"C:\large\data.zip", new FileStreamOptions +{ + Mode = FileMode.Open, + Access = FileAccess.Read, + Share = FileShare.Read, + Options = FileOptions.Asynchronous | FileOptions.SequentialScan +}); + +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "data.zip", TenantId = 5 }, + fs, ct); +``` + +### Download to Memory + +```csharp +byte[]? downloaded = default; +await storage.DownloadAsync(result.FileId, async (stream, token) => +{ + downloaded = await stream.ReadAllBytesAsync(token); +}, ct); +``` + +### Download to Disk + +```csharp +await using var destination = new FileStream(@"C:\output\downloaded.pdf", FileMode.Create); +await storage.DownloadAsync(result.FileId, async (source, token) => + await source.CopyToAsync(destination, 1024 * 1024, token), // 1 MB buffer + ct); +``` + +### Get Metadata + +```csharp +var metadata = await storage.GetMetadataAsync(result.FileId, ct); +if (metadata != null) +{ + Console.WriteLine($"Basket: {metadata.Basket}"); // uploads + Console.WriteLine($"Tenant: {metadata.TenantId}"); // 42 + Console.WriteLine($"Name: {metadata.FileName}"); // document.pdf + Console.WriteLine($"Type: {metadata.StorageType}"); // s3 +} +``` + +### Delete + +```csharp +bool deleted = await storage.DeleteAsync(result.FileId, ct); +// Always returns true if CanProcess(fileId) == true — S3 deletes silently ignore missing objects +``` + +--- + +## Settings Reference + +### S3FileStorageOptions + +| Property | Description | Default | +|----------|-------------|---------| +| `Endpoint` | S3-compatible endpoint URL | *(required)* | +| `AccessKey` | Access key ID | *(required)* | +| `SecretKey` | Secret access key | *(required)* | +| `Bucket` | Target bucket name | *(required)* | +| `Basket` | Scope/container prefix within the bucket | `"share"` | +| `Region` | AWS region for SigV4 signing | `"eu-central-1"` | +| `StorageType` | Scheme prefix in File ID | `"s3"` | +| `IsReadOnly` | Prevent write/delete operations | `false` | + +--- + +## Dependencies + +| Package | Purpose | +|---------|---------| +| `Sa.Data.S3` | S3 client (`IS3BucketClient`, `S3BucketClient`) | +| `Sa` | Shared utilities (`MimeTypeMap`) | + +The `AddSaS3FileStorage` extension method automatically registers `IS3BucketClient` via `AddSaS3BucketClient`. + +--- + +## License + +MIT diff --git a/src/Sa.HybridFileStorage.S3/Readme-ru.md b/src/Sa.HybridFileStorage.S3/Readme-ru.md new file mode 100644 index 00000000..01fc1fc8 --- /dev/null +++ b/src/Sa.HybridFileStorage.S3/Readme-ru.md @@ -0,0 +1,218 @@ +# Sa.HybridFileStorage.S3 + +Провайдер S3-совместимого облачного хранилища для `Sa.HybridFileStorage`. Использует `Sa.Data.S3` клиент с автосозданием бакета, автоопределением MIME-типов и SigV4 подписью. + +--- + +## Содержание + +- [Обзор](#обзор) +- [Формат File ID](#формат-file-id) +- [Установка](#установка) +- [Быстрый старт](#быстрый-старт) + - [Без DI](#без-di) + - [С DI](#с-di) +- [Примеры CRUD](#примеры-crud) +- [Справочник настроек](#справочник-настроек) +- [Зависимости](#зависимости) + +--- + +## Обзор + +`S3FileStorage` реализует `IFileStorage` поверх S3-совместимых хранилищ (AWS S3, MinIO, DigitalOcean Spaces и др.). Файлы хранятся в бакете по структуре: + +``` +{Bucket}/{Basket}/{TenantId}/{FileName} +``` + +Ключевые особенности: +- **Автосоздание бакета** — если бакет не существует, создаётся при первой загрузке (потокобезопасно через `Interlocked.CompareExchange`) +- **Авто-MIME** — определяет `Content-Type` из расширения файла через `MimeTypeMap` +- **SigV4 подпись** — поддерживает AWS Signature Version 4 +- **Потоковая передача** — потоки копируются напрямую без промежуточных буферов + +--- + +## Формат File ID + +``` +s3://{basket}/{tenantId}/{fileName} +``` + +**Примеры:** +- `s3://uploads/42/document.pdf` +- `s3://media/7/video.mp4` +- `s3://share/100/data.bin` + +--- + +## Установка + +```powershell +dotnet add package Sa.HybridFileStorage.S3 +``` + +--- + +## Быстрый старт + +### Без DI + +```csharp +using Sa.HybridFileStorage.S3; +using Sa.HybridFileStorage.Domain; + +var options = new S3FileStorageOptions +{ + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123", + Bucket = "mybucket", + Basket = "uploads" +}; + +// Требуется предварительно зарегистрированный IS3BucketClient +var client = new S3BucketClient(new S3BucketClientSetupSettings +{ + Endpoint = options.Endpoint, + AccessKey = options.AccessKey, + SecretKey = options.SecretKey, + Region = options.Region, + Bucket = options.Bucket +}); + +using var storage = new S3FileStorage(client, options); + +// Загрузка +using var stream = File.OpenRead(@"C:\temp\document.pdf"); +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "document.pdf", TenantId = 42 }, + stream, ct); + +Console.WriteLine(result.FileId); // s3://uploads/42/document.pdf +Console.WriteLine(result.AbsoluteUrl); // http://localhost:9000/mybucket/uploads/42/document.pdf +``` + +### С DI + +```csharp +using Sa.HybridFileStorage.S3; + +builder.Services.AddSaS3FileStorage(new S3FileStorageOptions +{ + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123", + Bucket = "mybucket", + Basket = "uploads" +}); + +// AddSaS3FileStorage автоматически регистрирует IS3BucketClient через AddSaS3BucketClient +``` + +--- + +## Примеры CRUD + +### Загрузка из Stream + +```csharp +using var stream = new MemoryStream(Encoding.UTF8.GetBytes("Hello, S3!")); +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "hello.txt", TenantId = 1 }, + stream, ct); + +// Файл сохранён: mybucket/uploads/1/hello.txt +// Content-Type: text/plain (автоопределён из расширения) +``` + +### Загрузка больших файлов + +```csharp +// Потоковая загрузка без загрузки в память +await using var fs = new FileStream(@"C:\large\data.zip", new FileStreamOptions +{ + Mode = FileMode.Open, + Access = FileAccess.Read, + Share = FileShare.Read, + Options = FileOptions.Asynchronous | FileOptions.SequentialScan +}); + +var result = await storage.UploadAsync( + new UploadFileInput { FileName = "data.zip", TenantId = 5 }, + fs, ct); +``` + +### Скачивание в память + +```csharp +byte[]? downloaded = default; +await storage.DownloadAsync(result.FileId, async (stream, token) => +{ + downloaded = await stream.ReadAllBytesAsync(token); +}, ct); +``` + +### Скачивание на диск + +```csharp +await using var destination = new FileStream(@"C:\output\downloaded.pdf", FileMode.Create); +await storage.DownloadAsync(result.FileId, async (source, token) => + await source.CopyToAsync(destination, 1024 * 1024, token), // 1 MB буфер + ct); +``` + +### Получение метаданных + +```csharp +var metadata = await storage.GetMetadataAsync(result.FileId, ct); +if (metadata != null) +{ + Console.WriteLine($"Корзина: {metadata.Basket}"); // uploads + Console.WriteLine($"Тенант: {metadata.TenantId}"); // 42 + Console.WriteLine($"Имя: {metadata.FileName}"); // document.pdf + Console.WriteLine($"Тип: {metadata.StorageType}"); // s3 +} +``` + +### Удаление + +```csharp +bool deleted = await storage.DeleteAsync(result.FileId, ct); +// Всегда возвращает true, если CanProcess(fileId) == true — S3 игнорирует отсутствие объектов +``` + +--- + +## Справочник настроек + +### S3FileStorageOptions + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `Endpoint` | URL S3-совместимого эндпоинта | *(обязательно)* | +| `AccessKey` | Ключ доступа | *(обязательно)* | +| `SecretKey` | Секретный ключ | *(обязательно)* | +| `Bucket` | Имя целевого бакета | *(обязательно)* | +| `Basket` | Префикс области/контейнера внутри бакета | `"share"` | +| `Region` | Регион AWS для SigV4 подписи | `"eu-central-1"` | +| `StorageType` | Префикс схемы в File ID | `"s3"` | +| `IsReadOnly` | Запрет операций записи/удаления | `false` | + +--- + +## Зависимости + +| Пакет | Назначение | +|-------|-----------| +| `Sa.Data.S3` | S3 клиент (`IS3BucketClient`, `S3BucketClient`) | +| `Sa` | Общие утилиты (`MimeTypeMap`) | + +Метод расширения `AddSaS3FileStorage` автоматически регистрирует `IS3BucketClient` через `AddSaS3BucketClient`. + +--- + +## Лицензия + +MIT diff --git a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs index 25fa4873..e1b42047 100644 --- a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs +++ b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs @@ -1,5 +1,6 @@ using Sa.Classes; using Sa.Data.S3; +using Sa.HybridFileStorage; using Sa.HybridFileStorage.Domain; using System.Globalization; using System.Runtime.CompilerServices; @@ -146,28 +147,14 @@ private static string GetFilePath(string fileId) { if (!CanProcess(fileId)) return null; - var filePath = GetFilePath(fileId); - ReadOnlySpan pathSpan = filePath.AsSpan(); - - // Парсинг пути: "scope/tenantId/filename" - int firstSlash = pathSpan.IndexOf('/'); - if (firstSlash == -1) return null; - - var afterScope = pathSpan[(firstSlash + 1)..]; - int secondSlash = afterScope.IndexOf('/'); - if (secondSlash == -1) return null; - - var tenantSpan = afterScope[..secondSlash]; - var fileNameSpan = afterScope[(secondSlash + 1)..]; - - if (!int.TryParse(tenantSpan, NumberStyles.None, CultureInfo.InvariantCulture, out int tenantId)) + if (!FileIdParser.TryParse(fileId, out var basket, out var tenantId, out _, out var fileName)) return null; return new FileMetadata { - Basket = Basket, + Basket = basket, StorageType = StorageType, - FileName = fileNameSpan.ToString(), + FileName = fileName, TenantId = tenantId }; } diff --git a/src/Sa.HybridFileStorage/FileIdParser.cs b/src/Sa.HybridFileStorage/FileIdParser.cs new file mode 100644 index 00000000..ed529d53 --- /dev/null +++ b/src/Sa.HybridFileStorage/FileIdParser.cs @@ -0,0 +1,104 @@ +using System.Globalization; + +namespace Sa.HybridFileStorage; + +/// +/// Utility for parsing and constructing file IDs in the format "storageType://basket/tenantId/timestamp/filename". +/// +public static class FileIdParser +{ + /// + /// The scheme separator used between storage type and path. + /// + public const string SchemeSeparator = "://"; + + /// + /// Tries to parse a file ID into its constituent parts. + /// + /// The file ID string to parse. + /// When this method returns, contains the basket (scope) name, or empty string if parsing failed. + /// When this method returns, contains the tenant identifier, or zero if parsing failed. + /// When this method returns, contains the Unix timestamp (seconds), or zero if parsing failed. + /// When this method returns, contains the file name, or empty string if parsing failed. + /// true if the file ID was parsed successfully; otherwise, false. + public static bool TryParse( + string fileId, + out string basket, + out int tenantId, + out long timestamp, + out string fileName) + { + tenantId = default; + timestamp = default; + fileName = string.Empty; + basket = string.Empty; + + if (string.IsNullOrEmpty(fileId)) return false; + + ReadOnlySpan span = fileId.AsSpan(); + int schemeEnd = span.IndexOf(SchemeSeparator.AsSpan()); + if (schemeEnd == -1) return false; + + var afterSpan = span[(schemeEnd + SchemeSeparator.Length)..]; + int scopeEnd = afterSpan.IndexOf('/'); + if (scopeEnd == -1) return false; + + basket = afterSpan[..scopeEnd].ToString(); + afterSpan = afterSpan[(scopeEnd + 1)..]; + + // For Postgres: "tenantId/timestamp/filename" + // For S3/FileSystem/InMemory: "tenantId/filename" (no timestamp) + int firstSlash = afterSpan.IndexOf('/'); + if (firstSlash == -1) return false; + + var firstPart = afterSpan[..firstSlash]; + if (!int.TryParse(firstPart, NumberStyles.None, CultureInfo.InvariantCulture, out tenantId)) + return false; + + var afterFirst = afterSpan[(firstSlash + 1)..]; + + // Check if second part is a timestamp (Postgres) or filename (others) + int secondSlash = afterFirst.IndexOf('/'); + if (secondSlash != -1 && long.TryParse(afterFirst[..secondSlash], NumberStyles.None, CultureInfo.InvariantCulture, out timestamp)) + { + // Postgres format: tenantId/timestamp/filename + fileName = afterFirst[(secondSlash + 1)..].ToString(); + } + else + { + // Simple format: tenantId/filename + fileName = afterFirst.ToString(); + } + + return !string.IsNullOrEmpty(fileName); + } + + /// + /// Formats a file ID using the Postgres-compatible format with timestamp. + /// + /// The storage type identifier (e.g., "pg"). + /// The basket (scope) name. + /// The tenant identifier. + /// The date associated with the file. + /// The file name. + /// A formatted file ID string. + public static string FormatToFileId( + string storageType, + string basket, + int tenantId, + DateTimeOffset date, + string fileName) + => $"{storageType}://{basket}/{tenantId}/{date.ToUnixTimeSeconds()}/{NormalizeFileName(fileName)}"; + + /// + /// Normalizes a file name by removing leading slashes/backslashes and converting backslashes to forward slashes. + /// + public static string NormalizeFileName(string fileName) + => fileName.TrimStart('\\', '/').Replace('\\', '/'); + + /// + /// Extracts the file extension from a file name. + /// + public static string GetFileExtension(string fileName) + => Path.GetExtension(fileName ?? string.Empty).ToLower().TrimStart('.'); +} diff --git a/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs b/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs index 2a75ebe9..96b0b0fb 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs @@ -2,18 +2,49 @@ namespace Sa.HybridFileStorage; +/// +/// Thread-safe container for multiple providers. +/// Uses copy-on-write semantics to allow safe enumeration during concurrent mutations. +/// internal sealed class HybridFileStorageContainer(IEnumerable storages) : IHybridFileStorageContainer { + private readonly ReaderWriterLockSlim _lock = new(LockRecursionPolicy.NoRecursion); private readonly List _storages = [.. storages]; - public IHybridFileStorageContainerConfiguration AddStorage(IFileStorage storage) + /// + public HybridFileStorageContainerConfiguration AddStorage(IFileStorage storage) { - if (!_storages.Contains(storage)) + _lock.EnterWriteLock(); + try { - _storages.Add(storage); + if (!_storages.Contains(storage)) + { + _storages.Add(storage); + } } - return this; + finally + { + _lock.ExitWriteLock(); + } + + return new HybridFileStorageContainerConfiguration(AddStorage); } - public IEnumerable Storages => _storages; + /// + public IEnumerable Storages + { + get + { + _lock.EnterReadLock(); + try + { + // Snapshot to avoid holding the lock during enumeration + return [.. _storages]; + } + finally + { + _lock.ExitReadLock(); + } + } + } } diff --git a/src/Sa.HybridFileStorage/HybridFileStorageContainerConfiguration.cs b/src/Sa.HybridFileStorage/HybridFileStorageContainerConfiguration.cs new file mode 100644 index 00000000..06a862ad --- /dev/null +++ b/src/Sa.HybridFileStorage/HybridFileStorageContainerConfiguration.cs @@ -0,0 +1,26 @@ +using Sa.HybridFileStorage.Domain; + +namespace Sa.HybridFileStorage; + +/// +/// Fluent builder for registering file storage providers into the hybrid container. +/// +public sealed class HybridFileStorageContainerConfiguration +{ + private readonly Func _addStorage; + + internal HybridFileStorageContainerConfiguration(Func addStorage) + { + _addStorage = addStorage ?? throw new ArgumentNullException(nameof(addStorage)); + } + + /// + /// Adds a file storage provider to the hybrid file storage container. + /// + /// The storage implementation to register. + /// The same instance for chaining. + public HybridFileStorageContainerConfiguration AddStorage(IFileStorage storage) + { + return _addStorage(storage); + } +} diff --git a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs index 58ee5566..35c0b8dd 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs @@ -1,14 +1,8 @@ using Sa.HybridFileStorage.Domain; +using System.Diagnostics.CodeAnalysis; namespace Sa.HybridFileStorage; -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Threading; -using System.Threading.Tasks; - /// /// Provides extension methods for common file storage operations on . /// @@ -184,20 +178,27 @@ public static async Task> CopyToScopeBatchAsync( } catch (Exception ex) { - // Блокировка для безопасного изменения списков и атомарного отчета о прогрессе + // Lock only for adding to failed list and incrementing counter + int failedIdx; + int completedCount; lock (lockObj) { failed.Add(new BatchError(fileId, ex, index)); completed++; - - progress?.Report(new BatchOperationProgress( - fileList.Count, - completed, - succeeded.Count, - failed.Count, - fileId, - ex)); + failedIdx = failed.Count - 1; + completedCount = completed; } + + // Progress.Report outside lock to avoid blocking other threads during user callback + var reportedError = failed[failedIdx]; + progress?.Report(new BatchOperationProgress( + fileList.Count, + completedCount, + succeeded.Count, + failed.Count, + reportedError.FileId, + reportedError.Exception)); + return null; } } @@ -231,18 +232,21 @@ await Parallel.ForEachAsync( if (result is not null) { + BatchOperationProgress progressSnapshot; lock (lockObj) { succeeded.Add(result); completed++; - - progress?.Report(new BatchOperationProgress( + progressSnapshot = new BatchOperationProgress( fileList.Count, completed, succeeded.Count, failed.Count, - fileId)); + fileId); } + + // Report outside lock to avoid blocking other threads during user callback + progress?.Report(progressSnapshot); } }); diff --git a/src/Sa.HybridFileStorage/HybridStorageBuilder.cs b/src/Sa.HybridFileStorage/HybridStorageBuilder.cs index e7b80fcd..cd1fe9bb 100644 --- a/src/Sa.HybridFileStorage/HybridStorageBuilder.cs +++ b/src/Sa.HybridFileStorage/HybridStorageBuilder.cs @@ -7,12 +7,12 @@ namespace Sa.HybridFileStorage; internal sealed class HybridStorageBuilder(IServiceCollection services) : IHybridFileStorageConfiguration { - private Action? _configureStorage; + private Action? _configureStorage; private Action? _configureInterceptors; private bool _logged = false; public IHybridFileStorageConfiguration ConfigureStorage( - Action configure) + Action configure) { _configureStorage = configure; return this; @@ -35,19 +35,25 @@ public void Build() { if (_logged) { - services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); } services.TryAddSingleton(sp => { InterceptorContainer interceptorContainer = new(); - interceptorContainer.AddLoggingInterceptor(sp.GetService()); + interceptorContainer.AddLoggingInterceptors( + sp.GetService(), + sp.GetService(), + sp.GetService()); _configureInterceptors?.Invoke(sp, interceptorContainer); HybridFileStorageContainer storageContainer = new(sp.GetServices()); - _configureStorage?.Invoke(sp, storageContainer); + var storageConfig = new HybridFileStorageContainerConfiguration(storageContainer.AddStorage); + _configureStorage?.Invoke(sp, storageConfig); return new HybridFileStorage(storageContainer, interceptorContainer); }); diff --git a/src/Sa.HybridFileStorage/IHybridFileStorageConfiguration.cs b/src/Sa.HybridFileStorage/IHybridFileStorageConfiguration.cs index 7115a356..d0fb709d 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorageConfiguration.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorageConfiguration.cs @@ -18,10 +18,10 @@ IHybridFileStorageConfiguration ConfigureInterceptors( /// /// Configures storage providers that will participate in the hybrid file storage system. /// - /// An action that receives a for registering storage implementations. + /// An action that receives a for registering storage implementations. /// The same instance for fluent chaining. IHybridFileStorageConfiguration ConfigureStorage( - Action configure); + Action configure); /// /// Enables automatic logging of file storage operations through registered interceptors. diff --git a/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs b/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs index a4da65ce..6c0efb51 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs @@ -2,10 +2,20 @@ namespace Sa.HybridFileStorage; -public interface IHybridFileStorageContainer : IHybridFileStorageContainerConfiguration +/// +/// Thread-safe container for multiple providers. +/// +public interface IHybridFileStorageContainer { /// /// Gets the collection of registered file storage providers. /// IEnumerable Storages { get; } + + /// + /// Adds a file storage provider to the hybrid container. + /// + /// The storage implementation to register. + /// The same instance for chaining. + HybridFileStorageContainerConfiguration AddStorage(IFileStorage storage); } diff --git a/src/Sa.HybridFileStorage/InMemoryFileStorage.cs b/src/Sa.HybridFileStorage/InMemoryFileStorage.cs index ae0d6a1e..8b4aaf6a 100644 --- a/src/Sa.HybridFileStorage/InMemoryFileStorage.cs +++ b/src/Sa.HybridFileStorage/InMemoryFileStorage.cs @@ -1,6 +1,5 @@ using Sa.HybridFileStorage.Domain; using System.Collections.Concurrent; -using System.Globalization; namespace Sa.HybridFileStorage; @@ -22,7 +21,7 @@ public sealed class InMemoryFileStorage( public const string DefaultStorageType = "mem"; private readonly ConcurrentDictionary _storage = []; - + private long _totalSizeBytes; /// /// Gets the basket (container) name used by this storage instance. @@ -64,6 +63,19 @@ await fileStream.CopyToAsync(memoryStream, cancellationToken) .ConfigureAwait(false); byte[] fileData = memoryStream.ToArray(); + // Check size limit before inserting + if (_options.MaxSizeBytes > 0) + { + long newSize = Interlocked.Add(ref _totalSizeBytes, fileData.Length); + if (newSize > _options.MaxSizeBytes) + { + // Rollback the addition and throw + Interlocked.Add(ref _totalSizeBytes, -fileData.Length); + throw new InvalidOperationException( + $"In-memory storage size limit ({_options.MaxSizeBytes} bytes) exceeded."); + } + } + string path = Path.Combine(Basket, metadata.TenantId.ToString(), metadata.FileName).Replace('\\', '/'); //"storageType://basket/tenant/filename" string fileId = $"{StorageType}{SchemeSeparator}{path}"; @@ -91,7 +103,14 @@ await loadStream(memoryStream, cancellationToken) public Task DeleteAsync(string fileId, CancellationToken cancellationToken) { EnsureWritable(); - return Task.FromResult(_storage.TryRemove(fileId, out _)); + + if (_storage.TryRemove(fileId, out var fileData)) + { + Interlocked.Add(ref _totalSizeBytes, -fileData.Length); + return Task.FromResult(true); + } + + return Task.FromResult(false); } public bool CanProcess(string fileId) => fileId.StartsWith(StorageType); @@ -100,35 +119,14 @@ public Task DeleteAsync(string fileId, CancellationToken cancellationToken { if (!_storage.ContainsKey(fileId)) return null; - //parse: "storageType://basket/tenant/filename" - ReadOnlySpan span = fileId.AsSpan(); - int schemeEnd = span.IndexOf(SchemeSeparator.AsSpan()); - if (schemeEnd == -1) - return null; - - var pathPart = span[(schemeEnd + SchemeSeparator.Length)..]; - - // "tenantId/filename" - int slashIndex = pathPart.IndexOf('/'); - if (slashIndex == -1) - return null; - - var scopeSpan = pathPart[..slashIndex]; - - var nextSpan = pathPart[(slashIndex + 1)..]; - slashIndex = nextSpan.IndexOf('/'); - - var tenantSpan = nextSpan[..slashIndex]; - var fileNameSpan = nextSpan[(slashIndex + 1)..]; - - if (!int.TryParse(tenantSpan, NumberStyles.None, CultureInfo.InvariantCulture, out int tenantId)) + if (!FileIdParser.TryParse(fileId, out var basket, out var tenantId, out _, out var fileName)) return null; var metadata = new FileMetadata { StorageType = StorageType, - Basket = scopeSpan.ToString(), - FileName = fileNameSpan.ToString(), + Basket = basket, + FileName = fileName, TenantId = tenantId }; diff --git a/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs b/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs index bf096fc3..0dab546c 100644 --- a/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs +++ b/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs @@ -3,6 +3,10 @@ /// /// Configuration options for the in-memory file storage provider. /// -/// The default basket (container) name. Defaults to "share". -/// true if the storage should reject write operations; otherwise, false. -public sealed record InMemoryFileStorageOptions(string Basket = "share", bool IsReadOnly = false); +/// The default basket (container) name. Defaults to "share". +/// true if the storage should reject write operations; otherwise, false. +/// Maximum total size in bytes for all stored files. Default is 1 GB (1_073_741_824). Set to zero or negative to disable the limit. +public sealed record InMemoryFileStorageOptions( + string Basket = "share", + bool IsReadOnly = false, + long MaxSizeBytes = 1_073_741_824); diff --git a/src/Sa.HybridFileStorage/Interceptors/DeleteLoggingInterceptor.cs b/src/Sa.HybridFileStorage/Interceptors/DeleteLoggingInterceptor.cs new file mode 100644 index 00000000..c5fe08fa --- /dev/null +++ b/src/Sa.HybridFileStorage/Interceptors/DeleteLoggingInterceptor.cs @@ -0,0 +1,71 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Sa.HybridFileStorage.Domain; + +namespace Sa.HybridFileStorage.Interceptors; + +/// +/// Logs delete-related operations across file storage providers. +/// +internal sealed partial class DeleteLoggingInterceptor(ILogger? logger = null) : IDeleteInterceptor +{ + private readonly ILogger _logger = logger ?? NullLogger.Instance; + + public ValueTask CanDeleteAsync(IFileStorage storage, string fileId, CancellationToken cancellationToken) + { + LogCanDelete(_logger, fileId, storage.StorageType); + return ValueTask.FromResult(true); + } + + public ValueTask AfterDeleteAsync( + IFileStorage storage, + string fileId, + bool success, + CancellationToken cancellationToken) + { + if (success) + { + LogDeleteSuccess(_logger, fileId, storage.StorageType); + } + else + { + LogDeleteFailure(_logger, fileId, storage.StorageType); + } + + return ValueTask.CompletedTask; + } + + public ValueTask OnDeleteErrorAsync( + IFileStorage storage, + string fileId, + Exception exception, + CancellationToken cancellationToken) + { + LogDeleteError(_logger, exception, fileId, storage.StorageType); + return ValueTask.CompletedTask; + } + + [LoggerMessage( + EventId = 2201, + Level = LogLevel.Trace, + Message = "Checking if can delete file with ID: `{FileId}` from storage: {Storage}")] + static partial void LogCanDelete(ILogger logger, string fileId, string storage); + + [LoggerMessage( + EventId = 2202, + Level = LogLevel.Information, + Message = "Successfully deleted file with ID: `{FileId}` from storage: {Storage}")] + static partial void LogDeleteSuccess(ILogger logger, string fileId, string storage); + + [LoggerMessage( + EventId = 2203, + Level = LogLevel.Warning, + Message = "Failed to delete file with ID: `{FileId}` from storage: {Storage}")] + static partial void LogDeleteFailure(ILogger logger, string fileId, string storage); + + [LoggerMessage( + EventId = 2204, + Level = LogLevel.Error, + Message = "Error occurred while deleting file with ID: `{FileId}` from storage: {Storage}")] + static partial void LogDeleteError(ILogger logger, Exception ex, string fileId, string storage); +} diff --git a/src/Sa.HybridFileStorage/Interceptors/DownloadLoggingInterceptor.cs b/src/Sa.HybridFileStorage/Interceptors/DownloadLoggingInterceptor.cs new file mode 100644 index 00000000..5a8f5b27 --- /dev/null +++ b/src/Sa.HybridFileStorage/Interceptors/DownloadLoggingInterceptor.cs @@ -0,0 +1,75 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Sa.HybridFileStorage.Domain; + +namespace Sa.HybridFileStorage.Interceptors; + +/// +/// Logs download-related operations across file storage providers. +/// +internal sealed partial class DownloadLoggingInterceptor(ILogger? logger = null) : IDownloadInterceptor +{ + private readonly ILogger _logger = logger ?? NullLogger.Instance; + + public ValueTask CanDownloadAsync( + IFileStorage storage, + string fileId, + Func loadStream, + CancellationToken cancellationToken) + { + LogCanDownload(_logger, fileId, storage.StorageType); + return ValueTask.FromResult(true); + } + + public ValueTask AfterDownloadAsync( + IFileStorage storage, + string fileId, + bool success, + CancellationToken cancellationToken) + { + if (success) + { + LogDownloadSuccess(_logger, fileId, storage.StorageType); + } + else + { + LogDownloadFailure(_logger, fileId, storage.StorageType); + } + + return ValueTask.CompletedTask; + } + + public ValueTask OnDownloadErrorAsync( + IFileStorage storage, + string fileId, + Exception exception, + CancellationToken cancellationToken) + { + LogDownloadError(_logger, exception, fileId, storage.StorageType); + return ValueTask.CompletedTask; + } + + [LoggerMessage( + EventId = 2301, + Level = LogLevel.Trace, + Message = "Checking if can download file with ID: `{FileId}` from storage: {Storage}")] + static partial void LogCanDownload(ILogger logger, string fileId, string storage); + + [LoggerMessage( + EventId = 2302, + Level = LogLevel.Information, + Message = "Successfully downloaded file with ID: `{FileId}` from storage: {Storage}")] + static partial void LogDownloadSuccess(ILogger logger, string fileId, string storage); + + [LoggerMessage( + EventId = 2303, + Level = LogLevel.Warning, + Message = "Failed to download file with ID: `{FileId}` from storage: {Storage}")] + static partial void LogDownloadFailure(ILogger logger, string fileId, string storage); + + [LoggerMessage( + EventId = 2304, + Level = LogLevel.Error, + Message = "Error occurred while downloading file with ID: `{FileId}` from storage: {Storage}")] + static partial void LogDownloadError(ILogger logger, Exception ex, string fileId, string storage); +} diff --git a/src/Sa.HybridFileStorage/Interceptors/LoggingInterceptor.cs b/src/Sa.HybridFileStorage/Interceptors/LoggingInterceptor.cs deleted file mode 100644 index 0e02fa8e..00000000 --- a/src/Sa.HybridFileStorage/Interceptors/LoggingInterceptor.cs +++ /dev/null @@ -1,176 +0,0 @@ -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Sa.HybridFileStorage.Domain; - -namespace Sa.HybridFileStorage.Interceptors; - -internal sealed partial class LoggingInterceptor(ILogger? logger = null) - : IDeleteInterceptor, IDownloadInterceptor, IUploadInterceptor -{ - private readonly ILogger _logger = logger ?? NullLogger.Instance; - - public ValueTask CanDeleteAsync(IFileStorage storage, string fileId, CancellationToken cancellationToken) - { - LogCanDelete(_logger, fileId, storage.StorageType); - return ValueTask.FromResult(true); - } - - public ValueTask AfterDeleteAsync( - IFileStorage storage, - string fileId, - bool success, - CancellationToken cancellationToken) - { - if (success) - { - LogDeleteSuccess(_logger, fileId, storage.StorageType); - } - else - { - LogDeleteFailure(_logger, fileId, storage.StorageType); - } - return ValueTask.CompletedTask; - } - - public ValueTask OnDeleteErrorAsync( - IFileStorage storage, - string fileId, - Exception exception, - CancellationToken cancellationToken) - { - LogDeleteError(_logger, exception, fileId, storage.StorageType); - return ValueTask.CompletedTask; - } - - public ValueTask CanDownloadAsync( - IFileStorage storage, - string fileId, - Func loadStream, - CancellationToken cancellationToken) - { - LogCanDownload(_logger, fileId, storage.StorageType); - return ValueTask.FromResult(true); - } - - public ValueTask AfterDownloadAsync( - IFileStorage storage, - string fileId, - bool success, - CancellationToken cancellationToken) - { - if (success) - { - LogDownloadSuccess(_logger, fileId, storage.StorageType); - } - else - { - LogDownloadFailure(_logger, fileId, storage.StorageType); - } - return ValueTask.CompletedTask; - } - - public ValueTask OnDownloadErrorAsync( - IFileStorage storage, - string fileId, - Exception exception, - CancellationToken cancellationToken) - { - LogDownloadError(_logger, exception, fileId, storage.StorageType); - return ValueTask.CompletedTask; - } - - public ValueTask CanUploadAsync( - IFileStorage storage, - UploadFileInput input, - Stream fileStream, - CancellationToken cancellationToken) - { - LogCanUpload(_logger, input, storage.StorageType); - return ValueTask.FromResult(true); - } - - public ValueTask AfterUploadAsync( - IFileStorage storage, - StorageResult result, - CancellationToken cancellationToken) - { - LogUploadSuccess(_logger, storage.StorageType, result); - return ValueTask.CompletedTask; - } - - public ValueTask OnUploadErrorAsync( - IFileStorage storage, - Exception exception, - CancellationToken cancellationToken) - { - LogUploadError(_logger, exception, storage.StorageType); - return ValueTask.CompletedTask; - } - - - [LoggerMessage( - EventId = 2201, - Level = LogLevel.Trace, - Message = "Checking if can delete file with ID: `{FileId}` from storage: {Storage}")] - static partial void LogCanDelete(ILogger logger, string fileId, string storage); - - [LoggerMessage( - EventId = 2202, - Level = LogLevel.Information, - Message = "Successfully deleted file with ID: `{FileId}` from storage: {Storage}")] - static partial void LogDeleteSuccess(ILogger logger, string fileId, string storage); - - [LoggerMessage( - EventId = 2203, - Level = LogLevel.Warning, - Message = "Failed to delete file with ID: `{FileId}` from storage: {Storage}")] - static partial void LogDeleteFailure(ILogger logger, string fileId, string storage); - - [LoggerMessage( - EventId = 2204, - Level = LogLevel.Error, - Message = "Error occurred while deleting file with ID: `{FileId}` from storage: {Storage}")] - static partial void LogDeleteError(ILogger logger, Exception ex, string fileId, string storage); - - [LoggerMessage( - EventId = 2301, - Level = LogLevel.Trace, - Message = "Checking if can download file with ID: `{FileId}` from storage: {Storage}")] - static partial void LogCanDownload(ILogger logger, string fileId, string storage); - - [LoggerMessage( - EventId = 2302, - Level = LogLevel.Information, - Message = "Successfully downloaded file with ID: `{FileId}` from storage: {Storage}")] - static partial void LogDownloadSuccess(ILogger logger, string fileId, string storage); - - [LoggerMessage( - EventId = 2303, - Level = LogLevel.Warning, - Message = "Failed to download file with ID: `{FileId}` from storage: {Storage}")] - static partial void LogDownloadFailure(ILogger logger, string fileId, string storage); - - [LoggerMessage( - EventId = 2304, - Level = LogLevel.Error, - Message = "Error occurred while downloading file with ID: `{FileId}` from storage: {Storage}")] - static partial void LogDownloadError(ILogger logger, Exception ex, string fileId, string storage); - - [LoggerMessage( - EventId = 2401, - Level = LogLevel.Trace, - Message = "Checking if can upload file: {Input} to storage: {Storage}")] - static partial void LogCanUpload(ILogger logger, UploadFileInput input, string storage); - - [LoggerMessage( - EventId = 2402, - Level = LogLevel.Information, - Message = "Successfully uploaded file to storage: {Storage} with result: {Result}")] - static partial void LogUploadSuccess(ILogger logger, string storage, StorageResult result); - - [LoggerMessage( - EventId = 2403, - Level = LogLevel.Error, - Message = "Error occurred while uploading file to storage: {Storage}")] - static partial void LogUploadError(ILogger logger, Exception ex, string storage); -} diff --git a/src/Sa.HybridFileStorage/Interceptors/Setup.cs b/src/Sa.HybridFileStorage/Interceptors/Setup.cs index 6ba8ea7d..41f9d8fe 100644 --- a/src/Sa.HybridFileStorage/Interceptors/Setup.cs +++ b/src/Sa.HybridFileStorage/Interceptors/Setup.cs @@ -2,16 +2,24 @@ internal static class Setup { - internal static IInterceptorContainer AddLoggingInterceptor( + /// + /// Adds all logging interceptors (upload, download, delete) to the container. + /// + internal static IInterceptorContainer AddLoggingInterceptors( this IInterceptorContainer container, - LoggingInterceptor? loggingInterceptor = null) + UploadLoggingInterceptor? uploadInterceptor = null, + DownloadLoggingInterceptor? downloadInterceptor = null, + DeleteLoggingInterceptor? deleteInterceptor = null) { - if (loggingInterceptor != null) - { - container.AddDeleteInterceptor(loggingInterceptor); - container.AddDownloadInterceptor(loggingInterceptor); - container.AddUploadInterceptor(loggingInterceptor); - } + if (uploadInterceptor != null) + container.AddUploadInterceptor(uploadInterceptor); + + if (downloadInterceptor != null) + container.AddDownloadInterceptor(downloadInterceptor); + + if (deleteInterceptor != null) + container.AddDeleteInterceptor(deleteInterceptor); + return container; } } diff --git a/src/Sa.HybridFileStorage/Interceptors/UploadLoggingInterceptor.cs b/src/Sa.HybridFileStorage/Interceptors/UploadLoggingInterceptor.cs new file mode 100644 index 00000000..bc0bc4bf --- /dev/null +++ b/src/Sa.HybridFileStorage/Interceptors/UploadLoggingInterceptor.cs @@ -0,0 +1,59 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Sa.HybridFileStorage.Domain; + +namespace Sa.HybridFileStorage.Interceptors; + +/// +/// Logs upload-related operations across file storage providers. +/// +internal sealed partial class UploadLoggingInterceptor(ILogger? logger = null) : IUploadInterceptor +{ + private readonly ILogger _logger = logger ?? NullLogger.Instance; + + public ValueTask CanUploadAsync( + IFileStorage storage, + UploadFileInput input, + Stream fileStream, + CancellationToken cancellationToken) + { + LogCanUpload(_logger, input, storage.StorageType); + return ValueTask.FromResult(true); + } + + public ValueTask AfterUploadAsync( + IFileStorage storage, + StorageResult result, + CancellationToken cancellationToken) + { + LogUploadSuccess(_logger, storage.StorageType, result); + return ValueTask.CompletedTask; + } + + public ValueTask OnUploadErrorAsync( + IFileStorage storage, + Exception exception, + CancellationToken cancellationToken) + { + LogUploadError(_logger, exception, storage.StorageType); + return ValueTask.CompletedTask; + } + + [LoggerMessage( + EventId = 2401, + Level = LogLevel.Trace, + Message = "Checking if can upload file: {Input} to storage: {Storage}")] + static partial void LogCanUpload(ILogger logger, UploadFileInput input, string storage); + + [LoggerMessage( + EventId = 2402, + Level = LogLevel.Information, + Message = "Successfully uploaded file to storage: {Storage} with result: {Result}")] + static partial void LogUploadSuccess(ILogger logger, string storage, StorageResult result); + + [LoggerMessage( + EventId = 2403, + Level = LogLevel.Error, + Message = "Error occurred while uploading file to storage: {Storage}")] + static partial void LogUploadError(ILogger logger, Exception ex, string storage); +} diff --git a/src/Sa.HybridFileStorage/Readme-ru.md b/src/Sa.HybridFileStorage/Readme-ru.md index 5fc77428..f08a71ff 100644 --- a/src/Sa.HybridFileStorage/Readme-ru.md +++ b/src/Sa.HybridFileStorage/Readme-ru.md @@ -4,26 +4,51 @@ --- +## Содержание + +- [Поддерживаемые провайдеры](#поддерживаемые-провайдеры) +- [Ключевые возможности](#ключевые-возможности) +- [Формат File ID](#формат-file-id) +- [Быстрый старт](#быстрый-старт) + - [Без DI](#без-di) + - [С DI (Generic Host)](#с-di-generic-host) +- [Примеры CRUD](#примеры-crud) + - [Загрузка](#загрузка) + - [Скачивание](#скачивание) + - [Удаление](#удаление) + - [Получение метаданных](#получение-метаданных) +- [Копирование между корзинами](#копирование-между-корзинами) +- [Пакетные операции](#пакетные-операции) +- [Перехватчики (Interceptors)](#перехватчики-interceptors) +- [Режим «только чтение»](#режим-только-чтение) +- [Справочник настроек](#справочник-настроек) +- [Доменные типы](#доменные-типы) +- [Исключения](#исключения) +- [Структура проекта](#структура-проекта) + +--- + ## Поддерживаемые провайдеры -| Провайдер | Класс | Сценарий использования | -|-----------|-------|----------------------| -| **Файловая система** | `FileSystemStorage` | Локальная разработка, on-premise развёртывания | -| **S3-совместимое** | `S3FileStorage` | Облачное хранилище (AWS S3, MinIO и др.) | -| **PostgreSQL** | `PostgresFileStorage` | Файлы внутри БД, транзакционная согласованность | -| **In-Memory** | `InMemoryFileStorage` | Тестирование, эфемерные сценарии | +| Провайдер | Пакет | Класс | Сценарий использования | +|-----------|-------|-------|----------------------| +| **In-Memory** | `Sa.HybridFileStorage` | `InMemoryFileStorage` | Тестирование, эфемерные сценарии | +| **Файловая система** | `Sa.HybridFileStorage.FileSystem` | `FileSystemStorage` | Локальная разработка, on-premise развёртывания | +| **S3-совместимое** | `Sa.HybridFileStorage.S3` | `S3FileStorage` | Облачное хранилище (AWS S3, MinIO и др.) | +| **PostgreSQL** | `Sa.HybridFileStorage.Postgres` | `PostgresFileStorage` | Файлы внутри БД, транзакционная согласованность, партиционирование | --- ## Ключевые возможности -- ✅ **Единый API** — Один интерфейс для всех провайдеров хранения -- ✅ **Изоляция Basket/Tenant** — Многопользовательская поддержка со scoped корзинами -- ✅ **Режим «только чтение»** — Защита от случайных модификаций -- ✅ **Потоковая передача** — Эффективная работа с памятью при передаче файлов +- ✅ **Единый API** — Один интерфейс `IHybridFileStorage` для всех провайдеров +- ✅ **Изоляция Basket/Tenant** — Многопользовательская поддержка со scoped контейнерами +- ✅ **Failover** — Автоматическое переключение провайдера при отказе бэкенда +- ✅ **Потоковая передача** — Эффективная работа с памятью через `Stream` - ✅ **Native AOT готово** — Полная совместимость с .NET 10 Native AOT -- ✅ **Пакетные операции** — Эффективная массовая обработка файлов с параллелизмом +- ✅ **Пакетные операции** — Массовая обработка файлов с настраиваемым параллелизмом - ✅ **Перехватчики (Interceptors)** — Хуки жизненного цикла загрузки/скачивания/удаления +- ✅ **Режим «только чтение»** — Защита от случайных модификаций --- @@ -32,14 +57,31 @@ Все файлы идентифицируются через унифицированный URI-подобный формат: ``` -{storageType}://{basket}/{tenantId}/{fileName} +{storageType}://{basket}/{tenantId}/{path} ``` -**Примеры:** -- `s3://share/42/document.pdf` -- `fs://root/100/report.xlsx` -- `pg://files/7/1773210911/some/data.bin` -- `mem://share/42/temp.txt` +Каждый провайдер добавляет свою глубину пути: + +| Провайдер | Пример File ID | Структура пути | +|-----------|---------------|----------------| +| **In-Memory** | `mem://share/42/document.pdf` | `{basket}/{tenantId}/{fileName}` | +| **Файловая система** | `fs://documents/100/report.xlsx` | `{basket}/{tenantId}/{fileName}` | +| **S3** | `s3://uploads/7/invoice.csv` | `{basket}/{tenantId}/{fileName}` | +| **PostgreSQL** | `pg://files/12/1751347200/photo.jpg` | `{basket}/{tenantId}/{unixTimestamp}/{fileName}` | + +> **Примечание:** PostgreSQL включает Unix-таймстамп в пути, потому что партиционирует по дате. Другие провайды таймстамп не включают. + +### Парсинг File ID + +Используйте статический утилитный класс `FileIdParser`: + +```csharp +if (FileIdParser.TryParse("pg://files/42/1751347200/report.pdf", out var basket, out var tenantId, out var timestamp, out var fileName)) +{ + Console.WriteLine($"Basket={basket}, Tenant={tenantId}, TS={timestamp}, Name={fileName}"); + // Basket=files, Tenant=42, TS=1751347200, Name=report.pdf +} +``` --- @@ -48,38 +90,99 @@ ### Без DI ```csharp +using Sa.HybridFileStorage; +using Sa.HybridFileStorage.Domain; + +// 1. Создаём провайдер in-memory using var memory = new InMemoryFileStorage(new InMemoryFileStorageOptions("share")); + +// 2. Собираем гибридный контейнер var container = new HybridFileStorageContainer([memory]); var storage = new HybridFileStorage(container, InterceptorContainer.Empty); -var stream = "Hello, HybridFileStorage!".ToStream(); +// 3. Загружаем файл +var stream = "Hello, HybridFileStorage!".ToUtf8Stream(); var result = await storage.UploadAsync( - "share", - new UploadFileInput { FileName = "file.txt", TenantId = 42 }, - stream, - ct); + basket: "share", + input: new UploadFileInput { FileName = "hello.txt", TenantId = 42 }, + fileStream: stream, + cancellationToken: ct); -await storage.DownloadAsync(result.FileId, async (fs, t) => +Console.WriteLine(result.FileId); // mem://share/42/hello.txt + +// 4. Скачиваем и обрабатываем +bool wasFound = await storage.DownloadAsync(result.FileId, async (stream, token) => { - var content = await fs.ToStrAsync(t); - Console.WriteLine(content); -}); + using var reader = new StreamReader(stream, Encoding.UTF8); + var content = await reader.ReadToEndAsync(token); + Console.WriteLine(content); // Hello, HybridFileStorage! +}, ct); + +// 5. Удаляем +bool deleted = await storage.DeleteAsync(result.FileId, ct); ``` -### С DI +### С DI (Generic Host) ```csharp -builder.Services.AddSaHybridFileStorage(configure => configure - .AddStorage(InMemoryFileStorage.New("share")) +using Microsoft.Extensions.Hosting; +using Sa.HybridFileStorage; +using Sa.HybridFileStorage.FileSystem; +using Sa.HybridFileStorage.S3; + +var builder = Host.CreateApplicationBuilder(args); + +// Регистрируем все провайдеры через fluent builder +builder.Services.AddSaHybridFileStorage(cfg => cfg + // In-Memory провайдер + .ConfigureStorage((sp, c) => c.AddStorage(new InMemoryFileStorage())) + + // Файловая система + .ConfigureStorage((sp, c) => c.AddStorage(new FileSystemStorage( + new FileSystemStorageSettings + { + BasePath = @"C:\data\files", + Basket = "documents" + }))) + + // S3 провайдер + .ConfigureStorage((sp, c) => c.AddStorage( + new S3FileStorage( + sp.GetRequiredService(), + new S3FileStorageOptions + { + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123", + Bucket = "mybucket", + Basket = "uploads" + }))) + + // Включаем встроенные логгирующие перехватчики .AddLogging()); -// Или регистрация отдельных провайдеров: +var host = builder.Build(); +var storage = host.Services.GetRequiredService(); + +// Используйте везде — внедряется через DI в ваши сервисы +``` + +#### Минимальная регистрация DI + +Для быстрых настроек каждый провайдер имеет собственный метод расширения: + +```csharp +// Только In-Memory +builder.Services.AddSaInMemoryFileStorage(); + +// Только файловая система builder.Services.AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = @"C:\data\files", Basket = "documents" }); +// Только S3 builder.Services.AddSaS3FileStorage(new S3FileStorageOptions { Endpoint = "http://localhost:9000", @@ -89,96 +192,154 @@ builder.Services.AddSaS3FileStorage(new S3FileStorageOptions Basket = "uploads" }); -// Использование: -var storage = serviceProvider.GetRequiredService(); +// Затем регистрируем гибридный слой +builder.Services.AddSaHybridFileStorage(cfg => cfg.AddLogging()); ``` --- -## Настройки +## Примеры CRUD -### FileSystemStorageSettings +### Загрузка -| Свойство | Описание | По умолчанию | -|----------|----------|-------------| -| `BasePath` | Корневая директория для файлов | *(обязательно)* | -| `Basket` | Имя области хранения | `"share"` | -| `StorageType` | Префикс схемы в File ID | `"fs"` | -| `IsReadOnly` | Запрет записи | `false` | -| `BufferSize` | Размер буфера чтения/записи | `256 КБ` | +Upload принимает имя корзины (контейнера), метаданные и `Stream`. Гибридный слой находит доступный провайдер, соответствующий корзине, и загружает файл. -### S3FileStorageOptions +```csharp +// Загрузка из Stream +using var stream = File.OpenRead(@"C:\temp\document.pdf"); +var result = await storage.UploadAsync( + basket: "documents", + input: new UploadFileInput { FileName = "document.pdf", TenantId = 42 }, + fileStream: stream, + cancellationToken: ct); -| Свойство | Описание | По умолчанию | -|----------|----------|-------------| -| `Endpoint` | URL S3-эндпоинта | *(обязательно)* | -| `AccessKey` | Ключ доступа S3 | *(обязательно)* | -| `SecretKey` | Секретный ключ S3 | *(обязательно)* | -| `Bucket` | Имя бакета | *(обязательно)* | -| `Basket` | Имя области хранения | `"share"` | -| `Region` | Регион для SigV4 | `"eu-central-1"` | -| `IsReadOnly` | Запрет записи | `false` | +Console.WriteLine($"Загружено: {result.FileId}"); +// Вывод: fs://documents/42/document.pdf +``` -### PostgresFileStorageOptions +Копируем локальный файл напрямую: -| Свойство | Описание | -|----------|----------| -| `SchemaName` | Схема PostgreSQL | -| `TableName` | Имя таблицы для данных файлов | -| `PartOptions.PgPartBy` | Стратегия партиционирования (day/month/year/list/range) | -| `CleanupOptions.ExpireDays` | Порог автоочистки | -| `StorageOptions.IsReadOnly` | Запрет записи | +```csharp +var result = await storage.CopyFromFileAsync( + filePath: @"C:\temp\image.png", + basket: "images", + input: new UploadFileInput { FileName = "avatar.png", TenantId = 7 }, + ct: ct); +``` -### InMemoryFileStorageOptions +### Скачивание -| Свойство | Описание | По умолчанию | -|----------|----------|-------------| -| `Basket` | Имя области хранения | `"share"` | -| `IsReadOnly` | Запрет записи | `false` | +Download делегирует поток файла колбэку `Func`. Это избегает загрузки всего файла в память. + +```csharp +// Обработка потока inline +bool found = await storage.DownloadAsync(result.FileId, async (stream, token) => +{ + using var reader = new StreamReader(stream, Encoding.UTF8); + string content = await reader.ReadToEndAsync(token); + Console.WriteLine(content); +}, ct); + +// Копирование в другой поток +using var destination = new FileStream(@"C:\output\copy.pdf", FileMode.Create); +await storage.DownloadAsync(result.FileId, async (source, token) => + await source.CopyToAsync(destination, 81920, token), + ct); +``` + +### Удаление + +```csharp +bool deleted = await storage.DeleteAsync(result.FileId, ct); +if (deleted) + Console.WriteLine("Файл удалён."); +else + Console.WriteLine("Файл не найден."); +``` + +### Получение метаданных + +```csharp +var metadata = await storage.GetMetadataAsync(result.FileId, ct); +if (metadata != null) +{ + Console.WriteLine($"Корзина: {metadata.Basket}"); + Console.WriteLine($"Тенант: {metadata.TenantId}"); + Console.WriteLine($"Имя: {metadata.FileName}"); + Console.WriteLine($"Тип: {metadata.StorageType}"); +} +``` --- -## Пакетные операции +## Копирование между корзинами -`HybridFileStorageExtensions` предоставляет высокоуровневые методы для массовой обработки файлов с встроенным параллелизмом, обработкой ошибок и отчётами о прогрессе. +Перемещайте или дублируйте файлы между корзинами (даже между разными провайдерами): ```csharp -// Копирование из локальной файловой системы -var result = await storage.CopyFromFileAsync( - @"C:\temp\document.pdf", - "archive", - new UploadFileInput { FileName = "archived.pdf", TenantId = 42 }); +// Копирование в пределах одной корзины +var copied = await storage.CopyToBasketAsync( + fileId: "fs://documents/42/report.pdf", + basket: "archive", + ct: ct); -// Копирование между корзинами/областями -var moved = await storage.CopyToBasketAsync( - "s3://share/42/doc.pdf", - "backup"); +// Кастомизация метаданных при копировании +var renamed = await storage.CopyToBasketAsync( + fileId: "fs://documents/42/report.pdf", + basket: "backup", + configure: meta => new UploadFileInput + { + TenantId = meta.TenantId, + FileName = $"renamed-{meta.FileName}" // меняем имя + }, + ct: ct); +``` -// Пакетное копирование с параллелизмом и прогрессом +--- + +## Пакетные операции + +Массовые параллельные операции с файлами, отчётами о прогрессе и обработкой ошибок: + +```csharp +// Пакетное копирование с параллелизмом var batchResult = await storage.CopyToScopeBatchAsync( - fileIds: ["s3://share/1/a.txt", "s3://share/2/b.txt"], + fileIds: + [ + "fs://documents/1/a.pdf", + "fs://documents/2/b.pdf", + "s3://uploads/3/c.pdf", + ], basket: "archive", options: new BatchOptions { MaxDegreeOfParallelism = 8, ContinueOnError = true, - Progress = new Progress() - }); + OperationTimeout = TimeSpan.FromSeconds(30), + Progress = new Progress(p => + { + Console.WriteLine($"{p.Completed}/{p.Total} — OK:{p.SuccessCount} Fail:{p.FailureCount}"); + }) + }, + ct: ct); foreach (var ok in batchResult.Succeeded) Console.WriteLine($"Скопировано: {ok.FileId}"); foreach (var err in batchResult.Failed) Console.WriteLine($"Ошибка #{err.Index}: {err.FileId} — {err.Exception.Message}"); + +// Или выбросить исключение при любой ошибке +batchResult.ThrowIfHasErrors(); // выбрасывает BatchOperationException ``` -### BatchResult +### BatchResult<T> | Член | Тип | Описание | |------|-----|----------| | `Succeeded` | `IReadOnlyList` | Успешные результаты | | `Failed` | `IReadOnlyList` | Ошибки с File ID и исключением | -| `Total` | `int` | Всего обработанных элементов | +| `Total` | `int` | Всего обработано элементов | | `HasErrors` | `bool` | Были ли ошибки | | `ThrowIfHasErrors()` | `void` | Выбрасывает `BatchOperationException` при наличии ошибок | @@ -187,81 +348,157 @@ foreach (var err in batchResult.Failed) | Свойство | Описание | По умолчанию | |----------|----------|-------------| | `MaxDegreeOfParallelism` | Одновременные операции | `4` | -| `ContinueOnError` | Продолжать после ошибок | `true` | -| `OperationTimeout` | Таймаут на операцию | `0` (бесконечность) | -| `Progress` | Отчётчик прогресса | `null` | +| `ContinueOnError` | Продолжать после отдельных ошибок | `true` | +| `OperationTimeout` | Таймаут на операцию (`0` = бесконечно) | `0` | +| `Progress` | Отчётчик `IProgress` | `null` | --- ## Перехватчики (Interceptors) -Хуки жизненного цикла для операций загрузки/скачивания/удаления. +Хуки жизненного цикла для операций загрузки/скачивания/удаления. Реализуйте один из трёх интерфейсов: ```csharp public interface IUploadInterceptor { + // Верните false для отклонения загрузки ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken ct); ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken ct); ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken ct); } -public interface IDownloadInterceptor { /* аналогично Can/After/Error */ } -public interface IDeleteInterceptor { /* аналогично Can/After/Error */ } +public interface IDownloadInterceptor { /* CanDownloadAsync / AfterDownloadAsync / OnDownloadErrorAsync */ } +public interface IDeleteInterceptor { /* CanDeleteAsync / AfterDeleteAsync / OnDeleteErrorAsync */ } ``` -Регистрация перехватчиков через fluent builder: +Пример — блокировка загрузки конкретных расширений: ```csharp -services.AddSaHybridFileStorage(cfg => cfg.ConfigureInterceptors((sp, container) => +public class DeniedExtensionInterceptor : IUploadInterceptor { - container.AddUploadInterceptor(myCustomInterceptor); - container.AddDownloadInterceptor(loggingInterceptor); -})); + private static readonly HashSet DeniedExtensions = ["exe", "bat", "cmd"]; + + public ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken ct) + { + var ext = Path.GetExtension(input.FileName)?.TrimStart('.').ToLowerInvariant(); + return ValueTask.FromResult(!DeniedExtensions.Contains(ext)); + } + + public ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken ct) + => ValueTask.CompletedTask; + + public ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken ct) + => ValueTask.CompletedTask; +} +``` + +Регистрация перехватчиков через fluent builder: + +```csharp +builder.Services.AddSaHybridFileStorage(cfg => cfg + .ConfigureInterceptors((sp, container) => + { + container.AddUploadInterceptor(new DeniedExtensionInterceptor()); + container.AddDownloadInterceptor(new LoggingDownloadInterceptor()); + })); ``` -Встроенный `LoggingInterceptor` доступен через `.AddLogging()`. +Встроенные логгирующие перехватчики доступны через `.AddLogging()`. --- ## Режим «только чтение» -Установите `IsReadOnly = true` для любого провайдера хранилища, чтобы запретить запись. Попытки записи вызывают `HybridFileStorageWritableException`: +Установите `IsReadOnly = true` для любого провайдера, чтобы запретить запись. Попытки записи вызывают `HybridFileStorageWritableException`: ```csharp -builder.Services.AddSaFileSystemFileStorage(settings => +builder.Services.AddSaFileSystemFileStorage(new FileSystemStorageSettings { - settings.BasePath = @"C:\readonly\data"; - settings.IsReadOnly = true; + BasePath = @"C:\readonly\data", + IsReadOnly = true // загрузки/удаления будут завершаться ошибкой }); ``` --- +## Справочник настроек + +### FileSystemStorageSettings + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `BasePath` | Корневая директория для файлов | *(обязательно)* | +| `Basket` | Имя контейнера (scopes) | `"share"` | +| `StorageType` | Префикс схемы в File ID | `"fs"` | +| `IsReadOnly` | Запрет записи | `false` | + +### S3FileStorageOptions + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `Endpoint` | URL S3-эндпоинта | *(обязательно)* | +| `AccessKey` | Ключ доступа S3 | *(обязательно)* | +| `SecretKey` | Секретный ключ S3 | *(обязательно)* | +| `Bucket` | Имя бакета | *(обязательно)* | +| `Basket` | Имя контейнера | `"share"` | +| `Region` | Регион для SigV4 подписи | `"eu-central-1"` | +| `StorageType` | Префикс схемы в File ID | `"s3"` | +| `IsReadOnly` | Запрет записи | `false` | + +### PostgresFileStorageOptions + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `StorageOptions.SchemaName` | Схема PostgreSQL | `"public"` | +| `StorageOptions.TableName` | Таблица для данных файлов | `"files"` | +| `StorageOptions.StorageType` | Префикс схемы в File ID | `"pg"` | +| `PartOptions.Basket` | Имя контейнера | `"share"` | +| `PartOptions.PgPartBy` | Гранулярность партиционирования | `PgPartBy.Day` | +| `PartOptions.MigrationScheduleForwardDays` | Дней заранее для предсоздания партиций | `2` | +| `CleanupOptions.ExpireDays` | Порог автоочистки (дней) | `365 * 3` | +| `StorageOptions.IsReadOnly` | Запрет записи | `false` | + +### InMemoryFileStorageOptions + +| Свойство | Описание | По умолчанию | +|----------|----------|-------------| +| `Basket` | Имя контейнера | `"share"` | +| `MaxSizeBytes` | Лимит в байтах (`0` = без лимита) | `0` | +| `IsReadOnly` | Запрет записи | `false` | + +--- + ## Доменные типы ### StorageResult +Результат операции загрузки. Содержит канонический File ID и публичный URL. + ```csharp public sealed record StorageResult( - string FileId, - string AbsoluteUrl, - string StorageType, + string FileId, // напр. "fs://documents/42/report.pdf" + string AbsoluteUrl, // напр. "C:\data\files\documents\42\report.pdf" + string StorageType, // напр. "fs", "s3", "pg", "mem" DateTimeOffset UploadedAt); ``` ### UploadFileInput +Входные метаданные для загрузки. + ```csharp public sealed record UploadFileInput { - public int TenantId { get; init; } - public string FileName { get; init; } - public static UploadFileInput Empty { get; } + public int TenantId { get; init; } // по умолчанию 0 + public string FileName { get; init; } = ""; // обязательно при валидации + public static UploadFileInput Empty { get; } // предварительно созданный пустой экземпляр } ``` ### FileMetadata +Неизменяемые метаданные, полученные через `GetMetadataAsync`. + ```csharp public sealed class FileMetadata { @@ -278,30 +515,39 @@ public sealed class FileMetadata | Исключение | Когда выбрасывается | |------------|-------------------| -| `HybridFileStorageNoAvailableException` | Не найдено хранилище для запрошенной корзины | +| `HybridFileStorageNoAvailableException` | Не найден провайдер для запрошенной корзины, либо все провайдеры завершились ошибкой | | `HybridFileStorageWritableException` | Попытка записи в хранилище «только чтение» | -| `HybridFileStorageAggregateException` | Несколько ошибок провайдеров агрегированы | -| `BatchOperationException` | Пакет с ошибками и `ContinueOnError = false` | +| `HybridFileStorageAggregateException` | Несколько ошибок провайдеров агрегированы при failover | +| `BatchOperationException` | Пакетная операция имела ошибки и `ContinueOnError = false` | --- ## Структура проекта ``` -src/Sa.HybridFileStorage/ -├── IHybridFileStorage.cs # Главный интерфейс -├── HybridFileStorage.cs # Реализация с failover -├── HybridFileStorageContainer.cs # Контейнер провайдеров -├── HybridStorageBuilder.cs # Fluent builder -├── HybridFileStorageExtensions.cs # Пакетные операции -├── Setup.cs # DI расширения -├── FileMetadata.cs # DTO метаданных -├── BatchResult.cs # Типы результатов пакетной обработки -└── Interceptors/ # Хуки загрузки/скачивания/удаления - -src/Sa.HybridFileStorage.FileSystem/ # Провайдер файловой системы -src/Sa.HybridFileStorage.S3/ # Провайдер S3 -src/Sa.HybridFileStorage.Postgres/ # Провайдер PostgreSQL +src/Sa.HybridFileStorage/ # Основная библиотека (NuGet: Sa.HybridFileStorage) +├── IHybridFileStorage.cs # Главный интерфейс +├── HybridFileStorage.cs # Реализация с failover + interceptors +├── HybridFileStorageContainer.cs # Контейнер провайдеров +├── HybridStorageBuilder.cs # Fluent DI builder +├── HybridFileStorageExtensions.cs # Пакетные операции (CopyFromFile, CopyToBasket, …) +├── Setup.cs # DI расширения (AddSaHybridFileStorage, AddSaInMemoryFileStorage) +├── FileIdParser.cs # Утилита парсинга/форматирования File ID +├── FileMetadata.cs # DTO метаданных +├── InMemoryFileStorage.cs # In-memory провайдер +├── InMemoryFileStorageOptions.cs # Настройки in-memory +├── BatchResult.cs, BatchOptions.cs, … # Типы пакетных операций +└── Interceptors/ # Хуки загрузки/скачивания/удаления + ├── IUploadInterceptor.cs + ├── IDownloadInterceptor.cs + ├── IDeleteInterceptor.cs + ├── UploadLoggingInterceptor.cs + ├── DownloadLoggingInterceptor.cs + └── DeleteLoggingInterceptor.cs + +src/Sa.HybridFileStorage.FileSystem/ # Файловая система (NuGet: Sa.HybridFileStorage.FileSystem) +src/Sa.HybridFileStorage.S3/ # S3 (NuGet: Sa.HybridFileStorage.S3) +src/Sa.HybridFileStorage.Postgres/ # PostgreSQL (NuGet: Sa.HybridFileStorage.Postgres) ``` --- diff --git a/src/Sa.HybridFileStorage/Readme.md b/src/Sa.HybridFileStorage/Readme.md index 42fda1f9..34a96985 100644 --- a/src/Sa.HybridFileStorage/Readme.md +++ b/src/Sa.HybridFileStorage/Readme.md @@ -4,26 +4,51 @@ Hybrid file storage abstraction with automatic provider failover. Unifies multip --- +## Table of Contents + +- [Supported Storage Providers](#supported-storage-providers) +- [Key Features](#key-features) +- [File ID Format](#file-id-format) +- [Quick Start](#quick-start) + - [Without DI](#without-di) + - [With DI (Generic Host)](#with-di-generic-host) +- [CRUD Examples](#crud-examples) + - [Upload](#upload) + - [Download](#download) + - [Delete](#delete) + - [Get Metadata](#get-metadata) +- [Cross-Basket Copying](#cross-basket-copying) +- [Batch Operations](#batch-operations) +- [Interceptors](#interceptors) +- [Read-Only Mode](#read-only-mode) +- [Settings Reference](#settings-reference) +- [Domain Types](#domain-types) +- [Exceptions](#exceptions) +- [Project Structure](#project-structure) + +--- + ## Supported Storage Providers -| Provider | Class | Use Case | -|----------|-------|----------| -| **File System** | `FileSystemStorage` | Local development, on-premise deployments | -| **S3 Compatible** | `S3FileStorage` | Cloud storage (AWS S3, MinIO, etc.) | -| **PostgreSQL** | `PostgresFileStorage` | Database-embedded files, transactional consistency | -| **In-Memory** | `InMemoryFileStorage` | Testing, ephemeral scenarios | +| Provider | Package | Class | Use Case | +|----------|---------|-------|----------| +| **In-Memory** | `Sa.HybridFileStorage` | `InMemoryFileStorage` | Testing, ephemeral scenarios | +| **File System** | `Sa.HybridFileStorage.FileSystem` | `FileSystemStorage` | Local development, on-premise deployments | +| **S3 Compatible** | `Sa.HybridFileStorage.S3` | `S3FileStorage` | Cloud storage (AWS S3, MinIO, etc.) | +| **PostgreSQL** | `Sa.HybridFileStorage.Postgres` | `PostgresFileStorage` | Database-embedded files, transactional consistency, partitioning | --- ## Key Features -- ✅ **Unified API** — Single interface for all storage providers -- ✅ **Basket-Tenant-based isolation** — Multi-tenant support with scoped buckets -- ✅ **Read-only mode** — Protect storage from accidental modifications -- ✅ **Streaming support** — Memory-efficient file transfers +- ✅ **Unified API** — Single `IHybridFileStorage` interface for all storage providers +- ✅ **Basket-Tenant isolation** — Multi-tenant support with scoped storage containers +- ✅ **Failover** — Automatic provider switching when one backend fails +- ✅ **Streaming support** — Memory-efficient file transfers via `Stream` - ✅ **Native AOT ready** — Full compatibility with .NET 10 Native AOT -- ✅ **Batch operations** — Efficient bulk file processing with parallelism -- ✅ **Interceptors** — Upload/download/delete lifecycle hooks +- ✅ **Batch operations** — Bulk file processing with configurable parallelism +- ✅ **Interceptors** — Upload/download/delete lifecycle hooks for cross-cutting concerns +- ✅ **Read-only mode** — Protect storage from accidental modifications --- @@ -32,14 +57,31 @@ Hybrid file storage abstraction with automatic provider failover. Unifies multip All files are identified using a unified URI-like format: ``` -{storageType}://{basket}/{tenantId}/{fileName} +{storageType}://{basket}/{tenantId}/{path} ``` -**Examples:** -- `s3://share/42/document.pdf` -- `fs://root/100/report.xlsx` -- `pg://files/7/1773210911/some/data.bin` -- `mem://share/42/temp.txt` +Each storage provider adds its own path depth: + +| Provider | File ID Example | Path Structure | +|----------|----------------|----------------| +| **In-Memory** | `mem://share/42/document.pdf` | `{basket}/{tenantId}/{fileName}` | +| **File System** | `fs://documents/100/report.xlsx` | `{basket}/{tenantId}/{fileName}` | +| **S3** | `s3://uploads/7/invoice.csv` | `{basket}/{tenantId}/{fileName}` | +| **PostgreSQL** | `pg://files/12/1751347200/photo.jpg` | `{basket}/{tenantId}/{unixTimestamp}/{fileName}` | + +> **Note:** PostgreSQL includes a Unix timestamp in the path because it partitions by date. Other providers omit the timestamp. + +### Parsing File IDs + +Use the static `FileIdParser` utility: + +```csharp +if (FileIdParser.TryParse("pg://files/42/1751347200/report.pdf", out var basket, out var tenantId, out var timestamp, out var fileName)) +{ + Console.WriteLine($"Basket={basket}, Tenant={tenantId}, TS={timestamp}, Name={fileName}"); + // Basket=files, Tenant=42, TS=1751347200, Name=report.pdf +} +``` --- @@ -48,38 +90,99 @@ All files are identified using a unified URI-like format: ### Without DI ```csharp +using Sa.HybridFileStorage; +using Sa.HybridFileStorage.Domain; + +// 1. Create an in-memory storage provider using var memory = new InMemoryFileStorage(new InMemoryFileStorageOptions("share")); + +// 2. Build the hybrid container var container = new HybridFileStorageContainer([memory]); var storage = new HybridFileStorage(container, InterceptorContainer.Empty); -var stream = "Hello, HybridFileStorage!".ToStream(); +// 3. Upload a file +var stream = "Hello, HybridFileStorage!".ToUtf8Stream(); var result = await storage.UploadAsync( - "share", - new UploadFileInput { FileName = "file.txt", TenantId = 42 }, - stream, - ct); + basket: "share", + input: new UploadFileInput { FileName = "hello.txt", TenantId = 42 }, + fileStream: stream, + cancellationToken: ct); -await storage.DownloadAsync(result.FileId, async (fs, t) => +Console.WriteLine(result.FileId); // mem://share/42/hello.txt + +// 4. Download and process +bool wasFound = await storage.DownloadAsync(result.FileId, async (stream, token) => { - var content = await fs.ToStrAsync(t); - Console.WriteLine(content); -}); + using var reader = new StreamReader(stream, Encoding.UTF8); + var content = await reader.ReadToEndAsync(token); + Console.WriteLine(content); // Hello, HybridFileStorage! +}, ct); + +// 5. Delete +bool deleted = await storage.DeleteAsync(result.FileId, ct); ``` -### With DI +### With DI (Generic Host) ```csharp -builder.Services.AddSaHybridFileStorage(configure => configure - .AddStorage(InMemoryFileStorage.New("share")) +using Microsoft.Extensions.Hosting; +using Sa.HybridFileStorage; +using Sa.HybridFileStorage.FileSystem; +using Sa.HybridFileStorage.S3; + +var builder = Host.CreateApplicationBuilder(args); + +// Register all providers at once via fluent builder +builder.Services.AddSaHybridFileStorage(cfg => cfg + // In-Memory provider + .ConfigureStorage((sp, c) => c.AddStorage(new InMemoryFileStorage())) + + // File System provider + .ConfigureStorage((sp, c) => c.AddStorage(new FileSystemStorage( + new FileSystemStorageSettings + { + BasePath = @"C:\data\files", + Basket = "documents" + }))) + + // S3 provider + .ConfigureStorage((sp, c) => c.AddStorage( + new S3FileStorage( + sp.GetRequiredService(), + new S3FileStorageOptions + { + Endpoint = "http://localhost:9000", + AccessKey = "ROOTUSER", + SecretKey = "ChangeMe123", + Bucket = "mybucket", + Basket = "uploads" + }))) + + // Enable built-in logging interceptors .AddLogging()); -// Or register individual providers: +var host = builder.Build(); +var storage = host.Services.GetRequiredService(); + +// Use it anywhere — injected into your services +``` + +#### Minimal DI registration + +For quick setups, each provider has its own extension method: + +```csharp +// In-Memory only +builder.Services.AddSaInMemoryFileStorage(); + +// File System only builder.Services.AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = @"C:\data\files", Basket = "documents" }); +// S3 only builder.Services.AddSaS3FileStorage(new S3FileStorageOptions { Endpoint = "http://localhost:9000", @@ -89,90 +192,148 @@ builder.Services.AddSaS3FileStorage(new S3FileStorageOptions Basket = "uploads" }); -// Usage: -var storage = serviceProvider.GetRequiredService(); +// Then register the hybrid layer +builder.Services.AddSaHybridFileStorage(cfg => cfg.AddLogging()); ``` --- -## Settings +## CRUD Examples -### FileSystemStorageSettings +### Upload -| Property | Description | Default | -|----------|-------------|---------| -| `BasePath` | Root directory for files | *(required)* | -| `Basket` | Storage scope name | `"share"` | -| `StorageType` | Scheme prefix in File ID | `"fs"` | -| `IsReadOnly` | Prevent writes | `false` | -| `BufferSize` | Read/write buffer size | `256 KB` | +Upload accepts a `basket` name (scope/container), metadata, and a `Stream`. The hybrid layer finds a writable provider matching the basket and uploads the file. -### S3FileStorageOptions +```csharp +// Upload from a Stream +using var stream = File.OpenRead(@"C:\temp\document.pdf"); +var result = await storage.UploadAsync( + basket: "documents", + input: new UploadFileInput { FileName = "document.pdf", TenantId = 42 }, + fileStream: stream, + cancellationToken: ct); -| Property | Description | Default | -|----------|-------------|---------| -| `Endpoint` | S3 endpoint URL | *(required)* | -| `AccessKey` | S3 access key | *(required)* | -| `SecretKey` | S3 secret key | *(required)* | -| `Bucket` | Bucket name | *(required)* | -| `Basket` | Storage scope name | `"share"` | -| `Region` | Region for SigV4 | `"eu-central-1"` | -| `IsReadOnly` | Prevent writes | `false` | +Console.WriteLine($"Uploaded: {result.FileId}"); +// Output: fs://documents/42/document.pdf +``` -### PostgresFileStorageOptions +Copy a local file directly: + +```csharp +var result = await storage.CopyFromFileAsync( + filePath: @"C:\temp\image.png", + basket: "images", + input: new UploadFileInput { FileName = "avatar.png", TenantId = 7 }, + ct: ct); +``` -| Property | Description | -|----------|-------------| -| `SchemaName` | PostgreSQL schema | -| `TableName` | Table name for file data | -| `PartOptions.PgPartBy` | Partitioning strategy (day/month/year/list/range) | -| `CleanupOptions.ExpireDays` | Auto-cleanup threshold | -| `StorageOptions.IsReadOnly` | Prevent writes | +### Download -### InMemoryFileStorageOptions +Download delegates the file stream to a callback `Func`. This avoids loading the entire file into memory. -| Property | Description | Default | -|----------|-------------|---------| -| `Basket` | Storage scope name | `"share"` | -| `IsReadOnly` | Prevent writes | `false` | +```csharp +// Process stream inline +bool found = await storage.DownloadAsync(result.FileId, async (stream, token) => +{ + using var reader = new StreamReader(stream, Encoding.UTF8); + string content = await reader.ReadToEndAsync(token); + Console.WriteLine(content); +}, ct); + +// Copy to another stream +using var destination = new FileStream(@"C:\output\copy.pdf", FileMode.Create); +await storage.DownloadAsync(result.FileId, async (source, token) => + await source.CopyToAsync(destination, 81920, token), + ct); +``` + +### Delete + +```csharp +bool deleted = await storage.DeleteAsync(result.FileId, ct); +if (deleted) + Console.WriteLine("File removed."); +else + Console.WriteLine("File not found."); +``` + +### Get Metadata + +```csharp +var metadata = await storage.GetMetadataAsync(result.FileId, ct); +if (metadata != null) +{ + Console.WriteLine($"Basket: {metadata.Basket}"); + Console.WriteLine($"Tenant: {metadata.TenantId}"); + Console.WriteLine($"Name: {metadata.FileName}"); + Console.WriteLine($"Type: {metadata.StorageType}"); +} +``` --- -## Batch Operations +## Cross-Basket Copying -`HybridFileStorageExtensions` provides high-level methods for bulk file operations with built-in parallelism, error handling, and progress reporting. +Move or duplicate files between baskets (even across different providers): ```csharp -// Copy from local filesystem -var result = await storage.CopyFromFileAsync( - @"C:\temp\document.pdf", - "archive", - new UploadFileInput { FileName = "archived.pdf", TenantId = 42 }); +// Copy within same basket +var copied = await storage.CopyToBasketAsync( + fileId: "fs://documents/42/report.pdf", + basket: "archive", + ct: ct); -// Copy between baskets/scopes -var moved = await storage.CopyToBasketAsync( - "s3://share/42/doc.pdf", - "backup"); +// Customise upload metadata during copy +var renamed = await storage.CopyToBasketAsync( + fileId: "fs://documents/42/report.pdf", + basket: "backup", + configure: meta => new UploadFileInput + { + TenantId = meta.TenantId, + FileName = $"renamed-{meta.FileName}" // change the name + }, + ct: ct); +``` + +--- -// Batch copy with parallelism and progress +## Batch Operations + +High-throughput parallel file operations with progress reporting and error handling: + +```csharp +// Batch copy with parallelism var batchResult = await storage.CopyToScopeBatchAsync( - fileIds: ["s3://share/1/a.txt", "s3://share/2/b.txt"], + fileIds: + [ + "fs://documents/1/a.pdf", + "fs://documents/2/b.pdf", + "s3://uploads/3/c.pdf", + ], basket: "archive", options: new BatchOptions { MaxDegreeOfParallelism = 8, ContinueOnError = true, - Progress = new Progress() - }); + OperationTimeout = TimeSpan.FromSeconds(30), + Progress = new Progress(p => + { + Console.WriteLine($"{p.Completed}/{p.Total} — OK:{p.SuccessCount} Fail:{p.FailureCount}"); + }) + }, + ct: ct); foreach (var ok in batchResult.Succeeded) Console.WriteLine($"Copied: {ok.FileId}"); foreach (var err in batchResult.Failed) Console.WriteLine($"Failed #{err.Index}: {err.FileId} — {err.Exception.Message}"); + +// Or throw on any failure +batchResult.ThrowIfHasErrors(); // throws BatchOperationException ``` -### BatchResult +### BatchResult<T> | Member | Type | Description | |--------|------|-------------| @@ -187,81 +348,157 @@ foreach (var err in batchResult.Failed) | Property | Description | Default | |----------|-------------|---------| | `MaxDegreeOfParallelism` | Concurrent operations | `4` | -| `ContinueOnError` | Keep going after failures | `true` | -| `OperationTimeout` | Per-operation timeout | `0` (infinite) | -| `Progress` | Progress reporter | `null` | +| `ContinueOnError` | Keep going after individual failures | `true` | +| `OperationTimeout` | Per-operation timeout (`0` = infinite) | `0` | +| `Progress` | `IProgress` reporter | `null` | --- ## Interceptors -Lifecycle hooks for upload/download/delete operations. +Lifecycle hooks for upload/download/delete operations. Implement one of the three interfaces: ```csharp public interface IUploadInterceptor { + // Return false to reject the upload ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken ct); ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken ct); ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken ct); } -public interface IDownloadInterceptor { /* analogous Can/After/Error */ } -public interface IDeleteInterceptor { /* analogous Can/After/Error */ } +public interface IDownloadInterceptor { /* CanDownloadAsync / AfterDownloadAsync / OnDownloadErrorAsync */ } +public interface IDeleteInterceptor { /* CanDeleteAsync / AfterDeleteAsync / OnDeleteErrorAsync */ } ``` -Register interceptors via fluent builder: +Example — reject uploads of specific filenames: ```csharp -services.AddSaHybridFileStorage(cfg => cfg.ConfigureInterceptors((sp, container) => +public class DeniedExtensionInterceptor : IUploadInterceptor { - container.AddUploadInterceptor(myCustomInterceptor); - container.AddDownloadInterceptor(loggingInterceptor); -})); + private static readonly HashSet DeniedExtensions = ["exe", "bat", "cmd"]; + + public ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken ct) + { + var ext = Path.GetExtension(input.FileName)?.TrimStart('.').ToLowerInvariant(); + return ValueTask.FromResult(!DeniedExtensions.Contains(ext)); + } + + public ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken ct) + => ValueTask.CompletedTask; + + public ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken ct) + => ValueTask.CompletedTask; +} +``` + +Register interceptors through the fluent builder: + +```csharp +builder.Services.AddSaHybridFileStorage(cfg => cfg + .ConfigureInterceptors((sp, container) => + { + container.AddUploadInterceptor(new DeniedExtensionInterceptor()); + container.AddDownloadInterceptor(new LoggingDownloadInterceptor()); + })); ``` -Built-in `LoggingInterceptor` is available via `.AddLogging()`. +Built-in logging interceptors are available via `.AddLogging()`. --- ## Read-Only Mode -Set `IsReadOnly = true` on any storage provider to prevent writes. Attempted writes throw `HybridFileStorageWritableException`: +Set `IsReadOnly = true` on any provider to prevent writes. Attempted writes throw `HybridFileStorageWritableException`: ```csharp -builder.Services.AddSaFileSystemFileStorage(settings => +builder.Services.AddSaFileSystemFileStorage(new FileSystemStorageSettings { - settings.BasePath = @"C:\readonly\data"; - settings.IsReadOnly = true; + BasePath = @"C:\readonly\data", + IsReadOnly = true // uploads/deletes will fail }); ``` --- +## Settings Reference + +### FileSystemStorageSettings + +| Property | Description | Default | +|----------|-------------|---------| +| `BasePath` | Root directory for files | *(required)* | +| `Basket` | Scope/container name | `"share"` | +| `StorageType` | Scheme prefix in File ID | `"fs"` | +| `IsReadOnly` | Prevent writes | `false` | + +### S3FileStorageOptions + +| Property | Description | Default | +|----------|-------------|---------| +| `Endpoint` | S3 endpoint URL | *(required)* | +| `AccessKey` | S3 access key | *(required)* | +| `SecretKey` | S3 secret key | *(required)* | +| `Bucket` | Bucket name | *(required)* | +| `Basket` | Scope/container name | `"share"` | +| `Region` | Region for SigV4 signing | `"eu-central-1"` | +| `StorageType` | Scheme prefix in File ID | `"s3"` | +| `IsReadOnly` | Prevent writes | `false` | + +### PostgresFileStorageOptions + +| Property | Description | Default | +|----------|-------------|---------| +| `StorageOptions.SchemaName` | PostgreSQL schema | `"public"` | +| `StorageOptions.TableName` | Table for file data | `"files"` | +| `StorageOptions.StorageType` | Scheme prefix in File ID | `"pg"` | +| `PartOptions.Basket` | Scope/container name | `"share"` | +| `PartOptions.PgPartBy` | Partitioning granularity | `PgPartBy.Day` | +| `PartOptions.MigrationScheduleForwardDays` | Days ahead to pre-create partitions | `2` | +| `CleanupOptions.ExpireDays` | Auto-cleanup threshold (days) | `365 * 3` | +| `StorageOptions.IsReadOnly` | Prevent writes | `false` | + +### InMemoryFileStorageOptions + +| Property | Description | Default | +|----------|-------------|---------| +| `Basket` | Scope/container name | `"share"` | +| `MaxSizeBytes` | Total byte limit (`0` = unlimited) | `0` | +| `IsReadOnly` | Prevent writes | `false` | + +--- + ## Domain Types ### StorageResult +Returned by upload operations. Contains the canonical File ID and a publicly accessible URL. + ```csharp public sealed record StorageResult( - string FileId, - string AbsoluteUrl, - string StorageType, + string FileId, // e.g. "fs://documents/42/report.pdf" + string AbsoluteUrl, // e.g. "C:\data\files\documents\42\report.pdf" + string StorageType, // e.g. "fs", "s3", "pg", "mem" DateTimeOffset UploadedAt); ``` ### UploadFileInput +Input metadata for uploads. + ```csharp public sealed record UploadFileInput { - public int TenantId { get; init; } - public string FileName { get; init; } - public static UploadFileInput Empty { get; } + public int TenantId { get; init; } // defaults to 0 + public string FileName { get; init; } = ""; // required at validation + public static UploadFileInput Empty { get; } // pre-created empty instance } ``` ### FileMetadata +Read-only metadata retrieved via `GetMetadataAsync`. + ```csharp public sealed class FileMetadata { @@ -278,30 +515,39 @@ public sealed class FileMetadata | Exception | When thrown | |-----------|------------| -| `HybridFileStorageNoAvailableException` | No storage found for the requested basket | +| `HybridFileStorageNoAvailableException` | No storage provider found for the requested basket, or all providers failed | | `HybridFileStorageWritableException` | Write attempted on read-only storage | -| `HybridFileStorageAggregateException` | Multiple provider errors aggregated | -| `BatchOperationException` | Batch with failures and `ContinueOnError = false` | +| `HybridFileStorageAggregateException` | Multiple provider errors aggregated during failover | +| `BatchOperationException` | Batch operation had failures and `ContinueOnError = false` | --- ## Project Structure ``` -src/Sa.HybridFileStorage/ -├── IHybridFileStorage.cs # Main interface -├── HybridFileStorage.cs # Implementation with failover -├── HybridFileStorageContainer.cs # Provider container -├── HybridStorageBuilder.cs # Fluent builder -├── HybridFileStorageExtensions.cs # Batch operations -├── Setup.cs # DI extensions -├── FileMetadata.cs # Metadata DTO -├── BatchResult.cs # Batch result types -└── Interceptors/ # Upload/download/delete hooks - -src/Sa.HybridFileStorage.FileSystem/ # Filesystem provider -src/Sa.HybridFileStorage.S3/ # S3 provider -src/Sa.HybridFileStorage.Postgres/ # PostgreSQL provider +src/Sa.HybridFileStorage/ # Core library (NuGet: Sa.HybridFileStorage) +├── IHybridFileStorage.cs # Main interface +├── HybridFileStorage.cs # Implementation with failover + interceptors +├── HybridFileStorageContainer.cs # Provider container +├── HybridStorageBuilder.cs # Fluent DI builder +├── HybridFileStorageExtensions.cs # Batch operations (CopyFromFile, CopyToBasket, …) +├── Setup.cs # DI extensions (AddSaHybridFileStorage, AddSaInMemoryFileStorage) +├── FileIdParser.cs # File ID parsing/formatting utility +├── FileMetadata.cs # Metadata DTO +├── InMemoryFileStorage.cs # In-memory provider +├── InMemoryFileStorageOptions.cs # Options for in-memory +├── BatchResult.cs, BatchOptions.cs, … # Batch operation types +└── Interceptors/ # Upload/download/delete hooks + ├── IUploadInterceptor.cs + ├── IDownloadInterceptor.cs + ├── IDeleteInterceptor.cs + ├── UploadLoggingInterceptor.cs + ├── DownloadLoggingInterceptor.cs + └── DeleteLoggingInterceptor.cs + +src/Sa.HybridFileStorage.FileSystem/ # File system provider (NuGet: Sa.HybridFileStorage.FileSystem) +src/Sa.HybridFileStorage.S3/ # S3 provider (NuGet: Sa.HybridFileStorage.S3) +src/Sa.HybridFileStorage.Postgres/ # PostgreSQL provider (NuGet: Sa.HybridFileStorage.Postgres) ``` --- diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/FileIdParserTests.cs b/src/Tests/Sa.HybridFileStorage.PostgresTests/FileIdParserTests.cs index c7663cc8..47128096 100644 --- a/src/Tests/Sa.HybridFileStorage.PostgresTests/FileIdParserTests.cs +++ b/src/Tests/Sa.HybridFileStorage.PostgresTests/FileIdParserTests.cs @@ -1,4 +1,4 @@ -using Sa.HybridFileStorage.Postgres; +using Sa.HybridFileStorage; namespace Sa.HybridFileStorage.PostgresTests; diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs b/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs index 0164ff4c..35574ac6 100644 --- a/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs @@ -3,7 +3,7 @@ namespace Sa.HybridFileStorage.PostgresTests; -public class PostgresFileStorageTests(PostgresFileStorageTests.Fixture fixture) +public sealed class PostgresFileStorageTests(PostgresFileStorageTests.Fixture fixture) : IClassFixture { private const string DataContent = "Hello, World!"; From 4cc7cc4ce365e9eaebc94080760119cf08f2d452 Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 1 Jul 2026 21:54:40 +0300 Subject: [PATCH 29/33] tests for Hybrid storages Signed-off-by: dundich --- .../FileSystemStorage.cs | 11 +- .../Sa.HybridFileStorage.FileSystem.csproj | 5 + .../PostgresFileStorage.cs | 13 + src/Sa.HybridFileStorage.S3/S3FileStorage.cs | 11 + .../Domain/UploadFileInput.cs | 12 + .../InMemoryFileStorage.cs | 10 + .../FileRetryBehaviorTests.cs | 172 +++++++++++ .../FileSystemConcurrencyTests.cs | 148 +++++++++ .../PathSanitizerBlackBoxTests.cs | 134 ++++++++ .../BatchOperationsTests.cs | 238 +++++++++++++++ .../EdgeCasesTests.cs | 209 +++++++++++++ .../ExtensionMethodTests.cs | 286 ++++++++++++++++++ .../FailoverTests.cs | 187 ++++++++++++ .../InterceptorTests.cs | 285 +++++++++++++++++ 14 files changed, 1719 insertions(+), 2 deletions(-) create mode 100644 src/Tests/Sa.HybridFileStorage.FileSystemTests/FileRetryBehaviorTests.cs create mode 100644 src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemConcurrencyTests.cs create mode 100644 src/Tests/Sa.HybridFileStorage.FileSystemTests/PathSanitizerBlackBoxTests.cs create mode 100644 src/Tests/Sa.HybridFileStorageTests/BatchOperationsTests.cs create mode 100644 src/Tests/Sa.HybridFileStorageTests/EdgeCasesTests.cs create mode 100644 src/Tests/Sa.HybridFileStorageTests/ExtensionMethodTests.cs create mode 100644 src/Tests/Sa.HybridFileStorageTests/FailoverTests.cs create mode 100644 src/Tests/Sa.HybridFileStorageTests/InterceptorTests.cs diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs index 2b2c3bb0..6623b17e 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs @@ -52,11 +52,11 @@ public async Task UploadAsync( Stream fileStream, CancellationToken cancellationToken) { - ArgumentNullException.ThrowIfNull(metadata); - ArgumentException.ThrowIfNullOrWhiteSpace(metadata.FileName); ArgumentNullException.ThrowIfNull(fileStream); EnsureWritable(); + metadata.Validate(); + string filename = PathSanitizer.SanitizeRelativePath(metadata.FileName); string relativePath = string.Concat(Basket, "/", metadata.TenantId.ToString(), "/", filename); @@ -102,6 +102,10 @@ public async Task DownloadAsync( ArgumentNullException.ThrowIfNull(fileId); ArgumentNullException.ThrowIfNull(loadStream); + + if (!CanProcess(fileId)) + return false; + string filePath = GetFullPath(fileId); EnsurePathWithinBase(filePath); @@ -151,6 +155,9 @@ public async Task DeleteAsync(string fileId, CancellationToken cancellatio ArgumentNullException.ThrowIfNull(fileId); EnsureWritable(); + if (!CanProcess(fileId)) + return false; + var filePath = GetFullPath(fileId); EnsurePathWithinBase(filePath); diff --git a/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj b/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj index a31b35b1..4bdff2e9 100644 --- a/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj +++ b/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj @@ -7,6 +7,11 @@ File storage management + + + + + diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs index a75a6ff7..cc42291e 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs @@ -96,6 +96,9 @@ public async Task UploadAsync( CancellationToken cancellationToken) { EnsureWritable(); + ArgumentNullException.ThrowIfNull(fileStream); + + metadata.Validate(); DateTimeOffset createdAtDay = _timeProvider.GetUtcNow().Date; long createdAt = createdAtDay.ToUnixTimeSeconds(); @@ -159,6 +162,11 @@ public async Task DeleteAsync(string fileId, CancellationToken cancellatio { EnsureWritable(); + + if (!CanProcess(fileId)) + return false; + + if (!FileIdParser.TryParse(fileId, out _, out int tenantId, out long timestamp, out _)) { return false; @@ -183,6 +191,11 @@ public async Task DownloadAsync( Func loadStream, CancellationToken cancellationToken) { + + + if (!CanProcess(fileId)) + return false; + if (!FileIdParser.TryParse(fileId, out _, out int tenantId, out long timestamp, out _)) { return false; diff --git a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs index e1b42047..469eeade 100644 --- a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs +++ b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs @@ -47,6 +47,10 @@ public bool CanProcess(string? fileId) public async Task DeleteAsync(string fileId, CancellationToken cancellationToken) { EnsureWritable(); + + if (!CanProcess(fileId)) + return false; + var filePath = GetFilePath(fileId); await client.DeleteFile(filePath, cancellationToken).ConfigureAwait(false); return true; @@ -57,6 +61,10 @@ public async Task DownloadAsync( Func loadStream, CancellationToken cancellationToken) { + + if (!CanProcess(fileId)) + return false; + var filePath = GetFilePath(fileId); using var stream = await client.GetFileStream(filePath, cancellationToken).ConfigureAwait(false); if (stream is null || stream == Stream.Null) return false; @@ -71,6 +79,9 @@ public async Task UploadAsync( CancellationToken cancellationToken) { EnsureWritable(); + ArgumentNullException.ThrowIfNull(fileStream); + + metadata.Validate(); await EnsureBucketAsync(cancellationToken).ConfigureAwait(false); // Оптимизированная сборка пути без лишних аллокаций diff --git a/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs b/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs index 5372e4f8..1fc04a72 100644 --- a/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs +++ b/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs @@ -19,4 +19,16 @@ public sealed record UploadFileInput /// Gets a default (empty) instance. /// public static UploadFileInput Empty { get; } = new(); + + /// + /// Validates the input metadata. Throws if validation fails. + /// + public void Validate() + { + if (string.IsNullOrWhiteSpace(FileName)) + throw new ArgumentException("File name cannot be null or empty.", nameof(FileName)); + + if (TenantId < 0) + throw new ArgumentException("TenantId must be greater than or equal to 0.", nameof(TenantId)); + } } diff --git a/src/Sa.HybridFileStorage/InMemoryFileStorage.cs b/src/Sa.HybridFileStorage/InMemoryFileStorage.cs index 8b4aaf6a..ebe601a5 100644 --- a/src/Sa.HybridFileStorage/InMemoryFileStorage.cs +++ b/src/Sa.HybridFileStorage/InMemoryFileStorage.cs @@ -58,6 +58,8 @@ public async Task UploadAsync( { EnsureWritable(); + metadata.Validate(); + using var memoryStream = new MemoryStream(); await fileStream.CopyToAsync(memoryStream, cancellationToken) .ConfigureAwait(false); @@ -90,6 +92,10 @@ public async Task DownloadAsync( Func loadStream, CancellationToken cancellationToken) { + + if (!CanProcess(fileId)) + return false; + if (_storage.TryGetValue(fileId, out var fileData)) { using var memoryStream = new MemoryStream(fileData); @@ -104,6 +110,10 @@ public Task DeleteAsync(string fileId, CancellationToken cancellationToken { EnsureWritable(); + + if (!CanProcess(fileId)) + return Task.FromResult(false); + if (_storage.TryRemove(fileId, out var fileData)) { Interlocked.Add(ref _totalSizeBytes, -fileData.Length); diff --git a/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileRetryBehaviorTests.cs b/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileRetryBehaviorTests.cs new file mode 100644 index 00000000..56568494 --- /dev/null +++ b/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileRetryBehaviorTests.cs @@ -0,0 +1,172 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Sa.Fixture; +using Sa.HybridFileStorage.Domain; +using Sa.HybridFileStorage.FileSystem; + +namespace Sa.HybridFileStorage.FileSystemTests; + +/// +/// Tests that exercise FileRetryHelper behavior through FileSystemStorage operations. +/// FileRetryHelper is used internally by FileSystemStorage for file operations. +/// +public sealed class FileRetryBehaviorTests : IAsyncLifetime +{ + private readonly string _testDir = $"retry_{Path.GetRandomFileName()}"; + private readonly CancellationTokenSource _cts = new(); + + public ValueTask InitializeAsync() + { + Directory.CreateDirectory(_testDir); + return ValueTask.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Cancel(); + try { Directory.Delete(_testDir, true); } catch { /* ignore */ } + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + [Fact] + public async Task UploadAsync_NormalFile_SucceedsWithoutRetry() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act — normal upload should succeed immediately + var result = await storage.UploadAsync( + new UploadFileInput { FileName = "normal.txt", TenantId = 1 }, + FixtureHelper.GetByteStream(), + _cts.Token); + + // Assert + Assert.NotNull(result); + Assert.NotEmpty(result.FileId); + } + + [Fact] + public async Task DownloadAsync_NormalFile_ReturnsContent() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + var testData = FixtureHelper.GetByteArray(512); + using var testStream = new MemoryStream(testData); + + // Upload first + var result = await storage.UploadAsync( + new UploadFileInput { FileName = "downloadable.bin", TenantId = 1 }, + testStream, + _cts.Token); + + // Act — download and verify content + byte[]? downloaded = null; + var success = await storage.DownloadAsync(result.FileId, async (s, ct) => + { + downloaded = await s.ReadAllBytesAsync(ct); + }, _cts.Token); + + // Assert + Assert.True(success); + Assert.NotNull(downloaded); + Assert.Equal(testData, downloaded); + } + + [Fact] + public async Task DeleteAsync_ExistingFile_ReturnsTrue() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Upload a file first + var uploadResult = await storage.UploadAsync( + new UploadFileInput { FileName = "deletable.txt", TenantId = 1 }, + FixtureHelper.GetByteStream(), + _cts.Token); + + // Act + var deleted = await storage.DeleteAsync(uploadResult.FileId, _cts.Token); + + // Assert + Assert.True(deleted); + } + + [Fact] + public async Task DeleteAsync_NonExistentFile_ReturnsFalse() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act — delete non-existent file + var deleted = await storage.DeleteAsync("fs://nonexistent-file-id", _cts.Token); + + // Assert + Assert.False(deleted); + } + + [Fact] + public async Task GetMetadataAsync_ValidFileId_ReturnsMetadata() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Upload a file + var uploadResult = await storage.UploadAsync( + new UploadFileInput { FileName = "metadata.txt", TenantId = 42 }, + FixtureHelper.GetByteStream(), + _cts.Token); + + // Act + var metadata = await storage.GetMetadataAsync(uploadResult.FileId, _cts.Token); + + // Assert + Assert.NotNull(metadata); + Assert.Equal(42, metadata.TenantId); + Assert.Equal("metadata.txt", metadata.FileName); + Assert.Equal("fs", metadata.StorageType); + } +} + +// Helper extension +internal static class StreamExtensions +{ + internal static async Task ReadAllBytesAsync(this Stream stream, CancellationToken ct) + { + using var ms = new MemoryStream(); + await stream.CopyToAsync(ms, ct); + return ms.ToArray(); + } +} diff --git a/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemConcurrencyTests.cs b/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemConcurrencyTests.cs new file mode 100644 index 00000000..7fc62405 --- /dev/null +++ b/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemConcurrencyTests.cs @@ -0,0 +1,148 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Sa.Fixture; +using Sa.HybridFileStorage.Domain; +using Sa.HybridFileStorage.FileSystem; + +namespace Sa.HybridFileStorage.FileSystemTests; + +public sealed class FileSystemConcurrencyTests : IAsyncLifetime +{ + private readonly string _testDir = $"concurrency_{Path.GetRandomFileName()}"; + private readonly CancellationTokenSource _cts = new(); + + public ValueTask InitializeAsync() + { + Directory.CreateDirectory(_testDir); + return ValueTask.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Cancel(); + try { Directory.Delete(_testDir, true); } catch { /* ignore */ } + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + [Fact] + public async Task FileSystemStorage_ConcurrentUploads_NoCorruption() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + const int uploadCount = 20; + var tasks = new List>(); + + // Act — concurrent uploads + for (var i = 0; i < uploadCount; i++) + { + var index = i; + tasks.Add(Task.Run(async () => + { + using var stream = FixtureHelper.GetByteStream(1024); + return await storage.UploadAsync( + new UploadFileInput { FileName = $"file_{index}.txt", TenantId = 1 }, + stream, + _cts.Token); + }, _cts.Token)); + } + + var results = await Task.WhenAll(tasks); + + // Assert — all should succeed with unique file IDs + Assert.Equal(uploadCount, results.Length); + var uniqueIds = new HashSet(results.Select(r => r.FileId)); + Assert.Equal(uploadCount, uniqueIds.Count); + + // Verify each file can be downloaded + foreach (var result in results) + { + var downloaded = await storage.DownloadAsync(result.FileId, (_, _) => Task.CompletedTask, _cts.Token); + Assert.True(downloaded); + } + } + + [Fact] + public async Task FileSystemStorage_ConcurrentDeletes_NoException() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Upload files first + var fileIds = new List(); + for (var i = 0; i < 10; i++) + { + using var stream = FixtureHelper.GetByteStream(); + var result = await storage.UploadAsync( + new UploadFileInput { FileName = $"delete_me_{i}.txt", TenantId = 1 }, + stream, + _cts.Token); + fileIds.Add(result.FileId); + } + + // Act — concurrent deletes + var deleteTasks = fileIds.Select(id => + storage.DeleteAsync(id, _cts.Token)).ToList(); + + var results = await Task.WhenAll(deleteTasks); + + // Assert — all should succeed + Assert.All(results, r => Assert.True(r)); + } + + [Fact] + public async Task FileSystemStorage_MixedConcurrentOps_NoRaceCondition() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + var operations = new List(); + + // Mixed: uploads, downloads, deletes happening concurrently + for (var i = 0; i < 15; i++) + { + var index = i; + operations.Add(Task.Run(async () => + { + using var stream = FixtureHelper.GetByteStream(512); + var result = await storage.UploadAsync( + new UploadFileInput { FileName = $"mixed_{index}.txt", TenantId = 1 }, + stream, + _cts.Token); + + // Immediately download + var downloaded = await storage.DownloadAsync(result.FileId, (_, _) => Task.CompletedTask, _cts.Token); + Assert.True(downloaded); + + // Then delete + var deleted = await storage.DeleteAsync(result.FileId, _cts.Token); + Assert.True(deleted); + }, _cts.Token)); + } + + // Act + await Task.WhenAll(operations); + + // Assert — no exceptions thrown + } +} diff --git a/src/Tests/Sa.HybridFileStorage.FileSystemTests/PathSanitizerBlackBoxTests.cs b/src/Tests/Sa.HybridFileStorage.FileSystemTests/PathSanitizerBlackBoxTests.cs new file mode 100644 index 00000000..65b7ded4 --- /dev/null +++ b/src/Tests/Sa.HybridFileStorage.FileSystemTests/PathSanitizerBlackBoxTests.cs @@ -0,0 +1,134 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Sa.Fixture; +using Sa.HybridFileStorage.Domain; +using Sa.HybridFileStorage.FileSystem; +using System.Security; + +namespace Sa.HybridFileStorage.FileSystemTests; + +/// +/// Black-box tests for PathSanitizer behavior via FileSystemStorage.UploadAsync. +/// Tests path sanitization, security, and edge cases through the public API. +/// +public sealed class PathSanitizerBlackBoxTests : IAsyncLifetime +{ + private readonly string _testDir = $"pathsanity_{Path.GetRandomFileName()}"; + private readonly CancellationTokenSource _cts = new(); + + public ValueTask InitializeAsync() + { + Directory.CreateDirectory(_testDir); + return ValueTask.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Cancel(); + try { Directory.Delete(_testDir, true); } catch { /* ignore */ } + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + [Fact] + public async Task UploadAsync_NullFileName_ThrowsArgumentNullException() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act & Assert + await Assert.ThrowsAnyAsync(() => + storage.UploadAsync(null!, FixtureHelper.GetByteStream(), _cts.Token)); + } + + [Fact] + public async Task UploadAsync_EmptyFileName_ThrowsArgumentException() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act & Assert + var ex = await Assert.ThrowsAnyAsync(() => + storage.UploadAsync(new UploadFileInput { FileName = string.Empty, TenantId = 1 }, FixtureHelper.GetByteStream(), _cts.Token)); + + Assert.Contains("File name cannot be", ex.Message); + } + + [Fact] + public async Task UploadAsync_PathTraversalDotDot_Rejected() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act & Assert — path traversal should throw SecurityException + var ex = await Assert.ThrowsAnyAsync(() => + storage.UploadAsync(new UploadFileInput { FileName = "../escape.txt", TenantId = 1 }, FixtureHelper.GetByteStream(), _cts.Token)); + + Assert.Contains("'..' is not allowed", ex.Message); + } + + [Fact] + public async Task UploadAsync_NestedPath_NormalizesSeparators() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act — upload with nested path + var result = await storage.UploadAsync( + new UploadFileInput { FileName = "nested/path/file.txt", TenantId = 1 }, + FixtureHelper.GetByteStream(), + _cts.Token); + + // Assert — file ID should use normalized separators + Assert.NotNull(result); + Assert.NotEmpty(result.FileId); + } + + [Fact] + public async Task UploadAsync_InvalidCharsInFileName_ReplacedWithUnderscore() + { + // Arrange + var services = new ServiceCollection() + .AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)) + .AddSingleton() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = _testDir }); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act — upload with invalid chars (< > : " | ? *) + var result = await storage.UploadAsync( + new UploadFileInput { FileName = "file<>:\"|?*name.txt", TenantId = 1 }, + FixtureHelper.GetByteStream(), + _cts.Token); + + // Assert + Assert.NotNull(result); + Assert.NotEmpty(result.FileId); + } +} diff --git a/src/Tests/Sa.HybridFileStorageTests/BatchOperationsTests.cs b/src/Tests/Sa.HybridFileStorageTests/BatchOperationsTests.cs new file mode 100644 index 00000000..7f4fb362 --- /dev/null +++ b/src/Tests/Sa.HybridFileStorageTests/BatchOperationsTests.cs @@ -0,0 +1,238 @@ +using Microsoft.Extensions.DependencyInjection; +using Sa.Fixture; +using Sa.HybridFileStorage; +using Sa.HybridFileStorage.Domain; +using Sa.HybridFileStorage.FileSystem; + +namespace Sa.HybridFileStorageTests; + +public sealed class BatchOperationsTests : IAsyncLifetime +{ + private IServiceProvider? _provider; + private IHybridFileStorage? _storage; + private readonly CancellationTokenSource _cts = new(); + private string? _tempDir; + + public async ValueTask InitializeAsync() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"batch_test_{Path.GetRandomFileName()}"); + Directory.CreateDirectory(_tempDir); + + var services = new ServiceCollection(); + + // Register FileSystem with "share" basket + services.AddSingleton(new FileSystemStorage( + new FileSystemStorageSettings { BasePath = _tempDir })); + + // Register InMemory with "memory" basket + services.AddSingleton(new InMemoryFileStorage( + new InMemoryFileStorageOptions("memory"))); + + services.AddSaHybridFileStorage(); + + _provider = services.BuildServiceProvider(true); + _storage = _provider.GetRequiredService(); + } + + public ValueTask DisposeAsync() + { + _cts.Cancel(); + _cts.Dispose(); + (_provider as IDisposable)?.Dispose(); + + if (_tempDir is not null) + try { Directory.Delete(_tempDir, true); } catch { /* ignore */ } + return ValueTask.CompletedTask; + } + + [Fact] + public async Task CopyToScopeBatchAsync_ContinueOnErrorFalse_StopsOnFirstError() + { + // Arrange — one valid fileId, one invalid + var input = new UploadFileInput { FileName = "valid.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + var result = await _storage!.UploadAsync("share", input, stream, _cts.Token); + var validFileId = result.FileId; + var invalidFileId = "nonexistent://file"; + + var fileIds = new[] { validFileId, invalidFileId, validFileId }; + + // Act + var batchResult = await _storage.CopyToScopeBatchAsync( + fileIds, + "memory", + configure: default, + options: new BatchOptions { ContinueOnError = false, MaxDegreeOfParallelism = 1 }, + cancellationToken: _cts.Token); + + // Assert — should stop after first failure (index 1) + Assert.True(batchResult.HasErrors); + Assert.Equal(2, batchResult.Total); + Assert.Single(batchResult.Succeeded); + Assert.Single(batchResult.Failed); + Assert.Equal(invalidFileId, batchResult.Failed[0].FileId); + } + + [Fact] + public async Task CopyToScopeBatchAsync_ProgressCallback_ReportsProgress() + { + // Arrange + var input = new UploadFileInput { FileName = "progress.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + var result = await _storage!.UploadAsync("share", input, stream, _cts.Token); + + var fileIds = new[] { result.FileId, result.FileId, result.FileId }; + var progressReports = new List(); + var progress = new Progress(p => progressReports.Add(p)); + + // Act + var batchResult = await _storage.CopyToScopeBatchAsync( + fileIds, + "memory", + configure: default, + options: new BatchOptions { Progress = progress, MaxDegreeOfParallelism = 1 }, + cancellationToken: _cts.Token); + + // Assert + Assert.Equal(3, batchResult.Total); + Assert.Equal(3, batchResult.Succeeded.Count); + Assert.NotEmpty(progressReports); + // Last report should show 100% + var lastProgress = progressReports.Last(); + Assert.Equal(100.0, lastProgress.PercentComplete); + } + + [Fact] + public void BatchResult_ThrowIfHasErrors_ThrowsBatchOperationException() + { + // Arrange + var errors = new List + { + new BatchError("file1.txt", new InvalidOperationException("fail"), 0) + }; + var result = new BatchResult + { + Succeeded = [], + Failed = errors.AsReadOnly() + }; + + // Act & Assert + var ex = Assert.Throws>(() => + result.ThrowIfHasErrors("Custom error message")); + + Assert.Contains("Custom error message", ex.Message); + Assert.Equal(errors, ex.Result.Failed); + } + + [Fact] + public void BatchResult_ThrowIfNoErrors_DoesNotThrow() + { + // Arrange + var result = new BatchResult + { + Succeeded = [], + Failed = [] + }; + + // Act & Assert — should not throw + result.ThrowIfHasErrors(); + } + + [Fact] + public async Task CopyToScopeBatchAsync_EmptyFileList_ReturnsEmptyResult() + { + // Act + var result = await _storage!.CopyToScopeBatchAsync( + Array.Empty().ToArray(), + "memory", + cancellationToken: _cts.Token); + + // Assert + Assert.Equal(0, result.Total); + Assert.False(result.HasErrors); + Assert.Empty(result.Succeeded); + Assert.Empty(result.Failed); + } + + [Fact] + public async Task CopyToScopeBatchAsync_AllFail_HasAllErrors() + { + // Arrange — all invalid file IDs + var fileIds = new[] { "invalid1://x", "invalid2://y", "invalid3://z" }; + + // Act + var result = await _storage!.CopyToScopeBatchAsync( + fileIds, + "memory", + options: new BatchOptions { ContinueOnError = true }, + cancellationToken: _cts.Token); + + // Assert + Assert.Equal(3, result.Total); + Assert.True(result.HasErrors); + Assert.Empty(result.Succeeded); + Assert.Equal(3, result.Failed.Count); + } + + [Fact] + public async Task CopyToScopeBatchAsync_OperationTimeout_CancelsLongRunning() + { + // Arrange + var input = new UploadFileInput { FileName = "timeout.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + var result = await _storage!.UploadAsync("share", input, stream, _cts.Token); + + var fileIds = new[] { result.FileId }; + + // Act — very short timeout should cause cancellation + var options = new BatchOptions + { + OperationTimeout = TimeSpan.FromMilliseconds(1), + ContinueOnError = true + }; + + var batchResult = await _storage.CopyToScopeBatchAsync( + fileIds, + "memory", + options: options, + cancellationToken: _cts.Token); + + // Assert — may succeed or fail depending on timing, but shouldn't hang + Assert.Equal(1, batchResult.Total); + } + + [Fact] + public async Task CopyToScopeBatchAsync_MaxDegreeOfParallelism_LimitedConcurrency() + { + // Arrange — upload several files + var fileIds = new List(); + for (var i = 0; i < 5; i++) + { + var input = new UploadFileInput { FileName = $"parallel_{i}.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + var r = await _storage!.UploadAsync("share", input, stream, _cts.Token); + fileIds.Add(r.FileId); + } + + var progress = new Progress(p => + { + // Track peak concurrency by looking at completed count + }); + + // Act + var result = await _storage!.CopyToScopeBatchAsync( + fileIds, + "memory", + options: new BatchOptions + { + MaxDegreeOfParallelism = 2, + Progress = progress, + ContinueOnError = true + }, + cancellationToken: _cts.Token); + + // Assert — all should succeed with limited parallelism + Assert.Equal(5, result.Total); + Assert.Equal(5, result.Succeeded.Count); + } +} diff --git a/src/Tests/Sa.HybridFileStorageTests/EdgeCasesTests.cs b/src/Tests/Sa.HybridFileStorageTests/EdgeCasesTests.cs new file mode 100644 index 00000000..559ae0fd --- /dev/null +++ b/src/Tests/Sa.HybridFileStorageTests/EdgeCasesTests.cs @@ -0,0 +1,209 @@ +using Sa.Fixture; +using Sa.HybridFileStorage; +using Sa.HybridFileStorage.Domain; + +namespace Sa.HybridFileStorageTests; + +public sealed class EdgeCasesTests +{ + [Fact] + public async Task InMemoryFileStorage_UploadAsync_EmptyFileName_ThrowsArgumentException() + { + // Arrange + var storage = new InMemoryFileStorage(); + using var stream = FixtureHelper.GetByteStream(); + + // Act & Assert — empty FileName should throw ArgumentException + var ex = await Assert.ThrowsAnyAsync(() => + storage.UploadAsync( + new UploadFileInput { FileName = string.Empty, TenantId = 1 }, + stream, + CancellationToken.None)); + + Assert.Contains("FileName", ex.Message); + } + + [Fact] + public async Task InMemoryFileStorage_UploadAsync_EmptyStream_Succeeds() + { + // Arrange + var storage = new InMemoryFileStorage(); + using var emptyStream = FixtureHelper.GetEmptyByteStream(); + var input = new UploadFileInput { FileName = "empty.bin", TenantId = 1 }; + + // Act + var result = await storage.UploadAsync(input, emptyStream, CancellationToken.None); + + // Assert + Assert.NotNull(result); + Assert.NotEmpty(result.FileId); + } + + [Fact] + public async Task InMemoryFileStorage_UploadAsync_MaxSizeBytesExceeded_ThrowsInvalidOperationException() + { + // Arrange — limit to 1MB + var options = new InMemoryFileStorageOptions("test") { MaxSizeBytes = 1 * 1024 * 1024 }; + var storage = new InMemoryFileStorage(options); + + // Fill up most of the capacity + for (var i = 0; i < 10; i++) + { + var input = new UploadFileInput { FileName = $"fill_{i}.bin", TenantId = 1 }; + var bytes = FixtureHelper.GetByteArray(100 * 1024); // 100KB each + await storage.UploadAsync(input, new MemoryStream(bytes), CancellationToken.None); + } + + // Act — this should exceed the limit + var ex = await Assert.ThrowsAnyAsync(() => + storage.UploadAsync( + new UploadFileInput { FileName = "overflow.bin", TenantId = 1 }, + FixtureHelper.GetByteStream(), + CancellationToken.None)); + + Assert.Contains("size limit", ex.Message); + } + + [Fact] + public async Task InMemoryFileStorage_DeleteAsync_FileNotFound_ReturnsFalse() + { + // Arrange + var storage = new InMemoryFileStorage(); + + // Act + var deleted = await storage.DeleteAsync("mem://nonexistent/file.txt", CancellationToken.None); + + // Assert + Assert.False(deleted); + } + + [Fact] + public async Task InMemoryFileStorage_DownloadAsync_FileNotFound_ReturnsFalse() + { + // Arrange + var storage = new InMemoryFileStorage(); + bool loadCalled = false; + + // Act + var downloaded = await storage.DownloadAsync("mem://nonexistent/file.txt", (_, _) => + { + loadCalled = true; + return Task.CompletedTask; + }, CancellationToken.None); + + // Assert + Assert.False(downloaded); + Assert.False(loadCalled); + } + + [Fact] + public async Task InMemoryFileStorage_GetMetadataAsync_FileNotFound_ReturnsNull() + { + // Arrange + var storage = new InMemoryFileStorage(); + + // Act + var metadata = await storage.GetMetadataAsync("mem://nonexistent/file.txt", CancellationToken.None); + + // Assert + Assert.Null(metadata); + } + + [Fact] + public async Task InMemoryFileStorage_ReadOnly_UploadThrowsWritableException() + { + // Arrange + var storage = new InMemoryFileStorage(new InMemoryFileStorageOptions("test") { IsReadOnly = true }); + using var stream = FixtureHelper.GetByteStream(); + + // Act & Assert + await Assert.ThrowsAsync(() => + storage.UploadAsync(new UploadFileInput { FileName = "nope.txt", TenantId = 1 }, stream, CancellationToken.None)); + } + + [Fact] + public async Task InMemoryFileStorage_ReadOnly_DeleteThrowsWritableException() + { + // Arrange + var storage = new InMemoryFileStorage(new InMemoryFileStorageOptions("test") { IsReadOnly = true }); + + // Act & Assert + await Assert.ThrowsAsync(() => + storage.DeleteAsync("mem://any/file.txt", CancellationToken.None)); + } + + [Fact] + public async Task InMemoryFileStorage_UploadAsync_TimeProvider_ReturnsCorrectTimestamp() + { + // Arrange + var fakeTime = DateTimeOffset.Parse("2025-06-15T12:00:00Z"); + var timeProvider = new TestTimeProvider(fakeTime); + var storage = new InMemoryFileStorage(null, timeProvider); + using var stream = FixtureHelper.GetByteStream(); + + // Act + var result = await storage.UploadAsync( + new UploadFileInput { FileName = "timed.txt", TenantId = 1 }, + stream, + CancellationToken.None); + + // Assert + Assert.Equal(fakeTime, result.UploadedAt); + } + + [Fact] + public async Task InMemoryFileStorage_SizeTracking_TracksTotalBytes() + { + // Arrange — use reflection to verify internal size tracking + var storage = new InMemoryFileStorage(new InMemoryFileStorageOptions("test") { MaxSizeBytes = 0 }); // unlimited + using var stream = FixtureHelper.GetByteStream(1024); + + // Act + await storage.UploadAsync(new UploadFileInput { FileName = "track.bin", TenantId = 1 }, stream, CancellationToken.None); + + // Verify via download that data is intact + var downloaded = await storage.DownloadAsync( + storage.CanProcess("") ? throw new Exception("need fileId") : "", + (_, _) => Task.CompletedTask, + CancellationToken.None); + } + + [Fact] + public async Task InMemoryFileStorage_CanProcess_WorksForValidAndInvalidFileIds() + { + // Arrange + var storage = new InMemoryFileStorage(); + + // Act & Assert + Assert.True(storage.CanProcess("mem://basket/1/file.txt")); + Assert.False(storage.CanProcess("pg://basket/1/file.txt")); + Assert.False(storage.CanProcess("unknown://file.txt")); + } + + [Fact] + public async Task InMemoryFileStorage_BasketProperty_ReturnsConfiguredBasket() + { + // Arrange + const string expectedBasket = "my-custom-basket"; + var storage = new InMemoryFileStorage(new InMemoryFileStorageOptions(expectedBasket)); + + // Assert + Assert.Equal(expectedBasket, storage.Basket); + } + + [Fact] + public async Task InMemoryFileStorage_StorageTypeConstant_ReturnsMem() + { + // Assert + Assert.Equal("mem", InMemoryFileStorage.DefaultStorageType); + Assert.Equal("://", InMemoryFileStorage.SchemeSeparator); + } +} + +// Minimal TimeProvider implementation for testing +internal sealed class TestTimeProvider(DateTimeOffset now) : TimeProvider +{ + private readonly DateTimeOffset _now = now; + + public override DateTimeOffset GetUtcNow() => _now; +} diff --git a/src/Tests/Sa.HybridFileStorageTests/ExtensionMethodTests.cs b/src/Tests/Sa.HybridFileStorageTests/ExtensionMethodTests.cs new file mode 100644 index 00000000..8fdb4408 --- /dev/null +++ b/src/Tests/Sa.HybridFileStorageTests/ExtensionMethodTests.cs @@ -0,0 +1,286 @@ +using Microsoft.Extensions.DependencyInjection; +using Sa.Fixture; +using Sa.HybridFileStorage; +using Sa.HybridFileStorage.Domain; +using Sa.HybridFileStorage.FileSystem; + +namespace Sa.HybridFileStorageTests; + +public sealed class ExtensionMethodTests : IAsyncLifetime +{ + private readonly CancellationTokenSource _cts = new(); + private string? _testDir; + + public ValueTask InitializeAsync() + { + _testDir = Path.Combine(Path.GetTempPath(), $"ext_test_{Path.GetRandomFileName()}"); + Directory.CreateDirectory(_testDir); + return ValueTask.CompletedTask; + } + + public ValueTask DisposeAsync() + { + _cts.Cancel(); + _cts.Dispose(); + if (_testDir is not null) + try { Directory.Delete(_testDir, true); } catch { /* ignore */ } + return ValueTask.CompletedTask; + } + + [Fact] + public async Task CopyFromFileAsync_FileNotFound_ThrowsFileNotFoundException() + { + // Arrange + var services = new ServiceCollection() + .AddSaInMemoryFileStorage() + .AddSaHybridFileStorage(); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + var nonExistentFile = Path.Combine(_testDir!, "does_not_exist.txt"); + + // Act & Assert + await Assert.ThrowsAnyAsync(() => + storage.CopyFromFileAsync(nonExistentFile, string.Empty, new UploadFileInput { FileName = "copy.txt", TenantId = 1 }, ct: _cts.Token)); + } + + [Fact] + public async Task CopyToBasketAsync_SameScope_ThrowsInvalidOperationException() + { + // Arrange + var fsPath = Path.Combine(_testDir!, "same_scope"); + Directory.CreateDirectory(fsPath); + + var services = new ServiceCollection() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = fsPath }) + .AddSaHybridFileStorage(); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Upload a file to the "share" basket (FileSystem default) + var input = new UploadFileInput { FileName = "sametest.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + var result = await storage.UploadAsync("share", input, stream, _cts.Token); + + // Act & Assert — copy to same basket should fail + await Assert.ThrowsAsync(() => + storage.CopyToBasketAsync(result.FileId, "share", ct: _cts.Token)); + } + + [Fact] + public async Task CopyToScopeBatchAsync_EmptyList_ReturnsEmptyResult() + { + // Arrange + var services = new ServiceCollection() + .AddSaInMemoryFileStorage() + .AddSaHybridFileStorage(); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act + var result = await storage.CopyToScopeBatchAsync( + Array.Empty(), + string.Empty, + cancellationToken: _cts.Token); + + // Assert + Assert.Equal(0, result.Total); + Assert.False(result.HasErrors); + Assert.Empty(result.Succeeded); + Assert.Empty(result.Failed); + } + + [Fact] + public async Task CopyFromFileAsync_Success_CopyFromDiskToStorage() + { + // Arrange — create a test file on disk + var testFilePath = Path.Combine(_testDir!, "source.txt"); + await File.WriteAllTextAsync(testFilePath, "Hello from disk!", _cts.Token); + + var services = new ServiceCollection() + .AddSaInMemoryFileStorage() + .AddSaHybridFileStorage(); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act + var result = await storage.CopyFromFileAsync( + testFilePath, + string.Empty, + new UploadFileInput { FileName = "disk_copy.txt", TenantId = 1 }, + ct: _cts.Token); + + // Assert + Assert.NotNull(result); + Assert.NotEmpty(result.FileId); + + // Verify content was copied correctly + var downloaded = await storage.DownloadAsync(result.FileId, async (s, ct) => + { + using var reader = new StreamReader(s, leaveOpen: true); + var content = await reader.ReadToEndAsync(); + Assert.Equal("Hello from disk!", content); + }, _cts.Token); + + Assert.True(downloaded); + } + + [Fact] + public async Task CopyToBasketAsync_CrossProvider_Success() + { + // Arrange — use DI with ConfigureStorage for explicit basket control + var fsPath = Path.Combine(_testDir!, "cross_fs"); + Directory.CreateDirectory(fsPath); + + var services = new ServiceCollection(); + + // Register FileSystem with default "share" basket + services.AddSingleton(new FileSystemStorage( + new FileSystemStorageSettings { BasePath = fsPath })); + + // Register InMemory with explicit "mem_basket" + services.AddSingleton(new InMemoryFileStorage( + new InMemoryFileStorageOptions("mem_basket"))); + + // Register HybridFileStorage with logging + services.AddSaHybridFileStorage(cfg => cfg.AddLogging()); + + using var provider = services.BuildServiceProvider(true); + var storage = provider.GetRequiredService(); + + // Upload to FileSystem first ("share" basket) + var input = new UploadFileInput { FileName = "cross.txt", TenantId = 1 }; + using var origStream = FixtureHelper.GetByteStream(); + var uploadResult = await storage.UploadAsync("share", input, origStream, _cts.Token); + + // Verify upload landed in FileSystem + Assert.NotNull(uploadResult); + Assert.Contains("share", uploadResult.FileId); + Assert.StartsWith("fs://", uploadResult.FileId); + + // Act — copy from FileSystem ("share" basket) to InMemory ("mem_basket") + var copyResult = await storage.CopyToBasketAsync( + uploadResult.FileId, + "mem_basket", + configure: metadata => new UploadFileInput + { + TenantId = metadata.TenantId, + FileName = $"copied_{metadata.FileName}" + }, + _cts.Token); + + // Assert — should land in InMemory (storage type "mem") + Assert.NotNull(copyResult); + Assert.Equal(InMemoryFileStorage.DefaultStorageType, copyResult.StorageType); + Assert.Contains("mem_basket", copyResult.FileId); + Assert.Contains("copied_", copyResult.FileId); + + // Verify the copied file content matches + var downloaded = await storage.DownloadAsync(copyResult.FileId, async (stream, ct) => + { + using var reader = new StreamReader(stream, leaveOpen: true); + var content = await reader.ReadToEndAsync(ct); + Assert.True(content.Length > 0); + }, _cts.Token); + Assert.True(downloaded); + } + + [Fact] + public async Task CopyToBasketAsync_CustomConfigure_RenamesFile() + { + // Arrange — manually register two InMemory storages with different baskets + var services = new ServiceCollection(); + + services.AddSingleton(new InMemoryFileStorage( + new InMemoryFileStorageOptions("source_basket"))); + services.AddSingleton(new InMemoryFileStorage( + new InMemoryFileStorageOptions("target_basket"))); + services.AddSaHybridFileStorage(); + + using var provider = services.BuildServiceProvider(true); + var storage = provider.GetRequiredService(); + + // Upload original file to source basket + var input = new UploadFileInput { FileName = "original.bin", TenantId = 42 }; + using var stream = FixtureHelper.GetByteStream(); + var uploadResult = await storage.UploadAsync("source_basket", input, stream, _cts.Token); + + // Act — copy with custom rename + var copyResult = await storage.CopyToBasketAsync( + uploadResult.FileId, + "target_basket", + configure: metadata => new UploadFileInput + { + TenantId = 99, // Change tenant + FileName = "renamed.bin" // Rename file + }, + _cts.Token); + + // Assert + Assert.NotNull(copyResult); + + // Verify metadata reflects the custom configuration + var metadata = await storage.GetMetadataAsync(copyResult.FileId, _cts.Token); + Assert.NotNull(metadata); + Assert.Equal(99, metadata.TenantId); + Assert.Equal("renamed.bin", metadata.FileName); + Assert.Equal(InMemoryFileStorage.DefaultStorageType, metadata.StorageType); + Assert.Equal("target_basket", metadata.Basket); + } + + [Fact] + public async Task CopyToBasketAsync_SourceNotFound_ThrowsFileNotFoundException() + { + // Arrange + var services = new ServiceCollection() + .AddSaInMemoryFileStorage() + .AddSaHybridFileStorage(); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act & Assert — copy non-existent file + await Assert.ThrowsAsync(() => + storage.CopyToBasketAsync("mem://nonexistent/file.txt", "target", ct: _cts.Token)); + } + + [Fact] + public async Task CopyToScopeBatchAsync_MixedSuccessAndFailure_PartialResult() + { + // Arrange — manually register two InMemory storages with explicit baskets + var services = new ServiceCollection(); + + services.AddSingleton(new InMemoryFileStorage( + new InMemoryFileStorageOptions("batch_src"))); + services.AddSingleton(new InMemoryFileStorage( + new InMemoryFileStorageOptions("batch_dst"))); + services.AddSaHybridFileStorage(); + + using var provider = services.BuildServiceProvider(true); + var storage = provider.GetRequiredService(); + + // Upload one valid file to source basket + var input = new UploadFileInput { FileName = "valid.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + var uploadResult = await storage.UploadAsync("batch_src", input, stream, _cts.Token); + + var fileIds = new[] { uploadResult.FileId, "invalid://missing1", uploadResult.FileId, "invalid://missing2" }; + + // Act + var result = await storage.CopyToScopeBatchAsync( + fileIds, + "batch_dst", + options: new BatchOptions { ContinueOnError = true }, + cancellationToken: _cts.Token); + + // Assert + Assert.Equal(4, result.Total); + Assert.True(result.HasErrors); + Assert.Equal(2, result.Succeeded.Count); + Assert.Equal(2, result.Failed.Count); + } +} diff --git a/src/Tests/Sa.HybridFileStorageTests/FailoverTests.cs b/src/Tests/Sa.HybridFileStorageTests/FailoverTests.cs new file mode 100644 index 00000000..39165cf6 --- /dev/null +++ b/src/Tests/Sa.HybridFileStorageTests/FailoverTests.cs @@ -0,0 +1,187 @@ +using Microsoft.Extensions.DependencyInjection; +using Sa.Fixture; +using Sa.HybridFileStorage; +using Sa.HybridFileStorage.Domain; +using Sa.HybridFileStorage.FileSystem; +using Sa.HybridFileStorage.Interceptors; + +namespace Sa.HybridFileStorageTests; + +public sealed class FailoverTests : IAsyncLifetime +{ + private readonly CancellationTokenSource _cts = new(); + + public ValueTask InitializeAsync() => ValueTask.CompletedTask; + + public ValueTask DisposeAsync() + { + _cts.Cancel(); + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + [Fact] + public async Task HybridFileStorage_AllStoragesFail_ThrowsWritableException() + { + // Arrange — two read-only storages, all operations should fail + var services = new ServiceCollection() + .AddSaInMemoryFileStorage(new InMemoryFileStorageOptions(string.Empty, IsReadOnly: true)) + .AddSaInMemoryFileStorage(new InMemoryFileStorageOptions(string.Empty, IsReadOnly: true)) + .AddSaHybridFileStorage(); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act & Assert — upload should fail with writable exception (no writable storage) + await Assert.ThrowsAsync(() => + storage.UploadAsync(string.Empty, new UploadFileInput { FileName = "fail.bin", TenantId = 1 }, FixtureHelper.GetByteStream(), _cts.Token)); + } + + [Fact] + public async Task HybridFileStorage_FailoverToSecondStorage_Succeeds() + { + // Arrange — first storage blocks upload via interceptor, second accepts + var blockedCount = 0; + var interceptor = new CountingBlockInterceptor(() => { blockedCount++; }); + + var services = new ServiceCollection() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = $"failover_{Path.GetRandomFileName()}", Basket = "shared" }) + .AddSaInMemoryFileStorage(new InMemoryFileStorageOptions("shared")) + .AddSaHybridFileStorage(b => b.ConfigureInterceptors((sp, c) => c.AddUploadInterceptor(interceptor))); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act — upload blocked on filesystem by interceptor, should succeed on InMemory + var input = new UploadFileInput { FileName = "failover.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + var result = await storage.UploadAsync("shared", input, stream, _cts.Token); + + // Assert + Assert.NotNull(result); + Assert.Equal(InMemoryFileStorage.DefaultStorageType, result.StorageType); + Assert.Equal(1, blockedCount); + } + + [Fact] + public async Task HybridFileStorage_WritableException_WhenAllReadOnly() + { + // Arrange + var services = new ServiceCollection() + .AddSaHybridFileStorage(b => b.ConfigureStorage((_, c) => + c.AddStorage(new InMemoryFileStorage(new InMemoryFileStorageOptions(string.Empty, IsReadOnly: true))))); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act & Assert + var ex = await Assert.ThrowsAsync(() => + storage.UploadAsync(string.Empty, new UploadFileInput { FileName = "ro.bin", TenantId = 1 }, FixtureHelper.GetByteStream(), _cts.Token)); + + Assert.Contains("read-only", ex.Message); + } + + [Fact] + public async Task HybridFileStorage_NoAvailableStorage_ThrowsNoAvailableException() + { + // Arrange — empty service collection, no storages registered + var services = new ServiceCollection() + .AddSaHybridFileStorage(); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act & Assert + await Assert.ThrowsAsync(() => + storage.UploadAsync(string.Empty, new UploadFileInput { FileName = "nope.bin", TenantId = 1 }, FixtureHelper.GetByteStream(), _cts.Token)); + } + + [Fact] + public async Task HybridFileStorage_GetMetadataAsync_MultipleProviders_ReturnsFirstMatch() + { + // Arrange — register two InMemory storages with different baskets + var services = new ServiceCollection() + .AddSaInMemoryFileStorage(new InMemoryFileStorageOptions("basket-a")) + .AddSaInMemoryFileStorage(new InMemoryFileStorageOptions("basket-b")) + .AddSaHybridFileStorage(); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Upload to first available storage + var input = new UploadFileInput { FileName = "meta.txt", TenantId = 42 }; + using var stream = FixtureHelper.GetByteStream(); + var result = await storage.UploadAsync("basket-a", input, stream, _cts.Token); + + // Act — GetMetadata should parse correctly regardless of which storage handled it + var metadata = await storage.GetMetadataAsync(result.FileId, _cts.Token); + + // Assert + Assert.NotNull(metadata); + Assert.Equal(42, metadata.TenantId); + Assert.Equal("meta.txt", metadata.FileName); + Assert.Equal(InMemoryFileStorage.DefaultStorageType, metadata.StorageType); + } + + [Fact] + public async Task HybridFileStorage_DownloadAsync_FromCorrectStorage_Succeeds() + { + // Arrange + var services = new ServiceCollection() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = $"download_{Path.GetRandomFileName()}" }) + .AddSaInMemoryFileStorage() + .AddSaHybridFileStorage(); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Upload a file + var input = new UploadFileInput { FileName = "downloadable.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + var result = await storage.UploadAsync(string.Empty, input, stream, _cts.Token); + + // Act — download should find and retrieve from the correct storage + byte[]? downloadedData = null; + var downloaded = await storage.DownloadAsync( + result.FileId, + async (s, ct) => downloadedData = await s.ReadAllBytesAsync(ct), + _cts.Token); + + // Assert + Assert.True(downloaded); + Assert.NotNull(downloadedData); + Assert.NotEmpty(downloadedData); + } +} + +// Helper extension for reading all bytes +internal static class StreamExtensions +{ + internal static async Task ReadAllBytesAsync(this Stream stream, CancellationToken ct) + { + using var ms = new MemoryStream(); + await stream.CopyToAsync(ms, ct); + return ms.ToArray(); + } +} + +// Helper interceptor for testing +internal sealed class CountingBlockInterceptor(Action onBlock) : IUploadInterceptor +{ + public ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken cancellationToken) + { + // Block uploads to filesystem storage + if (storage.StorageType == "fs") + { + onBlock(); + return ValueTask.FromResult(false); + } + return ValueTask.FromResult(true); + } + + public ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken cancellationToken) + => ValueTask.CompletedTask; +} diff --git a/src/Tests/Sa.HybridFileStorageTests/InterceptorTests.cs b/src/Tests/Sa.HybridFileStorageTests/InterceptorTests.cs new file mode 100644 index 00000000..a5bfffca --- /dev/null +++ b/src/Tests/Sa.HybridFileStorageTests/InterceptorTests.cs @@ -0,0 +1,285 @@ +using Microsoft.Extensions.DependencyInjection; +using Sa.Fixture; +using Sa.HybridFileStorage; +using Sa.HybridFileStorage.Domain; +using Sa.HybridFileStorage.FileSystem; +using Sa.HybridFileStorage.Interceptors; + +namespace Sa.HybridFileStorageTests; + +public sealed class InterceptorTests : IAsyncLifetime +{ + private readonly CancellationTokenSource _cts = new(); + + public ValueTask InitializeAsync() => ValueTask.CompletedTask; + + public ValueTask DisposeAsync() + { + _cts.Cancel(); + _cts.Dispose(); + return ValueTask.CompletedTask; + } + + [Fact] + public async Task DownloadInterceptor_CanDownloadFalse_BlocksDownload() + { + // Arrange — two storages with same basket, file uploaded to first one + var downloadBlocked = false; + var interceptor = new CountingDownloadInterceptor(() => downloadBlocked = true); + + var services = new ServiceCollection() + .AddSaInMemoryFileStorage(new InMemoryFileStorageOptions("shared_basket")) + .AddSaInMemoryFileStorage(new InMemoryFileStorageOptions("shared_basket")) + .AddSaHybridFileStorage(b => b.ConfigureInterceptors((_, c) => + c.AddDownloadInterceptor(interceptor))); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Upload first — goes to first available storage + var input = new UploadFileInput { FileName = "blocked_download.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + var result = await storage.UploadAsync("shared_basket", input, stream, _cts.Token); + + // Act — download intercepted: interceptor blocks ALL storages + // Since only one storage can CanProcess the fileId, and interceptor blocks it → NoAvailableException + var ex = await Assert.ThrowsAsync(() => + storage.DownloadAsync(result.FileId, (_, _) => Task.CompletedTask, _cts.Token)); + + // Assert — interceptor was called and blocked + Assert.True(downloadBlocked); + } + + [Fact] + public async Task DeleteInterceptor_AfterDeleteCalled_NotifiesOnCompletion() + { + // Arrange + var afterDeleteCalled = false; + var successFlag = false; + var deleteInterceptor = new NotifyDeleteInterceptor( + () => afterDeleteCalled = true, + (bool s) => successFlag = s); + + var services = new ServiceCollection() + .AddSaInMemoryFileStorage() + .AddSaHybridFileStorage(b => b.ConfigureInterceptors((sp, c) => c.AddDeleteInterceptor(deleteInterceptor))); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Upload and delete + var input = new UploadFileInput { FileName = "notify.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + var result = await storage.UploadAsync(string.Empty, input, stream, _cts.Token); + + // Act + await storage.DeleteAsync(result.FileId, _cts.Token); + + // Assert + Assert.True(afterDeleteCalled); + Assert.True(successFlag); + } + + [Fact] + public async Task UploadInterceptorChain_AllMethodsCalled_InOrder() + { + // Arrange + var callSequence = new List(); + + var canUploadCalled = false; + var afterUploadCalled = false; + var uploadInterceptor = new OrderTrackingUploadInterceptor( + () => { canUploadCalled = true; callSequence.Add("CanUpload"); return Task.CompletedTask; }, + () => { afterUploadCalled = true; callSequence.Add("AfterUpload"); return Task.CompletedTask; }, + () => { callSequence.Add("OnError"); return Task.CompletedTask; }); + + var services = new ServiceCollection() + .AddSaInMemoryFileStorage() + .AddSaHybridFileStorage(b => b.ConfigureInterceptors((sp, c) => c.AddUploadInterceptor(uploadInterceptor))); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act + var input = new UploadFileInput { FileName = "chain.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + await storage.UploadAsync(string.Empty, input, stream, _cts.Token); + + // Assert + Assert.True(canUploadCalled); + Assert.True(afterUploadCalled); + Assert.Contains("CanUpload", callSequence); + Assert.Contains("AfterUpload", callSequence); + Assert.DoesNotContain("OnError", callSequence); + } + + [Fact] + public async Task UploadInterceptor_CanUploadFalse_ReroutesToOtherStorage() + { + // Arrange — two storages with same basket, one blocked by interceptor + var uploadBlocked = false; + var interceptor = new CountingUploadInterceptor(() => uploadBlocked = true); + + var services = new ServiceCollection() + .AddSaFileSystemFileStorage(new FileSystemStorageSettings { BasePath = "interceptor_test", Basket = "shared_basket" }) + .AddSaInMemoryFileStorage(new InMemoryFileStorageOptions("shared_basket")) + .AddSaHybridFileStorage(b => b.ConfigureInterceptors((_, c) => + c.AddUploadInterceptor(interceptor))); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Act — upload blocked for filesystem by interceptor, should fall back to InMemory + var input = new UploadFileInput { FileName = "blocked.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + var result = await storage.UploadAsync("shared_basket", input, stream, _cts.Token); + + // Assert — should succeed in InMemory (filesystem was blocked by interceptor) + Assert.NotNull(result); + Assert.Equal(InMemoryFileStorage.DefaultStorageType, result.StorageType); + Assert.True(uploadBlocked); + + try { Directory.Delete("interceptor_test", true); } catch { /* ignore */ } + } + + [Fact] + public async Task DeleteInterceptor_CanDeleteFalse_PreventsDeletion() + { + // Arrange — single storage, interceptor blocks delete + var canDeleteChecked = false; + var interceptor = new CountingDeleteInterceptor( + () => canDeleteChecked = true); + + var services = new ServiceCollection() + .AddSaInMemoryFileStorage() + .AddSaHybridFileStorage(b => b.ConfigureInterceptors((sp, c) => c.AddDeleteInterceptor(interceptor))); + + using var provider = services.BuildServiceProvider(); + var storage = provider.GetRequiredService(); + + // Upload a file + var input = new UploadFileInput { FileName = "protected.txt", TenantId = 1 }; + using var stream = FixtureHelper.GetByteStream(); + var result = await storage.UploadAsync(string.Empty, input, stream, _cts.Token); + + // Act — delete intercepted: interceptor blocks the only storage → NoAvailableException + var ex = await Assert.ThrowsAsync(() => + storage.DeleteAsync(result.FileId, _cts.Token)); + + // Assert — interceptor was called + Assert.True(canDeleteChecked); + } +} + +// --- Helper interceptor implementations for testing --- + +internal sealed class BlockingDownloadInterceptor : IDownloadInterceptor +{ + public ValueTask CanDownloadAsync(IFileStorage storage, string fileId, Func loadStream, CancellationToken cancellationToken) + => ValueTask.FromResult(false); + + public ValueTask AfterDownloadAsync(IFileStorage storage, string fileId, bool success, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask OnDownloadErrorAsync(IFileStorage storage, string fileId, Exception exception, CancellationToken cancellationToken) + => ValueTask.CompletedTask; +} + +internal sealed class CountingDownloadInterceptor(Action onBlock) : IDownloadInterceptor +{ + public ValueTask CanDownloadAsync(IFileStorage storage, string fileId, Func loadStream, CancellationToken cancellationToken) + { + onBlock(); + return ValueTask.FromResult(false); + } + + public ValueTask AfterDownloadAsync(IFileStorage storage, string fileId, bool success, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask OnDownloadErrorAsync(IFileStorage storage, string fileId, Exception exception, CancellationToken cancellationToken) + => ValueTask.CompletedTask; +} + +internal sealed class NotifyDeleteInterceptor(Action onAfterDelete, Action onSuccess) : IDeleteInterceptor +{ + public ValueTask CanDeleteAsync(IFileStorage storage, string fileId, CancellationToken cancellationToken) + => ValueTask.FromResult(true); + + public ValueTask AfterDeleteAsync(IFileStorage storage, string fileId, bool success, CancellationToken cancellationToken) + { + onAfterDelete(); + onSuccess(success); + return ValueTask.CompletedTask; + } + + public ValueTask OnDeleteErrorAsync(IFileStorage storage, string fileId, Exception exception, CancellationToken cancellationToken) + => ValueTask.CompletedTask; +} + +internal sealed class CountingDeleteInterceptor(Action onCanDelete) : IDeleteInterceptor +{ + public ValueTask CanDeleteAsync(IFileStorage storage, string fileId, CancellationToken cancellationToken) + { + onCanDelete(); + return ValueTask.FromResult(false); // Always block + } + + public ValueTask AfterDeleteAsync(IFileStorage storage, string fileId, bool success, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask OnDeleteErrorAsync(IFileStorage storage, string fileId, Exception exception, CancellationToken cancellationToken) + => ValueTask.CompletedTask; +} + +internal sealed class OrderTrackingUploadInterceptor(Func onCan, Func onAfter, Func onError) : IUploadInterceptor +{ + public ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken cancellationToken) + { + onCan(); + return ValueTask.FromResult(true); + } + + public ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken cancellationToken) + { + onAfter(); + return ValueTask.CompletedTask; + } + + public ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken cancellationToken) + { + onError(); + return ValueTask.CompletedTask; + } +} + +internal sealed class BlockingUploadInterceptor(string blockedFileName) : IUploadInterceptor +{ + public ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken cancellationToken) + => ValueTask.FromResult(input.FileName != blockedFileName); + + public ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken cancellationToken) + => ValueTask.CompletedTask; +} + +internal sealed class CountingUploadInterceptor(Action onBlock) : IUploadInterceptor +{ + public ValueTask CanUploadAsync(IFileStorage storage, UploadFileInput input, Stream fileStream, CancellationToken cancellationToken) + { + // Block uploads to filesystem storage + if (storage.StorageType == "fs") + { + onBlock(); + return ValueTask.FromResult(false); + } + return ValueTask.FromResult(true); + } + + public ValueTask AfterUploadAsync(IFileStorage storage, StorageResult result, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask OnUploadErrorAsync(IFileStorage storage, Exception exception, CancellationToken cancellationToken) + => ValueTask.CompletedTask; +} From e40491d0887f08d1f0292c065b3f3335b76c8966 Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 1 Jul 2026 22:08:04 +0300 Subject: [PATCH 30/33] Virtual Folders (Baskets) Concept Signed-off-by: dundich --- src/Sa.HybridFileStorage/Readme-ru.md | 59 +++++++++++++++++++++++++++ src/Sa.HybridFileStorage/Readme.md | 59 +++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/src/Sa.HybridFileStorage/Readme-ru.md b/src/Sa.HybridFileStorage/Readme-ru.md index f08a71ff..88631da3 100644 --- a/src/Sa.HybridFileStorage/Readme-ru.md +++ b/src/Sa.HybridFileStorage/Readme-ru.md @@ -6,6 +6,7 @@ ## Содержание +- [Концепция виртуальных папок (корзин)](#концепция-виртуальных-папок-корзин) - [Поддерживаемые провайдеры](#поддерживаемые-провайдеры) - [Ключевые возможности](#ключевые-возможности) - [Формат File ID](#формат-file-id) @@ -28,6 +29,64 @@ --- +## Концепция виртуальных папок (корзин) + +**Sa.HybridFileStorage** работает с концепцией **виртуальных папок** — они называются **корзины (baskets)**. Корзина — это логический строковый контейнер, лежащий поверх физических бэкендов хранения. С точки зрения приложения вы работаете с простыми именами папок: `"черновик"`, `"документы"`, `"архив"`. За каждым именем корзины скрывается провайдер хранения (или список провайдеров), организующих данные по-разному. + +### Как корзины маппятся на хранилища + +Каждая корзина поддерживается одним или несколькими провайдерами `IFileStorage`. Одно и то же имя корзины может обслуживаться разными физическими системами, а несколько провайдеров можно комбинировать для отказоустойчивости или многоуровневого хранения: + +| Имя корзины | Физический бэкенд | Организация данных | Пример File ID | +|-------------|-------------------|--------------------|----------------| +| `черновик` | **Файловая система** (`fs://`) | Обычная древовидная структура на диске | `fs://черновик/42/заметки.txt` | +| `документы` | **PostgreSQL** (`pg://`) | Реляционная таблица с дата-партиционированием | `pg://документы/42/1751347200/договор.pdf` | +| `архив` | **S3 / MinIO** (`s3://`) | Облачный бакет с плоским пространством имён | `s3://архив/42/старый-отчёт.zip` | + +За каждой корзиной может скрываться **одно хранилище или список хранилищ** — гибридный слой прозрачно управляет failover. Вам не нужно знать, какая физическая система хранит ваш файл — вы ссылаетесь на него только через File ID. + +### Настройка маппинга корзин на бэкенды + +Регистрируйте каждое соответствие корзина → провайдер явно. Один провайдер привязан ровно к одной корзине: + +```csharp +builder.Services.AddSaHybridFileStorage(cfg => cfg + // Корзина "черновик" → файловая система + .ConfigureStorage((sp, c) => c.AddStorage(new FileSystemStorage( + new FileSystemStorageSettings { BasePath = @"C:\data\черновик", Basket = "черновик" }))) + + // Корзина "документы" → PostgreSQL с авто-партиционированием + .ConfigureStorage((sp, c) => c.AddStorage(new PostgresFileStorage(dataSource, new PostgresFileStorageOptions + { + PartOptions = new() { Basket = "документы" }, + StorageOptions = new() { SchemaName = "files", TableName = "files" } + }))) + + // Корзина "архив" → S3 облачное хранилище + .ConfigureStorage((sp, c) => c.AddStorage(new S3FileStorage(s3Client, new S3FileStorageOptions + { + Endpoint = "http://minio:9000", + Bucket = "company-archive", + Basket = "архив" + })))); +``` + +После настройки все CRUD-операции работают с именами корзин, а не спецификой провайдеров: + +```csharp +// Загрузка в виртуальную папку "черновик" — автоматически уходит на файловую систему +var result = await storage.UploadAsync("черновик", input, stream, ct); +// File ID: fs://черновик/42/мои-заметки.txt + +// Скачивание из "документы" — маршрутизируется в PostgreSQL прозрачно +await storage.DownloadAsync(result.FileId, processStream, ct); + +// Копирование из "черновик" в "архив" — беспрепятственно пересекает границу FS → S3 +await storage.CopyToBasketAsync(result.FileId, "архив", ct); +``` + +--- + ## Поддерживаемые провайдеры | Провайдер | Пакет | Класс | Сценарий использования | diff --git a/src/Sa.HybridFileStorage/Readme.md b/src/Sa.HybridFileStorage/Readme.md index 34a96985..4482f7f5 100644 --- a/src/Sa.HybridFileStorage/Readme.md +++ b/src/Sa.HybridFileStorage/Readme.md @@ -6,6 +6,7 @@ Hybrid file storage abstraction with automatic provider failover. Unifies multip ## Table of Contents +- [Virtual Folders (Baskets) Concept](#virtual-folders-baskets-concept) - [Supported Storage Providers](#supported-storage-providers) - [Key Features](#key-features) - [File ID Format](#file-id-format) @@ -28,6 +29,64 @@ Hybrid file storage abstraction with automatic provider failover. Unifies multip --- +## Virtual Folders (Baskets) Concept + +**Sa.HybridFileStorage** operates on the idea of **virtual folders** — called **baskets**. A basket is a logical, string-named scope that sits above physical storage backends. From your application's perspective, you work with simple folder names like `"drafts"`, `"documents"`, or `"archive"`. Behind each basket name lies a storage provider (or a list of providers) that organise data differently. + +### How baskets map to storage + +Each basket is backed by one or more `IFileStorage` providers. The same basket name can be served by different physical systems, and you can combine multiple providers for redundancy or tiered storage: + +| Basket name | Physical backend | Organisation | Example File ID | +|-------------|-------------------|--------------|-----------------| +| `drafts` | **FileSystem** (`fs://`) | Plain directory tree on disk | `fs://drafts/42/notes.txt` | +| `documents` | **PostgreSQL** (`pg://`) | Relational table with date partitioning | `pg://documents/42/1751347200/contract.pdf` | +| `archive` | **S3 / MinIO** (`s3://`) | Cloud bucket with flat namespace | `s3://archive/42/old-report.zip` | + +Behind every basket may hide **a single storage or a list of stores** — the hybrid layer manages failover transparently. You never need to know which physical system holds your file; you only reference it by its File ID. + +### Configuring baskets to backends + +Register each basket → provider mapping explicitly. One provider binds to exactly one basket: + +```csharp +builder.Services.AddSaHybridFileStorage(cfg => cfg + // Basket "drafts" → local filesystem + .ConfigureStorage((sp, c) => c.AddStorage(new FileSystemStorage( + new FileSystemStorageSettings { BasePath = @"C:\data\drafts", Basket = "drafts" }))) + + // Basket "documents" → PostgreSQL with auto-partitioning + .ConfigureStorage((sp, c) => c.AddStorage(new PostgresFileStorage(dataSource, new PostgresFileStorageOptions + { + PartOptions = new() { Basket = "documents" }, + StorageOptions = new() { SchemaName = "files", TableName = "files" } + }))) + + // Basket "archive" → S3 cloud storage + .ConfigureStorage((sp, c) => c.AddStorage(new S3FileStorage(s3Client, new S3FileStorageOptions + { + Endpoint = "http://minio:9000", + Bucket = "company-archive", + Basket = "archive" + })))); +``` + +Once configured, all CRUD operations use basket names — not provider specifics: + +```csharp +// Upload to the "drafts" virtual folder — goes to the filesystem automatically +var result = await storage.UploadAsync("drafts", input, stream, ct); +// File ID: fs://drafts/42/my-notes.txt + +// Download from "documents" — routed to PostgreSQL transparently +await storage.DownloadAsync(result.FileId, processStream, ct); + +// Copy from "drafts" to "archive" — crosses FS → S3 boundary seamlessly +await storage.CopyToBasketAsync(result.FileId, "archive", ct); +``` + +--- + ## Supported Storage Providers | Provider | Package | Class | Use Case | From f8060ac46bfb034327499cd9578399a98b4d60c6 Mon Sep 17 00:00:00 2001 From: dundich Date: Thu, 2 Jul 2026 12:53:33 +0300 Subject: [PATCH 31/33] docs: fix README inaccuracies and add full Sa API documentation --- README-ru.md | 13 +- README.md | 8 +- src/Sa/Readme.md | 352 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 365 insertions(+), 8 deletions(-) create mode 100644 src/Sa/Readme.md diff --git a/README-ru.md b/README-ru.md index 3bfdd6a4..fec05fa5 100644 --- a/README-ru.md +++ b/README-ru.md @@ -1,10 +1,19 @@ # Sa — Набор инфраструктурных библиотек для .NET 10 -Серия переиспользуемых .NET 10-библиотек, сфокусированных на инфраструктурных паттернах для распределённых систем. Целевая платформа — **.NET 10.0**, используется **Native AOT**, применяется паттерн **Central Package Management (CPM)** через `Directory.Packages.props`. +Нейрохерня - Серия переиспользуемых .NET 10-библиотек, сфокусированных на инфраструктурных паттернах для распределённых систем. Целевая платформа — **.NET 10.0**, используется **Native AOT**, применяется паттерн **Central Package Management (CPM)** через `Directory.Packages.props`. --- -### [Sa.Outbox.PostgreSql](src/Sa.Outbox.PostgreSql) +### [Sa](src/Sa) — Общие утилиты + +Ядро экосистемы **Sa**: базовые классы и методы расширения, линкуемые в другие пакеты через ``. Целевая платформа — **.NET 10.0**, совместимость с **Native AOT**, нулевые внешние зависимости. + + +Смотрите [полную документацию API](src/Sa/Readme.md). + +--- + +### [Sa.Outbox.PostgreSql](src/Sa.Outbox.PostgreSql) — Реализация Outbox на PostgreSQL Реализация паттерна **Transactional Outbox** на PostgreSQL для гарантированной доставки сообщений в распределённых системах. Предотвращает потерю сообщений и гарантирует обработку даже при сбоях. diff --git a/README.md b/README.md index dd5a2b02..5f0589ce 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,9 @@ Reusable infrastructure libraries for distributed .NET 10 systems — **Native A ### [Sa](src/Sa) — Shared Utilities -Common building blocks consumed by other packages via ``: +Core utility library consumed by other packages via ``. Targets **.NET 10.0**, **Native AOT compatible**, zero external dependencies. -- `LockRenewer` — automatic lock extension with configurable renewal interval -- `MurmurHash3` — compact hash for type identification and partitioning -- `Retry` — retry helpers with exponential backoff -- `ResetLazy` — lazily-evaluated, resettable cached value -- Extension methods: `DateTimeExtensions`, `EnumerableExtensions`, `ExceptionExtensions`, `SpanExtensions`, `StringExtensions`, `NumericExtensions`, `StrToExtensions`, `GuidExtensions` +See [full API reference](src/Sa/Readme.md). --- diff --git a/src/Sa/Readme.md b/src/Sa/Readme.md new file mode 100644 index 00000000..452f2faa --- /dev/null +++ b/src/Sa/Readme.md @@ -0,0 +1,352 @@ +# Sa — Shared Utilities + +Core utility library for the **Sa** ecosystem. Contains shared classes and extension methods consumed by other packages via ``. Targets **.NET 10.0**, **Native AOT compatible**. + +--- + +## Classes (namespace `Sa.Classes`) + +### LockRenewer + +Automatic lock extension with configurable renewal interval using `PeriodicTimer`. + +| Method | Description | +|--------|-------------| +| `KeepLocked(TimeSpan, Func, bool, CancellationToken)` | Runs a background task that periodically extends a lock. Returns `IAsyncDisposable` for clean shutdown. | +| `WaitForConditionAsync(Func>, TimeSpan, TimeSpan?, CancellationToken)` | Polls a predicate until it returns `true`, timeout expires, or cancellation is requested. | + +```csharp +var disposable = LockRenewer.KeepLocked( + lockExpiration: TimeSpan.FromSeconds(30), + extendLocked: ct => db.ExtendLockAsync(resourceId, ct), + blockImmediately: true); + +// ... later +await disposable.DisposeAsync(); +``` + +### MurmurHash3 + +Compact, fast hash function for type identification and partitioning. + +| Method | Description | +|--------|-------------| +| `Hash32(ReadOnlySpan, uint)` | Computes a 32-bit MurmurHash3 hash with seed. StackAlloc-friendly. | + +```csharp +var bytes = Encoding.UTF8.GetBytes("partition-key"); +uint hash = MurmurHash3.Hash32(bytes, seed: 0); +int partitionIndex = (int)(hash % numberOfPartitions); +``` + +### Retry + +Retry helpers with four strategies: **Constant**, **Linear**, **Exponential**, and **Jitter** (Azure-style decorrelated jitter). + +#### Strategies + +| Strategy | Parameters | Behavior | +|----------|------------|----------| +| `Constant` | `delay`, `fastFirst` | Fixed delay between retries. `fastFirst: true` skips initial delay. | +| `Linear` | `firstDelay`, `increment`, `maxDelay`, `count` | Delay increases linearly: `firstDelay + n * increment`, capped at `maxDelay`. | +| `Exponential` | `firstDelay`, `factor`, `maxDelay`, `count` | Delay doubles each attempt: `firstDelay * factor^n`, capped at `maxDelay`. | +| `Jitter` | `strategy`, `jitterSpan` | Applies random +/- jitter to any strategy's delay. | + +#### Execution + +| Method | Description | +|--------|-------------| +| `WaitAndRetry(IEnumerable, Func, CancellationToken)` | Executes a function with retry delays between attempts. Throws the last exception if cancelled during a delay after a failure. | + +```csharp +// Exponential backoff: 100ms → 200ms → 400ms → 800ms → 1600ms (max) +var strategy = Retry.Exponential(firstDelay: TimeSpan.FromMilliseconds(100), count: 5); +await Retry.WaitAndRetry(strategy, () => CallExternalApiAsync(), cancellationToken); +``` + +### ResetLazy\ + +Lazily-evaluated, resettable cached value with three thread-safety modes. + +| Property/Method | Description | +|-----------------|-------------| +| `Value` | Lazily initializes and returns the cached value. | +| `IsValueCreated` | `true` if the factory has been invoked. | +| `Load()` | Force initialization (no-op if already created). | +| `Reset()` | Clears the cache, optionally invoking a `valueReset` callback on the old value. | + +```csharp +var lazy = new ResetLazy(() => ConfigLoader.Load(), valueReset: cfg => cfg.Dispose()); +var config = lazy.Value; // first access triggers factory +lazy.Reset(); // clears cache, calls Dispose on old config +config = lazy.Value; // factory invoked again +``` + +### Levenshtein + +Damerau-Levenshtein distance algorithm for string similarity comparison. Optimized with stackalloc for minimal allocations. + +| Method | Description | +|--------|-------------| +| `Distance(string?, string?)` | Returns edit distance (0 = exact match). Null-safe. | +| `GetSimilarity(string?, string?)` | Returns similarity ratio 0.0–1.0 based on longest string. | +| `IsSimilar(string?, string?, double threshold)` | Checks if similarity ≥ threshold (default 0.8). | + +#### Levenshtein.Matcher + +Generic fuzzy matching over collections. + +| Method | Description | +|--------|-------------| +| `FindMatches(string?, IEnumerable, Func, double, bool)` | Yields all matches above `similarityThreshold`. Normalizes by default. | +| `FindBestMatch(...)` | Returns the single best match (highest similarity). | +| `FindBestMatch(IEnumerable>)` | Selects best from pre-filtered matches. | +| `FindBestMatch(string?, params string?[])` | Overload for plain-string candidate arrays. Returns `(bestMatch, distance)`. | + +```csharp +var best = Levenshtein.Matcher.FindBestMatch( + source: "recieve", + candidates: ["receive", "relief", "refuse"]); +// best.bestMatch == "receive", best.distance == 1 + +bool similar = Levenshtein.IsSimilar("hello", "hallo", threshold: 0.8); +// true +``` + +### MimeTypeMap + +Comprehensive MIME type lookup by file extension or filename (1000+ mappings sourced from Windows Registry + IANA). + +| Method | Description | +|--------|-------------| +| `TryGetMimeType(string, out string?)` | Looks up MIME type from filename or extension. Strips query strings automatically. | +| `GetMimeType(string)` | Same as `TryGetMimeType` but returns `"application/octet-stream"` on miss. | +| `GetExtension(string mimeType, bool throwErrorIfNotFound)` | Reverse lookup: MIME type → extension. | + +```csharp +string? mime; +if (MimeTypeMap.TryGetMimeType("document.pdf", out mime)) +{ + Console.WriteLine(mime); // "application/pdf" +} + +string ext = MimeTypeMap.GetExtension("image/png"); // ".png" +``` + +### ProcessExecutor / IProcessExecutor + +Asynchronous process executor with real-time output handling, stdin piping, and robust lifecycle management. + +| Method | Description | +|--------|-------------| +| `ExecuteAsync(ProcessStartInfo, Action?, Action?, TimeSpan?, CancellationToken)` | Real-time stdout/stderr callbacks. | +| `ExecuteWithResultAsync(...)` | Captures full output into `ProcessExecutionResult`. | +| `ExecuteStdOutAsync(...)` | Streams stdout to a callback while piping stdin and collecting stderr. | + +#### Public Types + +| Type | Description | +|------|-------------| +| `ProcessExecutionResult` | Record: `(int ExitCode, string StandardOutput, string StandardError)` | +| `ProcessExecutionException` | Thrown on non-zero exit code with `Exitcode` property. | +| `ProcessExecutionResultException` | Wraps `ProcessExecutionResult` as an exception. | +| `ProcessStartException` | Thrown when `Process.Start()` fails. | +| `ProcessTimeoutException` | Thrown on execution timeout. | + +```csharp +var result = await IProcessExecutor.Default.ExecuteWithResultAsync(new ProcessStartInfo +{ + FileName = "ffmpeg", + Arguments = "-i input.mp4 -vn -ab 128k output.mp3", + RedirectStandardOutput = true, + RedirectStandardError = true +}); + +if (result.ExitCode != 0) + Console.WriteLine(result.StandardError); +``` + +--- + +## Extensions (namespace `Sa.Extensions`) + +### DateTimeExtensions + +| Method | Description | +|--------|-------------| +| `ToUnixTimestamp(bool isInMilliseconds)` | Converts `DateTime` to Unix epoch seconds or milliseconds. Auto-converts non-UTC to UTC. | +| `StartOfDay()` | Returns `DateTimeOffset` at midnight with same offset. | +| `EndOfDay()` | Returns `DateTimeOffset` at the start of the next day (exclusive upper bound). | +| `StartOfMonth()` / `EndOfMonth()` | First/next-day-of-month boundaries. | +| `StartOfYear()` / `EndOfYear()` | First/next-year boundaries. | + +```csharp +var ts = DateTime.UtcNow.ToUnixTimestamp(); // seconds +var ms = someDate.ToUnixTimestamp(isInMilliseconds); // milliseconds +var today = dto.StartOfDay(); +``` + +### NumericExtensions + +| Method | Description | +|--------|-------------| +| `ToDateTimeFromUnixTimestamp(this uint)` | Unix timestamp → UTC `DateTime` (auto-detects seconds vs milliseconds). | +| `ToDateTimeFromUnixTimestamp(this long)` | Same for signed 64-bit. | +| `ToDateTimeFromUnixTimestamp(this ulong)` | Unsigned 64-bit. | +| `ToDateTimeFromUnixTimestamp(this double)` | Floating-point seconds, truncated to long. | +| `ToDateTimeFromUnixTimestamp(this string)` | Parses string → long → DateTime; returns `null` on parse failure. | +| Nullable overloads (`long?`, `ulong?`, `double?`) | Return `null` when input is `null`. | +| `ToDateTimeOffsetFromUnixTimestamp(this long)` | Timestamp → `DateTimeOffset`. | + +```csharp +DateTime dt = 1700000000L.ToDateTimeFromUnixTimestamp(); +DateTime? maybe = "1700000000".ToDateTimeFromUnixTimestamp(); // not null +``` + +### EnumerableExtensions + +| Method | Description | +|--------|-------------| +| `JoinByString(IEnumerable, string?)` | Joins elements using `string.Join`. Null-safe. | +| `JoinByString(IEnumerable, Func, string?)` | Maps then joins. Fast path uses `ICollection.Count` for pre-allocation. | +| `JoinByString(IEnumerable, Func, string?)` | Map-with-index then join. | + +```csharp +var csv = new[] { 1, 2, 3 }.JoinByString(", "); // "1, 2, 3" +var joined = items.JoinByString(x => x.Name, "|"); // "Name1|Name2|..." +``` + +### ExceptionExtensions + +| Method | Description | +|--------|-------------| +| `IsCritical(this Exception)` | Returns `true` for fatal CLR exceptions: `OutOfMemoryException`, `StackOverflowException`, `AppDomainUnloadedException`, `BadImageFormatException`, `CannotUnloadAppDomainException`, `InvalidProgramException`, `ThreadAbortException`. | +| `GetErrorMessages(this Exception)` | Concatenates all exception messages from root cause to outermost, one per line. | + +```csharp +if (ex.IsCritical()) Environment.FailFast(ex.Message); +Console.WriteLine(ex.GetErrorMessages()); +``` + +### SpanExtensions + +| Method | Description | +|--------|-------------| +| `GetChunks(Memory, int)` | Yields `Memory` chunks via iterator. | +| `GetChunksArray(Memory, int)` | Materialized `Memory[]` with pre-allocated capacity. | +| `SelectWhere(Span, Func, Func?)` | Combined Select+Where with index on `Span`. Returns trimmed array. | +| `SelectWhere(Span, Func, Func?)` | Same without index. | +| `SelectWhere(ReadOnlySpan, ...)` | Overloads for `ReadOnlySpan`. | + +```csharp +var chunks = someMemory.GetChunksArray(256); // Memory[] +var filtered = span.SelectWhere(x => x * 2, v => v > 10); +``` + +### StringExtensions + +| Method | Description | +|--------|-------------| +| `NullIfEmpty(this string?)` | Returns `null` if the string is null, empty, or whitespace-only. Otherwise returns the original. | +| `NormalizeWhiteSpace(bool isTrimmed)` | Collapses consecutive whitespace/separators/control chars into a single space. Zero-allocation fast path for clean strings. | +| `NormalizeWhiteSpaceSpan(ReadOnlySpan, Span, bool)` | Span-based zero-allocation variant. Writes into destination buffer, returns written length. | +| `GetMurmurHash3(uint seed)` | Computes MurmurHash3 of UTF-8 encoding without allocating a byte array. StackAlloc up to 512 bytes. | + +```csharp +string? cleaned = " hello world ".NormalizeWhiteSpace(); // "hello world" +uint hash = "key".GetMurmurHash3(seed: 42); +string? blank = " ".NullIfEmpty(); // null +``` + +### StrToExtensions + +Safe parsing extensions returning nullable result (`T?`) — never throw on bad input. + +| Method | Input Type | Returns | +|--------|-----------|---------| +| `StrToBool(string? / ReadOnlySpan)` | string / span | `bool?` | +| `StrToInt(string? / ReadOnlySpan)` | string / span | `int?` | +| `StrToShort(string? / ReadOnlySpan)` | string / span | `short?` | +| `StrToUShort(string? / ReadOnlySpan)` | string / span | `ushort?` | +| `StrToLong(string? / ReadOnlySpan)` | string / span | `long?` | +| `StrToULong(string? / ReadOnlySpan)` | string / span | `ulong?` | +| `StrToDouble(string? / ReadOnlySpan)` | string / span | `double?` | +| `StrToGuid(string? / ReadOnlySpan)` | string / span | `Guid?` | +| `StrToBytes(string, Encoding?)` | string | `byte[]` (UTF-8 by default) | +| `StrToEnum(string?, T defaultValue)` | string? | `T` (возвращает `defaultValue` при неудаче, case-insensitive) | +| `StrToDate(string? / ReadOnlySpan, IFormatProvider?, DateTimeStyles)` | string / span | `DateTime?` — tries ~60 date formats including ISO 8601 round-trip | + +```csharp +int? port = "8080".StrToInt(); // 8080 +Guid? id = "not-a-guid".StrToGuid(); // null +DateTime? dt = "2024-01-15".StrToDate(); // parsed or null +string? mode = "red".StrToEnum("black"); // "red" (case-insensitive) +``` + +### JsonExtensions + +| Method | Description | +|--------|-------------| +| `ToJson(T, JsonSerializerOptions?)` | Serializes to JSON string. | +| `FromJson(string, JsonSerializerOptions?)` | Deserializes from JSON string. | + +Both methods carry `[RequiresUnreferencedCode]` and `[RequiresDynamicCode]` attributes for AOT compatibility warnings. See `JsonHttpResultTrimmerWarning.SerializationUnreferencedCodeMessage` / `SerializationRequiresDynamicCodeMessage` for guidance. + +--- + +## Range & Section Types (namespace `Sa.Classes`) + +Interval types for defining retry delays, batch sizes, and other bounded ranges. + +### LimSection\ + +Closed interval `[min, max]`. + +| Member | Description | +|--------|-------------| +| Constructor `(T min, T max)` | Creates a closed interval. Throws if `min > max`. | +| `Min` / `Max` | Interval boundaries. | + +### HalfSection\ + +Half-open intervals: `OpenMin(min)` = `(min, ∞)` or `OpenMax(max)` = `(-∞, max]`. + +| Member | Description | +|--------|-------------| +| `Kind` | `OpenMin` or `OpenMax`. | +| `Bound` | The finite boundary value. | + +### Section\ + +Union type wrapping `LimSection` or `HalfSection`. Unified extension methods: + +| Extension | Description | +|-----------|-------------| +| `Contains(Range, T)` | Checks if value falls within the section. | +| `Expand(Section, T, T)` | Widens the section to include new bounds. | +| `Shrink(Section, T, T)` | Narrows the section. | +| `Center(Section)` | Midpoint of a `LimSection`. | +| `Width(LimSection)` | Distance between min and max. | +| `ApplyToBounds(Section, T, T)` | Clamps arbitrary bounds to the section. | +| `WithinBounds(Section, T, T)` | Checks if two values fit inside the section. | +| `Overlaps(Section, Section)` | Tests intersection between sections. | +| `MergeSections(Section, Section)` | Creates a section covering both inputs. | +| `GenerateValues(Section, int, Func>)` | Generates N values spanning the section (for LINQ `Range`). | + +```csharp +// Retry delays from 100ms to 5 seconds +Section delays = new LimSection( + TimeSpan.FromMilliseconds(100), + TimeSpan.FromSeconds(5)); + +bool inRange = delays.Contains(TimeSpan.FromSeconds(1)); // true +``` + +--- + +## Architecture Notes + +- All types are **internal** except where explicitly marked public (`ProcessExecutionResult`, `ProcessExecutionException`, etc.) +- Shared via `` pattern — linked into downstream projects, not referenced as NuGet packages +- Fully **Native AOT compatible** — no reflection-based serialization, no dynamic IL generation +- Zero-dependency: no external NuGet packages From ebbbb2a2d6021d54829867f3812ee3ccba9f36a5 Mon Sep 17 00:00:00 2001 From: dundich Date: Thu, 2 Jul 2026 12:59:06 +0300 Subject: [PATCH 32/33] refactor: Retry API overhaul, remove IArrayPool, add ConfigureAwait(false) --- src/Directory.Packages.props | 1 - src/Sa.Data.PostgreSql/PgRetryStrategy.cs | 10 +- .../Publication/OutboxMessagePublisher.cs | 8 +- src/Sa.Outbox/Sa.Outbox.csproj | 1 - src/Sa/Classes/IArrayPool.cs | 25 - src/Sa/Classes/LockRenewer.cs | 2 +- src/Sa/Classes/Retry.cs | 356 +++++++--- src/Sa/Extensions/StringExtensions.cs | 2 +- .../Configuration.Web.csproj | 4 - src/Samples/Configuration.Web/Program.cs | 10 +- src/Tests/SaTests/Classes/LockRenewerTests.cs | 2 +- src/Tests/SaTests/Classes/RetryTests.cs | 644 ++++++++++++++++-- 12 files changed, 834 insertions(+), 231 deletions(-) delete mode 100644 src/Sa/Classes/IArrayPool.cs diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 79e17c72..a7086e1f 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -23,6 +23,5 @@ - diff --git a/src/Sa.Data.PostgreSql/PgRetryStrategy.cs b/src/Sa.Data.PostgreSql/PgRetryStrategy.cs index 1e36b9ac..aeb2653c 100644 --- a/src/Sa.Data.PostgreSql/PgRetryStrategy.cs +++ b/src/Sa.Data.PostgreSql/PgRetryStrategy.cs @@ -7,17 +7,17 @@ public static class PgRetryStrategy public static ValueTask ExecuteWithRetry( Func> fun, int retryCount = 3, - int initialDelay = 530, + int medianFirstRetryDelay = 530, Func? next = null, CancellationToken cancellationToken = default) { return Classes.Retry.Jitter( fun: fun, retryCount: retryCount, - initialDelay: initialDelay - , next: (ex, i) => next != null + medianFirstRetryDelay: medianFirstRetryDelay, + shouldRetry: (ex, i) => next != null ? next(ex, i) - : (ex is NpgsqlException exception) && exception.IsTransient - , cancellationToken: cancellationToken); + : (ex is NpgsqlException exception) && exception.IsTransient, + cancellationToken: cancellationToken); } } diff --git a/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs b/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs index 66a7e985..d7029433 100644 --- a/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs +++ b/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs @@ -1,6 +1,6 @@ -using Sa.Classes; -using Sa.Outbox.Metadata; +using Sa.Outbox.Metadata; using Sa.Outbox.PlugServices; +using System.Buffers; namespace Sa.Outbox.Publication; @@ -40,7 +40,7 @@ private async ValueTask Send( ? maxBatchSize : messages.Count - start; - OutboxMessage[] payloads = DefaultArrayPool.Shared.Rent>(len); + OutboxMessage[] payloads = ArrayPool>.Shared.Rent(len); Span> payloadsSpan = payloads; try @@ -64,7 +64,7 @@ private async ValueTask Send( } finally { - DefaultArrayPool.Shared.Return(payloads); + ArrayPool>.Shared.Return(payloads); } start += len; diff --git a/src/Sa.Outbox/Sa.Outbox.csproj b/src/Sa.Outbox/Sa.Outbox.csproj index 848443ae..d8883459 100644 --- a/src/Sa.Outbox/Sa.Outbox.csproj +++ b/src/Sa.Outbox/Sa.Outbox.csproj @@ -10,7 +10,6 @@ - diff --git a/src/Sa/Classes/IArrayPool.cs b/src/Sa/Classes/IArrayPool.cs deleted file mode 100644 index 99c0881d..00000000 --- a/src/Sa/Classes/IArrayPool.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Buffers; - -namespace Sa.Classes; - -internal interface IArrayPool -{ - T[] Rent(int minimumLength); - - void Return(T[] array, bool clear = false); -} - -internal sealed class DefaultArrayPool : IArrayPool -{ - public static readonly IArrayPool Shared = new DefaultArrayPool(); - - public T[] Rent(int minimumLength) - { - return ArrayPool.Shared.Rent(minimumLength); - } - - public void Return(T[] array, bool clear = false) - { - ArrayPool.Shared.Return(array, clear); - } -} diff --git a/src/Sa/Classes/LockRenewer.cs b/src/Sa/Classes/LockRenewer.cs index a005c92c..f7b93108 100644 --- a/src/Sa/Classes/LockRenewer.cs +++ b/src/Sa/Classes/LockRenewer.cs @@ -46,7 +46,7 @@ public void Dispose() public async ValueTask DisposeAsync() { Timer.Dispose(); - await Task; + await Task.ConfigureAwait(false); } } diff --git a/src/Sa/Classes/Retry.cs b/src/Sa/Classes/Retry.cs index 7e98413c..6a3a1164 100644 --- a/src/Sa/Classes/Retry.cs +++ b/src/Sa/Classes/Retry.cs @@ -1,240 +1,360 @@ using System.Diagnostics; -using Sa.Extensions; namespace Sa.Classes; - +/// +/// Provides retry strategy helpers: constant, linear, exponential, and decorrelated jitter back-off. +/// Mirrors the Polly library patterns with a lightweight, allocation-minimal implementation targeting .NET 10 AOT. +/// internal static class Retry { + #region Strategy entry points + /// - /// For example: 500ms, 500ms, 500ms ... + /// Executes with constant (fixed-delay) retries ( variant). /// + /// + /// First retry has zero delay ("fast-first"). Subsequent retries use the fixed . + /// + /// + /// Delays: 0 ms (fast-first), 500 ms, 500 ms, 500 ms … + /// [DebuggerStepThrough] public static ValueTask Constant( Func> fun, I input, int retryCount = 3, int waitTime = 500, - Func? next = null, + Func? shouldRetry = null, CancellationToken cancellationToken = default) { - return Quartz.GenerateConstant(TimeSpan.FromMilliseconds(waitTime), retryCount, fastFirst: true) - .WaitAndRetry(fun, input, next, cancellationToken: cancellationToken); + return WaitAndRetry(Quartz.GenerateConstant(TimeSpan.FromMilliseconds(waitTime), retryCount, fastFirst: true), + fun, input, shouldRetry, cancellationToken); } + /// + /// Executes with constant (fixed-delay) retries (no input parameter). + /// + /// + /// First retry has zero delay ("fast-first"). Subsequent retries use the fixed . + /// + /// + /// Delays: 0 ms (fast-first), 500 ms, 500 ms, 500 ms … + /// [DebuggerStepThrough] public static ValueTask Constant( - Func> fun, - int retryCount = 3, - int waitTime = 500, - Func? next = null, - CancellationToken cancellationToken = default) + Func> fun, + int retryCount = 3, + int waitTime = 500, + Func? shouldRetry = null, + CancellationToken cancellationToken = default) { - return Quartz.GenerateConstant(TimeSpan.FromMilliseconds(waitTime), retryCount, fastFirst: true) - .WaitAndRetry(fun, next, cancellationToken: cancellationToken); + return WaitAndRetry(Quartz.GenerateConstant(TimeSpan.FromMilliseconds(waitTime), retryCount, fastFirst: true), + fun, shouldRetry, cancellationToken); } /// - /// For example: 100ms, 200ms, 400ms, 800ms, ... + /// Executes with exponential back-off retries ( variant). /// + /// + /// First retry has zero delay ("fast-first"). Subsequent delays grow as initialDelay × factorⁱ. + /// + /// + /// Delays: 0 ms (fast-first), 100 ms, 200 ms, 400 ms … + /// [DebuggerStepThrough] public static ValueTask Exponential( Func> fun, I input, int retryCount = 3, int initialDelay = 100, - Func? next = null, + double factor = 2.0, + Func? shouldRetry = null, CancellationToken cancellationToken = default) { - return Quartz.GenerateExponential(TimeSpan.FromMilliseconds(initialDelay), retryCount, fastFirst: true) - .WaitAndRetry(fun, input, next, cancellationToken: cancellationToken); + return WaitAndRetry(Quartz.GenerateExponential(TimeSpan.FromMilliseconds(initialDelay), retryCount, factor, fastFirst: true), + fun, input, shouldRetry, cancellationToken); } /// - /// For example: 100ms, 200ms, 400ms, 800ms, ... + /// Executes with exponential back-off retries (no input parameter). /// + /// + /// First retry has zero delay ("fast-first"). Subsequent delays grow as initialDelay × factorⁱ. + /// + /// + /// Delays: 0 ms (fast-first), 100 ms, 200 ms, 400 ms … + /// [DebuggerStepThrough] public static ValueTask Exponential( Func> fun, int retryCount = 3, int initialDelay = 100, - Func? next = null, + double factor = 2.0, + Func? shouldRetry = null, CancellationToken cancellationToken = default) { - return Quartz.GenerateExponential(TimeSpan.FromMilliseconds(initialDelay), retryCount, fastFirst: true) - .WaitAndRetry(fun, next, cancellationToken: cancellationToken); + return WaitAndRetry(Quartz.GenerateExponential(TimeSpan.FromMilliseconds(initialDelay), retryCount, factor, fastFirst: true), + fun, shouldRetry, cancellationToken); } /// - /// For example: 100ms, 200ms, 300ms, 400ms, .. + /// Executes with linear back-off retries ( variant). /// + /// + /// First retry has zero delay ("fast-first"). Subsequent delays grow linearly: initialDelay × factor × i. + /// + /// + /// Delays: 0 ms (fast-first), 100 ms, 200 ms, 300 ms … + /// [DebuggerStepThrough] public static ValueTask Linear( Func> fun, I input, int retryCount = 3, int initialDelay = 100, - Func? next = null, + double factor = 1.0, + Func? shouldRetry = null, CancellationToken cancellationToken = default) { - return Quartz.GenerateLinear(TimeSpan.FromMilliseconds(initialDelay), retryCount, fastFirst: true) - .WaitAndRetry(fun, input, next, cancellationToken); + return WaitAndRetry(Quartz.GenerateLinear(TimeSpan.FromMilliseconds(initialDelay), retryCount, factor, fastFirst: true), + fun, input, shouldRetry, cancellationToken); } - /// - /// For example: 100ms, 200ms, 300ms, 400ms, .. + /// Executes with linear back-off retries (no input parameter). /// + /// + /// First retry has zero delay ("fast-first"). Subsequent delays grow linearly: initialDelay × factor × i. + /// + /// + /// Delays: 0 ms (fast-first), 100 ms, 200 ms, 300 ms … + /// [DebuggerStepThrough] public static ValueTask Linear( Func> fun, int retryCount = 3, int initialDelay = 100, - Func? next = null, + double factor = 1.0, + Func? shouldRetry = null, CancellationToken cancellationToken = default) { - return Quartz.GenerateLinear(TimeSpan.FromMilliseconds(initialDelay), retryCount, fastFirst: true) - .WaitAndRetry(fun, next, cancellationToken); + return WaitAndRetry(Quartz.GenerateLinear(TimeSpan.FromMilliseconds(initialDelay), retryCount, factor, fastFirst: true), + fun, shouldRetry, cancellationToken); } - - /// - /// For example: 850ms, 1455ms, 3060ms. + /// Executes with Microsoft Azure-style decorrelated jitter retries ( variant). /// + /// + /// Each delay is uniformly sampled between 0 and 3× the median, avoiding thundering herd. + /// + /// + /// Delays: 0 ms (fast-first), ~530 ms, ~1455 ms, ~3060 ms … + /// [DebuggerStepThrough] public static ValueTask Jitter( Func> fun, I input, int retryCount = 3, - int initialDelay = 530, - Func? next = null, + int medianFirstRetryDelay = 530, + Func? shouldRetry = null, CancellationToken cancellationToken = default) { - return Quartz.GenerateJitter(TimeSpan.FromMilliseconds(initialDelay), retryCount, fastFirst: true) - .WaitAndRetry(fun, input, next, cancellationToken: cancellationToken); + return WaitAndRetry(Quartz.GenerateJitter(TimeSpan.FromMilliseconds(medianFirstRetryDelay), retryCount, fastFirst: true), + fun, input, shouldRetry, cancellationToken); } /// - /// For example: 850ms, 1455ms, 3060ms. + /// Executes with Microsoft Azure-style decorrelated jitter retries (no input parameter). /// + /// + /// Each delay is uniformly sampled between 0 and 3× the median, avoiding thundering herd. + /// + /// + /// Delays: 0 ms (fast-first), ~530 ms, ~1455 ms, ~3060 ms … + /// [DebuggerStepThrough] public static ValueTask Jitter( Func> fun, int retryCount = 3, - int initialDelay = 530, - Func? next = null, + int medianFirstRetryDelay = 530, + Func? shouldRetry = null, CancellationToken cancellationToken = default) { - return Quartz.GenerateJitter(TimeSpan.FromMilliseconds(initialDelay), retryCount, fastFirst: true) - .WaitAndRetry(fun, next, cancellationToken: cancellationToken); + return WaitAndRetry(Quartz.GenerateJitter(TimeSpan.FromMilliseconds(medianFirstRetryDelay), retryCount, fastFirst: true), + fun, shouldRetry, cancellationToken); } + #endregion + + #region WaitAndRetry core (shared by all strategies) + + /// + /// Retries by awaiting each delay in on failure. + /// + /// Sequence of delays produced by a Quartz generator. + /// + /// Returns to retry after the caught exception. When , + /// critical exceptions are re-thrown immediately while transient ones are retried. + /// [DebuggerStepThrough] +#pragma warning disable S3776 public static async ValueTask WaitAndRetry( - this IEnumerable timeSpans, +#pragma warning restore S3776 + IEnumerable timeSpans, Func> fun, I input, - Func? next = null, + Func? shouldRetry = null, CancellationToken cancellationToken = default) { TimeSpan[] points = [.. timeSpans]; + if (points.Length == 0) + return await fun(input, cancellationToken).ConfigureAwait(false); - for (int i = 0; i < points.Length - 1; i++) + Exception? lastEx = null; + for (int i = 0; i < points.Length; i++) { - if (cancellationToken.IsCancellationRequested) break; + if (cancellationToken.IsCancellationRequested) + break; try { - return await fun(input, cancellationToken); + return await fun(input, cancellationToken).ConfigureAwait(false); } - catch (Exception e) + catch (Exception e) when (!IsFatal(e)) { - if (e is TaskCanceledException) - { - break; - } - else if (e.IsCritical() || next != null && !next(e, i)) - { + if (e is OperationCanceledException oce && oce.CancellationToken == cancellationToken) throw; - } - await Wait(points[i], cancellationToken); + if (shouldRetry != null && !shouldRetry(e, i)) + throw; + + lastEx = e; + if (i < points.Length - 1) + { + await Delay(points[i], cancellationToken).ConfigureAwait(false); + } } } - if (points.Length > 0) - { - await Wait(points[^1], cancellationToken); - } + if (lastEx != null) + throw lastEx; - return await fun(input, cancellationToken); + throw new OperationCanceledException("Retry loop exited due to cancellation.", null, cancellationToken); } + /// + /// Retries by awaiting each delay in on failure (no input parameter). + /// [DebuggerStepThrough] +#pragma warning disable S3776 public static async ValueTask WaitAndRetry( - this IEnumerable timeSpans, +#pragma warning restore S3776 + IEnumerable timeSpans, Func> fun, - Func? next = null, + Func? shouldRetry = null, CancellationToken cancellationToken = default) { TimeSpan[] points = [.. timeSpans]; + if (points.Length == 0) + return await fun(cancellationToken).ConfigureAwait(false); - for (int i = 0; i < points.Length - 1; i++) + Exception? lastEx = null; + for (int i = 0; i < points.Length; i++) { - if (cancellationToken.IsCancellationRequested) break; + if (cancellationToken.IsCancellationRequested) + break; try { - return await fun(cancellationToken); + return await fun(cancellationToken).ConfigureAwait(false); } - catch (Exception e) + catch (Exception e) when (!IsFatal(e)) { - if (e is TaskCanceledException) - { - break; - } - else if (e.IsCritical() || next != null && !next(e, i)) - { + if (e is OperationCanceledException oce && oce.CancellationToken == cancellationToken) throw; - } - await Wait(points[i], cancellationToken); + if (shouldRetry != null && !shouldRetry(e, i)) + throw; + + lastEx = e; + if (i < points.Length - 1) + { + await Delay(points[i], cancellationToken).ConfigureAwait(false); + } } } - if (points.Length > 0) - { - await Wait(points[^1], cancellationToken); - } + if (lastEx != null) + throw lastEx; - return await fun(cancellationToken); + throw new OperationCanceledException("Retry loop exited due to cancellation.", null, cancellationToken); } - private static async Task Wait(TimeSpan delay, CancellationToken cancellationToken) + #endregion + + #region Private helpers + + [DebuggerStepThrough] + private static async Task Delay(TimeSpan delay, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) return; try { - await Task.Delay(delay, cancellationToken); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); } catch (TaskCanceledException) { - // ignore + // Ignored — cancellation already signalled. } } + #endregion + + #region Critical-exception detection + + /// + /// Determines whether is a critical (non-retriable) exception. + /// + [DebuggerStepThrough] + public static bool IsFatal(Exception ex) + { + return ex is OutOfMemoryException + or StackOverflowException + or AccessViolationException + or AppDomainUnloadedException + or BadImageFormatException + or CannotUnloadAppDomainException + or InvalidProgramException + or ThreadAbortException; + } + + #endregion + #region Quartz generator (delay sequence factories) + + /// + /// Generates sequences for various retry back-off strategies. + /// The first element is always when is , + /// enabling an immediate second attempt with zero wait. + /// public static class Quartz { private static IEnumerable Empty() => []; + /// + /// Validates common parameters for delay generators. + /// private static void ValidateParameters(TimeSpan delay, int retryCount, string delayParamName) { - if (delay < TimeSpan.Zero) throw new ArgumentOutOfRangeException(delayParamName, delay, "should be >= 0ms"); - if (retryCount < 0) throw new ArgumentOutOfRangeException(nameof(retryCount), retryCount, "should be >= 0"); + if (delay < TimeSpan.Zero) throw new ArgumentOutOfRangeException(delayParamName, delay, "Delay must be ≥ 0."); + if (retryCount < 0) throw new ArgumentOutOfRangeException(nameof(retryCount), retryCount, "retryCount must be ≥ 0."); } + /// + /// Generates a constant-delay sequence: 0, D, D, D, … (fastFirst=True) or D, D, D, … (fastFirst=False). + /// [DebuggerStepThrough] public static IEnumerable GenerateConstant(TimeSpan delay, int retryCount, bool fastFirst = false) { @@ -242,26 +362,34 @@ public static IEnumerable GenerateConstant(TimeSpan delay, int retryCo return retryCount == 0 ? Empty() : Generator.GenConstant(delay, retryCount, fastFirst); } + /// + /// Generates a linear-back-off sequence: 0, I, 2I, 3I, … (fastFirst=True) where I = initialDelay × factor. + /// [DebuggerStepThrough] public static IEnumerable GenerateLinear( TimeSpan initialDelay, int retryCount, double factor = 1.0, bool fastFirst = true) { ValidateParameters(initialDelay, retryCount, nameof(initialDelay)); - if (factor < 0) throw new ArgumentOutOfRangeException(nameof(factor), factor, "should be >= 0"); - + if (factor < 0) throw new ArgumentOutOfRangeException(nameof(factor), factor, "Factor must be ≥ 0."); return retryCount == 0 ? Empty() : Generator.GenLinear(initialDelay, retryCount, factor, fastFirst); } + /// + /// Generates an exponential-back-off sequence: 0, I, I·F, I·F², … (fastFirst=True) where F = factor. + /// [DebuggerStepThrough] public static IEnumerable GenerateExponential( TimeSpan initialDelay, int retryCount, double factor = 2.0, bool fastFirst = true) { ValidateParameters(initialDelay, retryCount, nameof(initialDelay)); - if (factor < 1.0) throw new ArgumentOutOfRangeException(nameof(factor), factor, "should be >= 1.0"); - + if (factor < 1.0) throw new ArgumentOutOfRangeException(nameof(factor), factor, "Factor must be ≥ 1.0."); return retryCount == 0 ? Empty() : Generator.GenExponential(initialDelay, retryCount, factor, fastFirst); } + /// + /// Generates an Azure-style decorrelated-jitter sequence using the intrinsic-value formula. + /// Each delay is sampled uniformly between 0 and 3× the median to prevent thundering herd. + /// [DebuggerStepThrough] public static IEnumerable GenerateJitter( TimeSpan medianFirstRetryDelay, int retryCount, bool fastFirst = true) @@ -270,81 +398,87 @@ public static IEnumerable GenerateJitter( return retryCount == 0 ? Empty() : Generator.GenJitter(medianFirstRetryDelay, retryCount, fastFirst); } + #region Generator implementations static class Generator { - public static IEnumerable GenConstant(TimeSpan delay, int retryCount, bool fastFirst) { if (fastFirst) - { yield return TimeSpan.Zero; - } for (int i = fastFirst ? 1 : 0; i < retryCount; i++) - { yield return delay; - } } public static IEnumerable GenLinear( TimeSpan initialDelay, int retryCount, double factor, bool fastFirst) { if (fastFirst) - { yield return TimeSpan.Zero; - } double ms = initialDelay.TotalMilliseconds; double increment = factor * ms; for (int i = fastFirst ? 1 : 0; i < retryCount; i++, ms += increment) - { yield return TimeSpan.FromMilliseconds(ms); - } } public static IEnumerable GenExponential( TimeSpan initialDelay, int retryCount, double factor, bool fastFirst) { if (fastFirst) - { yield return TimeSpan.Zero; - } double ms = initialDelay.TotalMilliseconds; for (int i = fastFirst ? 1 : 0; i < retryCount; i++, ms *= factor) - { yield return TimeSpan.FromMilliseconds(ms); - } } + /// + /// Implements Microsoft's "decorrelated jitter" algorithm from AWS Architecture Blog. + /// See: https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + /// public static IEnumerable GenJitter(TimeSpan medianFirstRetryDelay, int retryCount, bool fastFirst) { - const double pFactor = 4.0; - const double rpScalingFactor = 1 / 1.4d; - double maxTimeSpanDouble = (double)TimeSpan.MaxValue.Ticks - 1000; - if (fastFirst) - { yield return TimeSpan.Zero; - } - long targetTicksFirstDelay = medianFirstRetryDelay.Ticks; + long targetTicks = medianFirstRetryDelay.Ticks; double prev = 0.0; for (int i = fastFirst ? 1 : 0; i < retryCount; i++) { + // Intrinsic-value method: t = i + U(0,1), then transform. double t = i + Random.Shared.NextDouble(); - double next = Math.Pow(2, t) * Math.Tanh(Math.Sqrt(pFactor * t)); + double next = Math.Pow(2, t) * Math.Tanh(Math.Sqrt(JitterConstants.PFactor * t)); double formulaIntrinsicValue = next - prev; yield return TimeSpan.FromTicks( - (long)Math.Min(formulaIntrinsicValue * rpScalingFactor * targetTicksFirstDelay, maxTimeSpanDouble)); + (long)Math.Min( + formulaIntrinsicValue * JitterConstants.RpScalingFactor * targetTicks, + JitterConstants.MaxTicks)); + prev = next; } } + + /// + /// Constants for the decorrelated jitter algorithm (AWS blog reference). + /// PFactor = 4.0 controls the shape of the distribution. + /// RpScalingFactor ≈ 0.714 derives from the integral normalization. + /// + static class JitterConstants + { + public const double PFactor = 4.0; + public const double RpScalingFactor = 1.0 / 1.4; + public static readonly long MaxTicks = TimeSpan.MaxValue.Ticks - 1000; + } } + + #endregion } + + #endregion } diff --git a/src/Sa/Extensions/StringExtensions.cs b/src/Sa/Extensions/StringExtensions.cs index 7c333be7..fe6bfe48 100644 --- a/src/Sa/Extensions/StringExtensions.cs +++ b/src/Sa/Extensions/StringExtensions.cs @@ -53,7 +53,7 @@ public static string NormalizeWhiteSpace(this string? str, bool isTrimmed = true // No whitespace found — just trim and return return isTrimmed ? str.Trim() : str; } - + // Slow path: span-based normalization — allocates one new string, avoids StringBuilder heap churn Span dest = stackalloc char[len]; int w = 0; diff --git a/src/Samples/Configuration.Web/Configuration.Web.csproj b/src/Samples/Configuration.Web/Configuration.Web.csproj index 57d3ba1f..bdfe44bf 100644 --- a/src/Samples/Configuration.Web/Configuration.Web.csproj +++ b/src/Samples/Configuration.Web/Configuration.Web.csproj @@ -8,10 +8,6 @@ true - - - - diff --git a/src/Samples/Configuration.Web/Program.cs b/src/Samples/Configuration.Web/Program.cs index fd2c4f5d..8d96d49f 100644 --- a/src/Samples/Configuration.Web/Program.cs +++ b/src/Samples/Configuration.Web/Program.cs @@ -41,16 +41,9 @@ INSERT INTO settings (key, value) options.SerializerOptions.TypeInfoResolverChain.Insert(0, AppJsonSerializerContext.Default); }); -// Learn more about configuring OpenAPI at https://aka.ms/aspnet/openapi -builder.Services.AddOpenApi(); var app = builder.Build(); -if (app.Environment.IsDevelopment()) -{ - app.MapOpenApi(); -} - var todosApi = app.MapGroup("/settings"); @@ -65,10 +58,13 @@ INSERT INTO settings (key, value) app.Run(); +#pragma warning disable S3903 public sealed record Settings(string Key, string? Value); + [JsonSerializable(typeof(Settings[]))] internal partial class AppJsonSerializerContext : JsonSerializerContext { } +#pragma warning restore S3903 diff --git a/src/Tests/SaTests/Classes/LockRenewerTests.cs b/src/Tests/SaTests/Classes/LockRenewerTests.cs index 05653322..d17017b6 100644 --- a/src/Tests/SaTests/Classes/LockRenewerTests.cs +++ b/src/Tests/SaTests/Classes/LockRenewerTests.cs @@ -194,7 +194,7 @@ Task Predicate(CancellationToken ct) for (int i = 1; i < callTimes.Count; i++) { var interval = (callTimes[i] - callTimes[i - 1]).TotalMilliseconds; - Assert.InRange(interval, 80, 150); // Allow some jitter due to scheduling + Assert.InRange(interval, 70, 150); // Allow some jitter due to scheduling } } diff --git a/src/Tests/SaTests/Classes/RetryTests.cs b/src/Tests/SaTests/Classes/RetryTests.cs index 9997b32d..b38a4816 100644 --- a/src/Tests/SaTests/Classes/RetryTests.cs +++ b/src/Tests/SaTests/Classes/RetryTests.cs @@ -2,145 +2,649 @@ namespace SaTests.Classes; +/// +/// Thrown by test stubs to simulate transient/retriable failures without pulling in external deps. +/// +public class TransientException : Exception +{ + public TransientException() : base() + { + } + + public TransientException(string message) : base(message) + { + } +} + +/// +/// Thrown by test stubs to simulate non-transient (terminal) failures. +/// +public class NonTransientException(string message) : Exception(message); public class RetryTests { + #region Constant strategy + [Fact] - public async Task Constant_Retry_Succeeds_After_2_Attempts() + public async Task Constant_SuccessOnThirdAttempt_ReturnsInput() { // Arrange - int attemptCount = 0; - ValueTask func(int input, CancellationToken token) + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) { - attemptCount++; - if (attemptCount < 3) - { - throw new Exception("Simulated failure"); - } - return new ValueTask(input); + attempts++; + if (attempts < 3) + throw new TransientException("fail"); + return new(input); } // Act - int result = await Retry.Constant(func, 42, retryCount: 3, waitTime: 10, cancellationToken: TestContext.Current.CancellationToken); + int result = await Retry.Constant( + Func, + 42, + retryCount: 3, + waitTime: 1, + cancellationToken: TestContext.Current.CancellationToken); // Assert Assert.Equal(42, result); - Assert.Equal(3, attemptCount); + Assert.Equal(3, attempts); + } + + [Fact] + public async Task Constant_ExhaustRetries_ThrowsLastException() + { + // Arrange + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) + { + attempts++; + throw new TransientException("always fails"); + } + + // Act & Assert + var ex = await Assert.ThrowsAsync(() => + Retry.Constant(Func, 42, retryCount: 3, waitTime: 1, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + + Assert.Equal("always fails", ex.Message); + Assert.Equal(3, attempts); + } + + [Theory] + [InlineData(0)] // zero retries → single call, no delay + [InlineData(1)] // one retry + [InlineData(5)] // many retries + public async Task Constant_VaryingRetryCounts_CallCountMatches(int retryCount) + { + // Arrange + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) + { + attempts++; + return new(input); + } + + // Act + int result = await Retry.Constant( + Func, + 99, + retryCount: retryCount, + waitTime: 1, + cancellationToken: TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(99, result); + Assert.Equal(1, attempts); // succeeds on first try regardless of retryCount } [Fact] - public async Task Exponential_Retry_Succeeds_After_2_Attempts() + public async Task Constant_NoInput_SuccessOnFirstCall() { // Arrange - int attemptCount = 0; - ValueTask func(int input, CancellationToken token) + int attempts = 0; + ValueTask Func(CancellationToken ct) { - attemptCount++; - if (attemptCount < 3) - { - throw new Exception("Simulated failure"); - } - return new ValueTask(input); + attempts++; + return new(777); } // Act - int result = await Retry.Exponential(func, 42, retryCount: 3, initialDelay: 10, cancellationToken: TestContext.Current.CancellationToken); + int result = await Retry.Constant( + Func, + retryCount: 3, + waitTime: 1, + cancellationToken: TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(777, result); + Assert.Equal(1, attempts); + } + + [Fact] + public async Task Constant_NoInput_ExhaustsRetries() + { + // Arrange + int attempts = 0; + ValueTask Func(CancellationToken ct) + { + attempts++; + throw new TransientException("no-input fail"); + } + + // Act & Assert + var _ = await Assert.ThrowsAsync(() => + Retry.Constant(Func, retryCount: 2, waitTime: 1, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + + Assert.Equal(2, attempts); + } + + #endregion + + #region Linear strategy + + [Theory] + [InlineData(3)] + [InlineData(5)] + public async Task Linear_SuccessAfterFailures_ReturnsInput(int retryCount) + { + // Arrange + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) + { + attempts++; + if (attempts == retryCount) + return new(input); + throw new TransientException("fail"); + } + + // Act + int result = await Retry.Linear( + Func, + 42, + retryCount: retryCount, + initialDelay: 10, + factor: 1.0, + cancellationToken: TestContext.Current.CancellationToken); // Assert Assert.Equal(42, result); - Assert.Equal(3, attemptCount); + Assert.Equal(retryCount, attempts); } [Fact] - public async Task Linear_Retry_Succeeds_After_2_Attempts() + public async Task Linear_NoInput_ThrowsAfterExhaustion() + { + // Arrange + int attempts = 0; + ValueTask Func(CancellationToken ct) + { + attempts++; + throw new TransientException("linear fail"); + } + + // Act & Assert + await Assert.ThrowsAsync(() => + Retry.Linear(Func, retryCount: 3, initialDelay: 10, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + + Assert.Equal(3, attempts); + } + + #endregion + + #region Exponential strategy + + [Theory] + [InlineData(3, 2.0)] + [InlineData(4, 1.5)] + public async Task Exponential_SuccessAfterFailures_ReturnsInput(int retryCount, double factor) { // Arrange - int attemptCount = 0; - ValueTask func(int input, CancellationToken token) + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) { - attemptCount++; - if (attemptCount < 3) - { - throw new Exception("Simulated failure"); - } - return new ValueTask(input); + attempts++; + if (attempts == retryCount) + return new(input); + throw new TransientException("fail"); } // Act - int result = await Retry.Linear(func, 42, retryCount: 3, initialDelay: 10, cancellationToken: TestContext.Current.CancellationToken); + int result = await Retry.Exponential( + Func, + 42, + retryCount: retryCount, + initialDelay: 10, + factor: factor, + cancellationToken: TestContext.Current.CancellationToken); // Assert Assert.Equal(42, result); - Assert.Equal(3, attemptCount); + Assert.Equal(retryCount, attempts); } [Fact] - public async Task DecorrelatedJitter_Retry_Succeeds_After_2_Attempts() + public async Task Exponential_NoInput_ThrowsAfterExhaustion() + { + // Arrange + int attempts = 0; + ValueTask Func(CancellationToken ct) + { + attempts++; + throw new TransientException("exp fail"); + } + + // Act & Assert + await Assert.ThrowsAsync(() => + Retry.Exponential(Func, retryCount: 2, initialDelay: 10, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + + Assert.Equal(2, attempts); + } + + #endregion + + #region Jitter strategy + + [Theory] + [InlineData(3)] + [InlineData(5)] + public async Task Jitter_SuccessAfterFailures_ReturnsInput(int retryCount) { // Arrange - int attemptCount = 0; - ValueTask func(int input, CancellationToken token) + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) { - attemptCount++; - if (attemptCount < 3) - { - throw new Exception("Simulated failure"); - } - return new ValueTask(input); + attempts++; + if (attempts == retryCount) + return new(input); + throw new TransientException("fail"); } // Act - int result = await Retry.Jitter(func, 42, retryCount: 3, initialDelay: 10, cancellationToken: TestContext.Current.CancellationToken); + int result = await Retry.Jitter( + Func, + 42, + retryCount: retryCount, + medianFirstRetryDelay: 10, + cancellationToken: TestContext.Current.CancellationToken); // Assert Assert.Equal(42, result); - Assert.Equal(3, attemptCount); + Assert.Equal(retryCount, attempts); } [Fact] - public async Task Retry_Throws_Original_Exception_After_Max_Retries() + public async Task Jitter_NoInput_ThrowsAfterExhaustion() { // Arrange - int attemptCount = 0; - ValueTask func(int input, CancellationToken token) + int attempts = 0; + ValueTask Func(CancellationToken ct) { - attemptCount++; - throw new Exception("Simulated failure"); + attempts++; + throw new TransientException("jitter fail"); } - // Act and Assert - await Assert.ThrowsAsync(() => Retry.Constant(func, 42, retryCount: 3, waitTime: 10, cancellationToken: TestContext.Current.CancellationToken).AsTask()); - Assert.Equal(3, attemptCount); + // Act & Assert + await Assert.ThrowsAsync(() => + Retry.Jitter(Func, retryCount: 3, medianFirstRetryDelay: 10, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + + Assert.Equal(3, attempts); } + #endregion + + #region shouldRetry predicate + [Fact] - public async Task Retry_Cancels_After_CancellationToken_Is_Cancelled() + public async Task Constant_shouldRetry_True_ResumesUntilSuccess() { // Arrange - int attemptCount = 0; - async ValueTask func(int input, CancellationToken token) + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) { - if (token.IsCancellationRequested) - { - return 111; - } + attempts++; + if (attempts == 3) + return new(input); + throw new TransientException("transient"); + } + + // Act + int result = await Retry.Constant( + Func, + 42, + retryCount: 5, + waitTime: 1, + shouldRetry: (ex, _) => ex is TransientException, + cancellationToken: TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(42, result); + Assert.Equal(3, attempts); + } - attemptCount++; - await Task.Delay(100, CancellationToken.None); - throw new Exception("Simulated failure"); + [Fact] + public async Task Constant_shouldRetry_False_StopsImmediately() + { + // Arrange + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) + { + attempts++; + throw new NonTransientException("terminal"); } - using CancellationTokenSource cts = new(); + // Act & Assert + var ex = await Assert.ThrowsAsync(() => + Retry.Constant( + Func, + 42, + retryCount: 5, + waitTime: 1, + shouldRetry: (ex, _) => ex is TransientException, + cancellationToken: TestContext.Current.CancellationToken).AsTask()); - _ = Task.Run(async () => + Assert.Equal("terminal", ex.Message); + Assert.Equal(1, attempts); + } + + [Fact] + public async Task WaitAndRetry_shouldRetry_False_StopsImmediately() + { + // Arrange + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) { - await Task.Delay(200, TestContext.Current.CancellationToken); - await cts.CancelAsync(); - }, TestContext.Current.CancellationToken); + attempts++; + throw new NonTransientException("terminal"); + } - int result = await Retry.Constant(func, 42, retryCount: 3, waitTime: 10, cancellationToken: cts.Token); + // Act & Assert + var ex = await Assert.ThrowsAsync(() => + Retry.WaitAndRetry( + [], + Func, + 42, + shouldRetry: (ex, _) => ex is TransientException, + cancellationToken: TestContext.Current.CancellationToken).AsTask()); - Assert.True(attemptCount > 0); - Assert.Equal(111, result); + Assert.Equal("terminal", ex.Message); + Assert.Equal(1, attempts); } + + #endregion + + #region Cancellation + + [Fact] + public async Task Constant_PreCancelledToken_ThrowsOCEWithoutCallingFunc() + { + // Arrange + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) + { + attempts++; + return new(input); + } + + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + + // Act & Assert + var _ = await Assert.ThrowsAsync(() => + Retry.Constant(Func, 42, retryCount: 3, waitTime: 1, cancellationToken: cts.Token).AsTask()); + + Assert.Equal(0, attempts); + } + + [Fact] + public async Task Constant_DuringWait_CancelsAndThrowsLastException() + { + // Arrange + int attempts = 0; + var gate = new TaskCompletionSource(); + ValueTask Func(int input, CancellationToken ct) + { + attempts++; + if (attempts == 1) + gate.SetResult(); + throw new TransientException("always fails"); + } + + using var cts = new CancellationTokenSource(); + var task = Retry.Constant(Func, 42, retryCount: 3, waitTime: 500, cancellationToken: cts.Token); + + await gate.Task; + cts.Cancel(); + + // Act & Assert + var ex = await Assert.ThrowsAsync(() => task.AsTask()); + Assert.Equal("always fails", ex.Message); + Assert.True(attempts > 1, $"Expected more than 1 attempt after cancellation, got {attempts}"); + } + + #endregion + + #region Fatal exceptions + + [Fact] + public async Task Constant_FatalOutOfMemory_ReThrowImmediately() + { + // Arrange + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) + { + attempts++; + throw new OutOfMemoryException(); + } + + // Act & Assert + await Assert.ThrowsAsync(() => + Retry.Constant(Func, 42, retryCount: 3, waitTime: 1, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + + Assert.Equal(1, attempts); + } + + [Fact] + public async Task Constant_FatalStackOverflow_ReThrowImmediately() + { + // Arrange + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) + { + attempts++; + throw new StackOverflowException(); + } + + // Act & Assert + await Assert.ThrowsAsync(() => + Retry.Constant(Func, 42, retryCount: 3, waitTime: 1, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + + Assert.Equal(1, attempts); + } + + [Fact] + public async Task Constant_FatalInvalidProgram_ReThrowImmediately() + { + // Arrange + int attempts = 0; + ValueTask Func(int input, CancellationToken ct) + { + attempts++; + throw new InvalidProgramException(); + } + + // Act & Assert + await Assert.ThrowsAsync(() => + Retry.Constant(Func, 42, retryCount: 3, waitTime: 1, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + + Assert.Equal(1, attempts); + } + + [Theory] + [InlineData(typeof(OutOfMemoryException), true)] + [InlineData(typeof(TransientException), false)] + public void IsFatal_ReturnsExpected(Type exceptionType, bool expected) + { + // Arrange / Act + var ex = (Exception)Activator.CreateInstance(exceptionType)!; + var actual = Retry.IsFatal(ex); + + // Assert + Assert.Equal(expected, actual); + } + + #endregion + + #region Quartz generators + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(3)] + [InlineData(10)] + public void GenerateConstant_CountMatchesRetryCount(int retryCount) + { + // Act + var delays = Retry.Quartz.GenerateConstant(TimeSpan.FromMilliseconds(100), retryCount, fastFirst: true).ToList(); + + // Assert + Assert.Equal(retryCount, delays.Count); + } + + [Fact] + public void GenerateConstant_FirstElement_Zero_WhenFastFirst() + { + // Act + var delays = Retry.Quartz.GenerateConstant(TimeSpan.FromMilliseconds(500), 3, fastFirst: true).ToList(); + + // Assert + Assert.Equal(TimeSpan.Zero, delays[0]); + for (int i = 1; i < delays.Count; i++) + Assert.Equal(TimeSpan.FromMilliseconds(500), delays[i]); + } + + [Fact] + public void GenerateConstant_NoFastFirst_NoZero() + { + // Act + var delays = Retry.Quartz.GenerateConstant(TimeSpan.FromMilliseconds(500), 3, fastFirst: false).ToList(); + + // Assert + Assert.NotEqual(TimeSpan.Zero, delays[0]); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(5)] + public void GenerateLinear_CountMatchesRetryCount(int retryCount) + { + // Act + var delays = Retry.Quartz.GenerateLinear(TimeSpan.FromMilliseconds(100), retryCount, factor: 1.0, fastFirst: true).ToList(); + + // Assert + Assert.Equal(retryCount, delays.Count); + } + + [Fact] + public void GenerateLinear_IncreasesLinearly() + { + // Act + var delays = Retry.Quartz.GenerateLinear(TimeSpan.FromMilliseconds(100), 4, factor: 1.0, fastFirst: true).ToList(); + + // Assert + Assert.Equal(TimeSpan.Zero, delays[0]); + Assert.Equal(TimeSpan.FromMilliseconds(100), delays[1]); + Assert.Equal(TimeSpan.FromMilliseconds(200), delays[2]); + Assert.Equal(TimeSpan.FromMilliseconds(300), delays[3]); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(5)] + public void GenerateExponential_CountMatchesRetryCount(int retryCount) + { + // Act + var delays = Retry.Quartz.GenerateExponential(TimeSpan.FromMilliseconds(100), retryCount, factor: 2.0, fastFirst: true).ToList(); + + // Assert + Assert.Equal(retryCount, delays.Count); + } + + [Fact] + public void GenerateExponential_GrowsByFactor() + { + // Act + var delays = Retry.Quartz.GenerateExponential(TimeSpan.FromMilliseconds(100), 4, factor: 2.0, fastFirst: true).ToList(); + + // Assert + Assert.Equal(TimeSpan.Zero, delays[0]); + Assert.Equal(TimeSpan.FromMilliseconds(100), delays[1]); + Assert.Equal(TimeSpan.FromMilliseconds(200), delays[2]); + Assert.Equal(TimeSpan.FromMilliseconds(400), delays[3]); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(5)] + public void GenerateJitter_CountMatchesRetryCount(int retryCount) + { + // Act + var delays = Retry.Quartz.GenerateJitter(TimeSpan.FromMilliseconds(530), retryCount, fastFirst: true).ToList(); + + // Assert + Assert.Equal(retryCount, delays.Count); + } + + [Fact] + public void GenerateJitter_AllPositiveExceptFirst() + { + // Act + var delays = Retry.Quartz.GenerateJitter(TimeSpan.FromMilliseconds(530), 5, fastFirst: true).ToList(); + + // Assert + Assert.Equal(TimeSpan.Zero, delays[0]); + for (int i = 1; i < delays.Count; i++) + Assert.True(delays[i] > TimeSpan.Zero, $"delay[{i}] should be positive"); + } + + [Theory] + [InlineData(-1)] + [InlineData(-100)] + public void GenerateConstant_NegativeDelay_ThrowsArgumentOutOfRangeException(int ms) + { + var ex = Assert.Throws(() => + Retry.Quartz.GenerateConstant(TimeSpan.FromMilliseconds(ms), 3)); + Assert.Equal("delay", ex.ParamName); + } + + [Theory] + [InlineData(-1)] + public void GenerateConstant_NegativeRetryCount_ThrowsArgumentOutOfRangeException(int count) + { + var ex = Assert.Throws(() => + Retry.Quartz.GenerateConstant(TimeSpan.FromMilliseconds(100), count)); + Assert.Equal("retryCount", ex.ParamName); + } + + [Theory] + [InlineData(0.5)] + public void GenerateExponential_FactorLessThanOne_ThrowsArgumentOutOfRangeException(double factor) + { + var ex = Assert.Throws(() => + Retry.Quartz.GenerateExponential(TimeSpan.FromMilliseconds(100), 3, factor)); + Assert.Equal("factor", ex.ParamName); + } + + [Theory] + [InlineData(-1.0)] + public void GenerateLinear_NegativeFactor_ThrowsArgumentOutOfRangeException(double factor) + { + var ex = Assert.Throws(() => + Retry.Quartz.GenerateLinear(TimeSpan.FromMilliseconds(100), 3, factor)); + Assert.Equal("factor", ex.ParamName); + } + + #endregion } From e7e1a9dc894d4cd19e539ebb09c4cd214a91f312 Mon Sep 17 00:00:00 2001 From: dundich Date: Thu, 2 Jul 2026 13:08:38 +0300 Subject: [PATCH 33/33] chore: bump all library versions to 0.10.0 --- .../Sa.Configuration.PostgreSql.csproj | 2 +- src/Sa.Configuration/Sa.Configuration.csproj | 2 +- src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj | 2 +- src/Sa.Data.S3/Sa.Data.S3.csproj | 2 +- .../Sa.HybridFileStorage.FileSystem.csproj | 2 +- .../Sa.HybridFileStorage.Postgres.csproj | 2 +- src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj | 2 +- src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj | 2 +- src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj | 2 +- src/Sa.Media/Sa.Media.csproj | 2 +- src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj | 2 +- src/Sa.Outbox/Sa.Outbox.csproj | 2 +- src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj | 2 +- src/Sa.Schedule/Sa.Schedule.csproj | 2 +- src/Sa.Utils.WorkQueue/Sa.Utils.WorkQueue.csproj | 2 +- src/Sa/Sa.csproj | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Sa.Configuration.PostgreSql/Sa.Configuration.PostgreSql.csproj b/src/Sa.Configuration.PostgreSql/Sa.Configuration.PostgreSql.csproj index af0b5247..546a04fe 100644 --- a/src/Sa.Configuration.PostgreSql/Sa.Configuration.PostgreSql.csproj +++ b/src/Sa.Configuration.PostgreSql/Sa.Configuration.PostgreSql.csproj @@ -3,7 +3,7 @@ - 0.9.1 + 0.10.0 add a PostgreSQL-based configuration source to IConfiguration diff --git a/src/Sa.Configuration/Sa.Configuration.csproj b/src/Sa.Configuration/Sa.Configuration.csproj index 6041109a..4258918b 100644 --- a/src/Sa.Configuration/Sa.Configuration.csproj +++ b/src/Sa.Configuration/Sa.Configuration.csproj @@ -3,7 +3,7 @@ - 0.9.1 + 0.10.0 extensions for Configuration diff --git a/src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj b/src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj index 1862684d..e3e4f2ba 100644 --- a/src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj +++ b/src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj @@ -3,7 +3,7 @@ - 0.9.0 + 0.10.0 Simple client for Npqsql diff --git a/src/Sa.Data.S3/Sa.Data.S3.csproj b/src/Sa.Data.S3/Sa.Data.S3.csproj index 8bb96edb..c52b3843 100644 --- a/src/Sa.Data.S3/Sa.Data.S3.csproj +++ b/src/Sa.Data.S3/Sa.Data.S3.csproj @@ -3,7 +3,7 @@ - 0.9.0 + 0.10.0 Sa.Data.S3 Simple client for S3 (Sa.Data.S3) s3 diff --git a/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj b/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj index 4bdff2e9..569c3ced 100644 --- a/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj +++ b/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj @@ -3,7 +3,7 @@ - 0.9.1 + 0.10.0 File storage management diff --git a/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj b/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj index 4bb00065..e517a8ec 100644 --- a/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj +++ b/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj @@ -3,7 +3,7 @@ - 0.9.1 + 0.10.0 File storage management in Pg diff --git a/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj b/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj index 8923c345..7c7e4314 100644 --- a/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj +++ b/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj @@ -3,7 +3,7 @@ - 0.9.1 + 0.10.0 File storage management in S3 diff --git a/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj b/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj index e8c9630d..4a6e54e6 100644 --- a/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj +++ b/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj @@ -3,7 +3,7 @@ - 0.9.1 + 0.10.0 File storage management diff --git a/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj b/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj index 343e86cc..044a9652 100644 --- a/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj +++ b/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj @@ -3,7 +3,7 @@ - 0.9.1 + 0.10.0 FFmpeg wrapper true win-x64;win-arm64;linux-x64;linux-arm64;osx-x64 diff --git a/src/Sa.Media/Sa.Media.csproj b/src/Sa.Media/Sa.Media.csproj index 83488990..09776e2a 100644 --- a/src/Sa.Media/Sa.Media.csproj +++ b/src/Sa.Media/Sa.Media.csproj @@ -3,7 +3,7 @@ - 0.9.0 + 0.10.0 Async WAV file reader for .NET diff --git a/src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj b/src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj index cd3c74a6..49bdf112 100644 --- a/src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj +++ b/src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj @@ -3,7 +3,7 @@ - 0.9.0 + 0.10.0 Simple Outbox for Pg (publishing and using messages) diff --git a/src/Sa.Outbox/Sa.Outbox.csproj b/src/Sa.Outbox/Sa.Outbox.csproj index d8883459..c0fa7d85 100644 --- a/src/Sa.Outbox/Sa.Outbox.csproj +++ b/src/Sa.Outbox/Sa.Outbox.csproj @@ -3,7 +3,7 @@ - 0.9.1 + 0.10.0 Simple Outbox infra for publishing and using messages diff --git a/src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj b/src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj index a1c52b7c..99304efe 100644 --- a/src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj +++ b/src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj @@ -3,7 +3,7 @@ - 0.9.0 + 0.10.0 For managing table partitioning in PostgreSQL diff --git a/src/Sa.Schedule/Sa.Schedule.csproj b/src/Sa.Schedule/Sa.Schedule.csproj index b2728de6..453cc916 100644 --- a/src/Sa.Schedule/Sa.Schedule.csproj +++ b/src/Sa.Schedule/Sa.Schedule.csproj @@ -3,7 +3,7 @@ - 0.9.1 + 0.10.0 Execute jobs on a schedule diff --git a/src/Sa.Utils.WorkQueue/Sa.Utils.WorkQueue.csproj b/src/Sa.Utils.WorkQueue/Sa.Utils.WorkQueue.csproj index de70e49e..94127a5a 100644 --- a/src/Sa.Utils.WorkQueue/Sa.Utils.WorkQueue.csproj +++ b/src/Sa.Utils.WorkQueue/Sa.Utils.WorkQueue.csproj @@ -3,7 +3,7 @@ - 0.9.0 + 0.10.0 WorkQueue wrapper for Channels diff --git a/src/Sa/Sa.csproj b/src/Sa/Sa.csproj index bb41c3e7..a76f6dcf 100644 --- a/src/Sa/Sa.csproj +++ b/src/Sa/Sa.csproj @@ -3,7 +3,7 @@ - 0.6.0 + 0.10.0